diff --git a/packages/web/server/database/migrations/0001_nosy_barracuda.sql b/packages/web/server/database/migrations/0001_nosy_barracuda.sql new file mode 100644 index 0000000..fcf5b9b --- /dev/null +++ b/packages/web/server/database/migrations/0001_nosy_barracuda.sql @@ -0,0 +1,3 @@ +CREATE INDEX `idx_repo_id_number` ON `issues` (`repo_id`,`number`);--> statement-breakpoint +CREATE INDEX `idx_state` ON `issues` (`state`);--> statement-breakpoint +CREATE UNIQUE INDEX `repos_full_name_unique` ON `repos` (`full_name`); \ No newline at end of file diff --git a/packages/web/server/database/migrations/meta/0001_snapshot.json b/packages/web/server/database/migrations/meta/0001_snapshot.json new file mode 100644 index 0000000..7dc41e3 --- /dev/null +++ b/packages/web/server/database/migrations/meta/0001_snapshot.json @@ -0,0 +1,175 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "55227258-4533-4b49-934a-3eb7e270134a", + "prevId": "8bd5743b-0287-458d-8f76-b2b06a24d1ca", + "tables": { + "issues": { + "name": "issues", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "repo_id": { + "name": "repo_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "number": { + "name": "number", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "embeddings": { + "name": "embeddings", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mtime": { + "name": "mtime", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_repo_id_number": { + "name": "idx_repo_id_number", + "columns": [ + "repo_id", + "number" + ], + "isUnique": false + }, + "idx_state": { + "name": "idx_state", + "columns": [ + "state" + ], + "isUnique": false + } + }, + "foreignKeys": { + "issues_repo_id_repos_id_fk": { + "name": "issues_repo_id_repos_id_fk", + "tableFrom": "issues", + "tableTo": "repos", + "columnsFrom": [ + "repo_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "repos": { + "name": "repos", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "node_id": { + "name": "node_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "full_name": { + "name": "full_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "private": { + "name": "private", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "indexed": { + "name": "indexed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "repos_full_name_unique": { + "name": "repos_full_name_unique", + "columns": [ + "full_name" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} diff --git a/packages/web/server/database/migrations/meta/_journal.json b/packages/web/server/database/migrations/meta/_journal.json index a6e0341..a3eeeed 100644 --- a/packages/web/server/database/migrations/meta/_journal.json +++ b/packages/web/server/database/migrations/meta/_journal.json @@ -8,6 +8,13 @@ "when": 1742739411042, "tag": "0000_clean_hulk", "breakpoints": true + }, + { + "idx": 1, + "version": "6", + "when": 1745789636896, + "tag": "0001_nosy_barracuda", + "breakpoints": true } ] } diff --git a/packages/web/server/database/schema.ts b/packages/web/server/database/schema.ts index 21f3d75..d3eb720 100644 --- a/packages/web/server/database/schema.ts +++ b/packages/web/server/database/schema.ts @@ -1,9 +1,9 @@ -import { integer, sqliteTable, text } from 'drizzle-orm/sqlite-core' +import { index, integer, sqliteTable, text } from 'drizzle-orm/sqlite-core' export const repos = sqliteTable('repos', { id: integer('id').primaryKey(), node_id: text('node_id').notNull(), - full_name: text('full_name').notNull(), + full_name: text('full_name').notNull().unique(), private: integer('private').notNull(), indexed: integer('indexed').notNull().default(0), }) @@ -18,4 +18,7 @@ export const issues = sqliteTable('issues', { embeddings: text('embeddings').notNull(), hash: text('hash').notNull(), mtime: integer('mtime').notNull(), -}) +}, table => [ + index('idx_repo_id_number').on(table.repoId, table.number), + index('idx_state').on(table.state), +]) diff --git a/packages/web/server/utils/embeddings.ts b/packages/web/server/utils/embeddings.ts index f65fb30..a8f0fc3 100644 --- a/packages/web/server/utils/embeddings.ts +++ b/packages/web/server/utils/embeddings.ts @@ -7,6 +7,30 @@ type RestIssue = RestEndpointMethodTypes['issues']['get']['response']['data'] export type IssueSegments = 'labels' | 'title' | 'body' +// Cache for repository IDs to avoid repeated lookups +const repoIdCache = new Map() + +// Function to get repository ID with caching +async function getRepoId(owner: string, repo: string): Promise { + const cacheKey = `${owner}/${repo}`.toLowerCase() + + // Check cache first + if (repoIdCache.has(cacheKey)) { + return repoIdCache.get(cacheKey) as number | null + } + + // Query database if not in cache + const result = await useDrizzle() + .select({ repoId: tables.repos.id }) + .from(tables.repos) + .where(eq(tables.repos.full_name, cacheKey)) + .get() + + const repoId = result?.repoId ?? null + repoIdCache.set(cacheKey, repoId) + return repoId +} + export function storageKeyForIssue(owner: string, repo: string, number: number | string) { return `issue:${owner}:${repo}:${number}` } @@ -16,7 +40,7 @@ export function storagePrefixForRepo(owner: string, repo: string) { } export async function getStoredEmbeddingsForRepo(owner: string, repo: string, state: 'open' | 'closed' | 'all' = 'open') { - const { repoId } = await useDrizzle().select({ repoId: tables.repos.id }).from(tables.repos).where(eq(tables.repos.full_name, `${owner}/${repo}`)).get() || {} + const repoId = await getRepoId(owner, repo) if (!repoId) { return [] } @@ -35,7 +59,7 @@ export async function getStoredEmbeddingsForRepo(owner: string, repo: string, st } export async function removeStoredEmbeddingsForRepo(owner: string, repo: string) { - const { repoId } = await useDrizzle().select({ repoId: tables.repos.id }).from(tables.repos).where(eq(tables.repos.full_name, `${owner}/${repo}`)).get() || {} + const repoId = await getRepoId(owner, repo) if (!repoId) { return } @@ -45,7 +69,7 @@ export async function removeStoredEmbeddingsForRepo(owner: string, repo: string) } export async function removeIssue(issue: Pick, repo: { owner: { login: string }, name: string }) { - const { repoId } = await useDrizzle().select({ repoId: tables.repos.id }).from(tables.repos).where(eq(tables.repos.full_name, `${repo.owner.login}/${repo.name}`)).get() || {} + const repoId = await getRepoId(repo.owner.login, repo.name) if (!repoId) { return } @@ -63,7 +87,7 @@ export async function indexIssue(issue: Issue | RestIssue, repo: { owner: { logi if (issue.pull_request) { return } - const { repoId } = await useDrizzle().select({ repoId: tables.repos.id }).from(tables.repos).where(eq(tables.repos.full_name, `${repo.owner.login}/${repo.name}`)).get() || {} + const repoId = await getRepoId(repo.owner.login, repo.name) if (!repoId) { console.error('repo not found:', repo.owner.login, repo.name) return @@ -89,7 +113,11 @@ export async function indexIssue(issue: Issue | RestIssue, repo: { owner: { logi } const drizzle = useDrizzle() - const res = await drizzle.select().from(tables.issues).where(eq(tables.issues.number, issue.number)).get() + // Use both repoId and number to narrow the query + const res = await drizzle.select().from(tables.issues).where(and( + eq(tables.issues.repoId, repoId), + eq(tables.issues.number, issue.number), + )).get() if (res && (res.mtime <= issueUpdatedTime || res.hash === issueHash)) { return await Promise.all([ @@ -142,6 +170,16 @@ export async function indexIssue(issue: Issue | RestIssue, repo: { owner: { logi return embeddings } +// Clear the repo ID cache when repos are updated +export function clearRepoIdCache(owner?: string, repo?: string) { + if (owner && repo) { + repoIdCache.delete(`${owner}/${repo}`.toLowerCase()) + } + else { + repoIdCache.clear() + } +} + export type Label = string | { name: string, color?: string } export interface IssueMetadata {