From ed7af3988004156e7df3af9b7068692bc31e686f Mon Sep 17 00:00:00 2001 From: Trezy Date: Tue, 17 Feb 2026 14:48:40 +0000 Subject: [PATCH] feat: merge network and local lexicons --- migrations/20260221000000_merge_network_lexicons.sql | 13 +++++++++++++ src/admin/lexicons.rs | 47 ++++++++++++++++++++++++++++++++++++++--------- src/admin/network_lexicons.rs | 44 ++++++++++++-------------------------------- src/admin/types.rs | 3 +++ src/main.rs | 34 ++++++++++++---------------------- src/tap.rs | 17 ++++++----------- tests/common/db.rs | 2 +- tests/e2e_network_lexicons.rs | 44 ++++++++++++-------------------------------- web/package-lock.json | 20 -------------------- web/src/app/(dashboard)/lexicons/page.tsx | 712 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- web/src/app/(dashboard)/network-lexicons/page.tsx | 322 ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- web/src/components/app-sidebar.tsx | 2 -- web/src/components/data-table/data-table-column-header.tsx | 1 + web/src/components/data-table/data-table-faceted-filter.tsx | 1 + web/src/components/data-table/data-table-pagination.tsx | 2 ++ web/src/components/data-table/data-table-toolbar.tsx | 1 + web/src/components/data-table/data-table-view-options.tsx | 1 + web/src/components/data-table/data-table.tsx | 2 ++ web/src/lib/api.ts | 3 +++ web/src/lib/data-table.ts | 2 +- 20 file(s) changed, 637 insertion(s)(+), 636 deletion(s)(-) diff --git a/migrations/20260221000000_merge_network_lexicons.sql b/migrations/20260221000000_merge_network_lexicons.sql new file mode 100644 --- /dev/null +++ b/migrations/20260221000000_merge_network_lexicons.sql @@ -0,0 +1,13 @@ +-- Merge network_lexicons metadata into the lexicons table. +ALTER TABLE lexicons ADD COLUMN source TEXT NOT NULL DEFAULT 'manual'; +ALTER TABLE lexicons ADD COLUMN authority_did TEXT; +ALTER TABLE lexicons ADD COLUMN last_fetched_at TIMESTAMPTZ; + +UPDATE lexicons +SET source = 'network', + authority_did = nl.authority_did, + last_fetched_at = nl.last_fetched_at +FROM network_lexicons nl +WHERE lexicons.id = nl.nsid; + +DROP TABLE network_lexicons; diff --git a/src/admin/lexicons.rs b/src/admin/lexicons.rs --- a/src/admin/lexicons.rs +++ b/src/admin/lexicons.rs @@ -63,13 +63,14 @@ // Upsert into database let row: (i32,) = sqlx::query_as( r#" - INSERT INTO lexicons (id, lexicon_json, backfill, target_collection, action) - VALUES ($1, $2, $3, $4, $5) + INSERT INTO lexicons (id, lexicon_json, backfill, target_collection, action, source) + VALUES ($1, $2, $3, $4, $5, 'manual') ON CONFLICT (id) DO UPDATE SET lexicon_json = EXCLUDED.lexicon_json, backfill = EXCLUDED.backfill, target_collection = EXCLUDED.target_collection, action = EXCLUDED.action, + source = 'manual', revision = lexicons.revision + 1, updated_at = NOW() RETURNING revision @@ -117,9 +118,9 @@ State(state): State, _admin: AdminAuth, ) -> Result>, AppError> { #[allow(clippy::type_complexity)] - let rows: Vec<(String, i32, Value, bool, Option, Option, chrono::DateTime, chrono::DateTime)> = + let rows: Vec<(String, i32, Value, bool, Option, Option, String, Option, Option>, chrono::DateTime, chrono::DateTime)> = sqlx::query_as( - "SELECT id, revision, lexicon_json, backfill, action, target_collection, created_at, updated_at FROM lexicons ORDER BY id", + "SELECT id, revision, lexicon_json, backfill, action, target_collection, source, authority_did, last_fetched_at, created_at, updated_at FROM lexicons ORDER BY id", ) .fetch_all(&state.db) .await @@ -128,7 +129,19 @@ let summaries: Vec = rows .into_iter() .map( - |(id, revision, json, backfill, action, target_collection, created_at, updated_at)| { + |( + id, + revision, + json, + backfill, + action, + target_collection, + source, + authority_did, + last_fetched_at, + created_at, + updated_at, + )| { let lexicon_type = ParsedLexicon::parse(json, revision, None, ProcedureAction::Upsert) .map(|p| format!("{:?}", p.lexicon_type).to_lowercase()) @@ -141,6 +154,9 @@ lexicon_type, backfill, action, target_collection, + source, + authority_did, + last_fetched_at, created_at, updated_at, } @@ -158,17 +174,27 @@ _admin: AdminAuth, Path(id): Path, ) -> Result, AppError> { #[allow(clippy::type_complexity)] - let row: Option<(String, i32, Value, bool, Option, chrono::DateTime, chrono::DateTime)> = + let row: Option<(String, i32, Value, bool, Option, String, Option, Option>, chrono::DateTime, chrono::DateTime)> = sqlx::query_as( - "SELECT id, revision, lexicon_json, backfill, action, created_at, updated_at FROM lexicons WHERE id = $1", + "SELECT id, revision, lexicon_json, backfill, action, source, authority_did, last_fetched_at, created_at, updated_at FROM lexicons WHERE id = $1", ) .bind(&id) .fetch_optional(&state.db) .await .map_err(|e| AppError::Internal(format!("failed to get lexicon: {e}")))?; - let (id, revision, lexicon_json, backfill, action, created_at, updated_at) = - row.ok_or_else(|| AppError::NotFound(format!("lexicon '{id}' not found")))?; + let ( + id, + revision, + lexicon_json, + backfill, + action, + source, + authority_did, + last_fetched_at, + created_at, + updated_at, + ) = row.ok_or_else(|| AppError::NotFound(format!("lexicon '{id}' not found")))?; Ok(Json(serde_json::json!({ "id": id, @@ -176,6 +202,9 @@ "revision": revision, "lexicon_json": lexicon_json, "backfill": backfill, "action": action, + "source": source, + "authority_did": authority_did, + "last_fetched_at": last_fetched_at, "created_at": created_at, "updated_at": updated_at, }))) diff --git a/src/admin/network_lexicons.rs b/src/admin/network_lexicons.rs --- a/src/admin/network_lexicons.rs +++ b/src/admin/network_lexicons.rs @@ -43,32 +43,17 @@ ProcedureAction::Upsert, ) .map_err(|e| AppError::BadRequest(format!("failed to parse lexicon: {e}")))?; - // Insert into network_lexicons table. - sqlx::query( - r#" - INSERT INTO network_lexicons (nsid, authority_did, target_collection, last_fetched_at) - VALUES ($1, $2, $3, NOW()) - ON CONFLICT (nsid) DO UPDATE SET - authority_did = EXCLUDED.authority_did, - target_collection = EXCLUDED.target_collection, - last_fetched_at = NOW() - "#, - ) - .bind(nsid) - .bind(&authority_did) - .bind(&body.target_collection) - .execute(&state.db) - .await - .map_err(|e| AppError::Internal(format!("failed to insert network lexicon: {e}")))?; - - // Upsert into lexicons table. + // Upsert into lexicons table with network source. let row: (i32,) = sqlx::query_as( r#" - INSERT INTO lexicons (id, lexicon_json, backfill, target_collection) - VALUES ($1, $2, false, $3) + INSERT INTO lexicons (id, lexicon_json, backfill, target_collection, source, authority_did, last_fetched_at) + VALUES ($1, $2, false, $3, 'network', $4, NOW()) ON CONFLICT (id) DO UPDATE SET lexicon_json = EXCLUDED.lexicon_json, target_collection = EXCLUDED.target_collection, + source = 'network', + authority_did = EXCLUDED.authority_did, + last_fetched_at = NOW(), revision = lexicons.revision + 1, updated_at = NOW() RETURNING revision @@ -77,9 +62,10 @@ ) .bind(nsid) .bind(&lexicon_json) .bind(&body.target_collection) + .bind(&authority_did) .fetch_one(&state.db) .await - .map_err(|e| AppError::Internal(format!("failed to upsert lexicon: {e}")))?; + .map_err(|e| AppError::Internal(format!("failed to upsert network lexicon: {e}")))?; let revision = row.0; @@ -114,9 +100,9 @@ State(state): State, _admin: AdminAuth, ) -> Result>, AppError> { #[allow(clippy::type_complexity)] - let rows: Vec<(String, String, Option, Option>, chrono::DateTime)> = + let rows: Vec<(String, Option, Option, Option>, chrono::DateTime)> = sqlx::query_as( - "SELECT nsid, authority_did, target_collection, last_fetched_at, created_at FROM network_lexicons ORDER BY nsid", + "SELECT id, authority_did, target_collection, last_fetched_at, created_at FROM lexicons WHERE source = 'network' ORDER BY id", ) .fetch_all(&state.db) .await @@ -128,7 +114,7 @@ .map( |(nsid, authority_did, target_collection, last_fetched_at, created_at)| { NetworkLexiconSummary { nsid, - authority_did, + authority_did: authority_did.unwrap_or_default(), target_collection, last_fetched_at, created_at, @@ -146,7 +132,7 @@ State(state): State, _admin: AdminAuth, Path(nsid): Path, ) -> Result { - let result = sqlx::query("DELETE FROM network_lexicons WHERE nsid = $1") + let result = sqlx::query("DELETE FROM lexicons WHERE id = $1 AND source = 'network'") .bind(&nsid) .execute(&state.db) .await @@ -157,12 +143,6 @@ return Err(AppError::NotFound(format!( "network lexicon '{nsid}' not found" ))); } - - // Also remove from lexicons table and registry. - let _ = sqlx::query("DELETE FROM lexicons WHERE id = $1") - .bind(&nsid) - .execute(&state.db) - .await; state.lexicons.remove(&nsid).await; notify_collections(&state).await; diff --git a/src/admin/types.rs b/src/admin/types.rs --- a/src/admin/types.rs +++ b/src/admin/types.rs @@ -13,6 +13,9 @@ pub(super) lexicon_type: String, pub(super) backfill: bool, pub(super) action: Option, pub(super) target_collection: Option, + pub(super) source: String, + pub(super) authority_did: Option, + pub(super) last_fetched_at: Option>, pub(super) created_at: chrono::DateTime, pub(super) updated_at: chrono::DateTime, } diff --git a/src/main.rs b/src/main.rs --- a/src/main.rs +++ b/src/main.rs @@ -38,11 +38,12 @@ .expect("failed to load lexicons"); // Re-fetch all network lexicons from their respective PDSes. let http = reqwest::Client::new(); - let network_rows: Vec<(String, String, Option)> = - sqlx::query_as("SELECT nsid, authority_did, target_collection FROM network_lexicons") - .fetch_all(&db) - .await - .unwrap_or_default(); + let network_rows: Vec<(String, Option, Option)> = sqlx::query_as( + "SELECT id, authority_did, target_collection FROM lexicons WHERE source = 'network'", + ) + .fetch_all(&db) + .await + .unwrap_or_default(); for (nsid, _authority_did, target_collection) in &network_rows { match resolve_nsid_authority(&http, &config.plc_url, nsid).await { @@ -56,35 +57,24 @@ target_collection.clone(), ProcedureAction::Upsert, ) { Ok(parsed) => { - // Upsert into lexicons table. if let Err(e) = sqlx::query( r#" - INSERT INTO lexicons (id, lexicon_json, backfill, target_collection) - VALUES ($1, $2, false, $3) - ON CONFLICT (id) DO UPDATE SET - lexicon_json = EXCLUDED.lexicon_json, - target_collection = EXCLUDED.target_collection, - revision = lexicons.revision + 1, + UPDATE lexicons + SET lexicon_json = $2, + last_fetched_at = NOW(), + revision = revision + 1, updated_at = NOW() + WHERE id = $1 AND source = 'network' "#, ) .bind(nsid) .bind(&lexicon_json) - .bind(target_collection) .execute(&db) .await { - warn!(nsid, "failed to upsert network lexicon into DB: {e}"); + warn!(nsid, "failed to update network lexicon in DB: {e}"); continue; } - - // Update last_fetched_at. - let _ = sqlx::query( - "UPDATE network_lexicons SET last_fetched_at = NOW() WHERE nsid = $1", - ) - .bind(nsid) - .execute(&db) - .await; lexicons.upsert(parsed).await; info!(nsid, "refreshed network lexicon"); diff --git a/src/tap.rs b/src/tap.rs --- a/src/tap.rs +++ b/src/tap.rs @@ -384,7 +384,7 @@ let nsid = &record.rkey; // Check if this NSID is one we're tracking and the DID matches the authority. let tracked: Option<(Option,)> = sqlx::query_as( - "SELECT target_collection FROM network_lexicons WHERE nsid = $1 AND authority_did = $2", + "SELECT target_collection FROM lexicons WHERE id = $1 AND source = 'network' AND authority_did = $2", ) .bind(nsid) .bind(did) @@ -419,14 +419,15 @@ }; let is_record = parsed.lexicon_type == crate::lexicon::LexiconType::Record; - // Upsert into lexicons table. + // Upsert into lexicons table with last_fetched_at. if let Err(e) = sqlx::query( r#" - INSERT INTO lexicons (id, lexicon_json, backfill, target_collection) - VALUES ($1, $2, false, $3) + INSERT INTO lexicons (id, lexicon_json, backfill, target_collection, source, authority_did, last_fetched_at) + VALUES ($1, $2, false, $3, 'network', $4, NOW()) ON CONFLICT (id) DO UPDATE SET lexicon_json = EXCLUDED.lexicon_json, target_collection = EXCLUDED.target_collection, + last_fetched_at = NOW(), revision = lexicons.revision + 1, updated_at = NOW() "#, @@ -434,19 +435,13 @@ ) .bind(nsid) .bind(rec) .bind(&target_collection) + .bind(did) .execute(db) .await { tracing::warn!(nsid, "failed to upsert lexicon from event: {e}"); return; } - - // Update last_fetched_at. - let _ = - sqlx::query("UPDATE network_lexicons SET last_fetched_at = NOW() WHERE nsid = $1") - .bind(nsid) - .execute(db) - .await; lexicons.upsert(parsed).await; tracing::info!(nsid, "updated network lexicon from tap event"); diff --git a/tests/common/db.rs b/tests/common/db.rs --- a/tests/common/db.rs +++ b/tests/common/db.rs @@ -19,7 +19,7 @@ } /// Truncate all application tables, preserving schema. pub async fn truncate_all(pool: &PgPool) { - sqlx::query("TRUNCATE records, lexicons, backfill_jobs, admins, network_lexicons RESTART IDENTITY CASCADE") + sqlx::query("TRUNCATE records, lexicons, backfill_jobs, admins RESTART IDENTITY CASCADE") .execute(pool) .await .expect("failed to truncate tables"); diff --git a/tests/e2e_network_lexicons.rs b/tests/e2e_network_lexicons.rs --- a/tests/e2e_network_lexicons.rs +++ b/tests/e2e_network_lexicons.rs @@ -41,35 +41,22 @@ } /// Set up mocks for NSID authority resolution: /// - DNS TXT is not mockable in e2e, so we test at the API level by mocking -/// the PLC directory and PDS responses and seeding the network_lexicons table directly. +/// the PLC directory and PDS responses and seeding the lexicons table directly. async fn seed_network_lexicon(app: &TestApp, nsid: &str, authority_did: &str) { - sqlx::query( - r#" - INSERT INTO network_lexicons (nsid, authority_did, last_fetched_at) - VALUES ($1, $2, NOW()) - ON CONFLICT (nsid) DO NOTHING - "#, - ) - .bind(nsid) - .bind(authority_did) - .execute(&app.state.db) - .await - .expect("failed to seed network lexicon"); - - // Also seed the lexicons table so it's consistent. let lexicon_json = fixtures::game_record_lexicon(); sqlx::query( r#" - INSERT INTO lexicons (id, lexicon_json, backfill) - VALUES ($1, $2, false) + INSERT INTO lexicons (id, lexicon_json, backfill, source, authority_did, last_fetched_at) + VALUES ($1, $2, false, 'network', $3, NOW()) ON CONFLICT (id) DO NOTHING "#, ) .bind(nsid) .bind(&lexicon_json) + .bind(authority_did) .execute(&app.state.db) .await - .expect("failed to seed lexicon"); + .expect("failed to seed network lexicon"); } // --------------------------------------------------------------------------- @@ -136,20 +123,13 @@ .unwrap(); assert_eq!(resp.status(), StatusCode::NO_CONTENT); - // Verify network_lexicons table is empty. - let count: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM network_lexicons WHERE nsid = $1") - .bind(nsid) - .fetch_one(&app.state.db) - .await - .unwrap(); - assert_eq!(count.0, 0); - - // Verify lexicons table is also cleaned up. - let count: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM lexicons WHERE id = $1") - .bind(nsid) - .fetch_one(&app.state.db) - .await - .unwrap(); + // Verify lexicon is removed. + let count: (i64,) = + sqlx::query_as("SELECT COUNT(*) FROM lexicons WHERE id = $1 AND source = 'network'") + .bind(nsid) + .fetch_one(&app.state.db) + .await + .unwrap(); assert_eq!(count.0, 0); } diff --git a/web/package-lock.json b/web/package-lock.json --- a/web/package-lock.json +++ b/web/package-lock.json @@ -112,7 +112,6 @@ "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.0.tgz", "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -547,7 +546,6 @@ "version": "6.3.1", "resolved": "https://registry.npmjs.org/@dnd-kit/core/-/core-6.3.1.tgz", "integrity": "sha512-xkGBRQQab4RLwgXxoqETICr6S5JlogafbhNsidmrkVv2YRs5MLwpjoF2qpiGjQt8S9AoxtIV603s0GIUpY5eYQ==", "license": "MIT", - "peer": true, "dependencies": { "@dnd-kit/accessibility": "^3.1.1", "@dnd-kit/utilities": "^3.2.2", @@ -755,7 +753,6 @@ "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -1951,7 +1948,6 @@ "resolved": "https://registry.npmjs.org/@noble/ciphers/-/ciphers-1.3.0.tgz", "integrity": "sha512-2I0gnIVPtfnMw9ee9h1dJG7tp81+8Ob3OJb3Mv37rx5L40/b0i7djjCVvGOVqc9AEIQyvyu1i6ypKdFw8R8gQw==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": "^14.21.3 || >=16" }, @@ -4116,7 +4112,6 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.33.tgz", "integrity": "sha512-Rs1bVAIdBs5gbTIKza/tgpMuG1k3U/UMJLWecIMxNdJFDMzcM5LOiLVRYh3PilWEYDIeUDv7bpiHPLPsbydGcw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~6.21.0" } @@ -4127,7 +4122,6 @@ "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.14.tgz", "integrity": "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "csstype": "^3.2.2" } @@ -4138,7 +4132,6 @@ "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.2.3.tgz", "integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==", "devOptional": true, "license": "MIT", - "peer": true, "peerDependencies": { "@types/react": "^19.2.0" } @@ -4202,7 +4195,6 @@ "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.55.0.tgz", "integrity": "sha512-4z2nCSBfVIMnbuu8uinj+f0o4qOeggYJLbjpPHka3KH1om7e+H9yLKTYgksTaHcGco+NClhhY2vyO3HsMH1RGw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.55.0", "@typescript-eslint/types": "8.55.0", @@ -4716,7 +4708,6 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -5103,7 +5094,6 @@ "resolved": "https://registry.npmjs.org/babel-plugin-react-compiler/-/babel-plugin-react-compiler-1.0.0.tgz", "integrity": "sha512-Ixm8tFfoKKIPYdCCKYTsqv+Fd4IJ0DQqMyEimo+pxUOMUR9cVPlwTrFt9Avu+3cb6Zp3mAzl+t1MrG2fxxKsxw==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/types": "^7.26.0" } @@ -5193,7 +5183,6 @@ "url": "https://github.com/sponsors/ai" } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -6391,7 +6380,6 @@ "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.2.tgz", "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -6577,7 +6565,6 @@ "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.32.0.tgz", "integrity": "sha512-whOE1HFo/qJDyX4SnXzP4N6zOWn79WhnCUY/iDR0mPfQZO8wcYE4JClzI2oZrhBnnMUCBCHZhO6VQyoBU95mZA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@rtsao/scc": "^1.1.0", "array-includes": "^3.1.9", @@ -6897,7 +6884,6 @@ "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", @@ -7636,7 +7622,6 @@ "resolved": "https://registry.npmjs.org/hono/-/hono-4.11.9.tgz", "integrity": "sha512-Eaw2YTGM6WOxA6CXbckaEvslr2Ne4NFsKrvc0v97JD5awbmeBLO5w9Ho9L9kmKonrwF9RJlW6BxT1PVv/agBHQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=16.9.0" } @@ -10192,7 +10177,6 @@ "version": "19.2.3", "resolved": "https://registry.npmjs.org/react/-/react-19.2.3.tgz", "integrity": "sha512-Ku/hhYbVjOQnXDZFv2+RibmLFGwFdeeKHFcOTlrt7xplBnya5OGn/hIRDsqDiSUcfORsDC7MPxwork8jBwsIWA==", "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -10223,7 +10207,6 @@ "version": "19.2.3", "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.3.tgz", "integrity": "sha512-yELu4WmLPw5Mr/lmeEpox5rw3RETacE++JgHqQzd2dg+YbJuat3jH4ingc+WPZhxaoFzdv9y33G+F7Nl5O0GBg==", "license": "MIT", - "peer": true, "dependencies": { "scheduler": "^0.27.0" }, @@ -11495,7 +11478,6 @@ "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -11753,7 +11735,6 @@ "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -12389,7 +12370,6 @@ "version": "4.3.6", "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", "license": "MIT", - "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/web/src/app/(dashboard)/lexicons/page.tsx b/web/src/app/(dashboard)/lexicons/page.tsx --- a/web/src/app/(dashboard)/lexicons/page.tsx +++ b/web/src/app/(dashboard)/lexicons/page.tsx @@ -1,19 +1,38 @@ -"use client" +"use client"; -import { useCallback, useEffect, useMemo, useState } from "react" +import { + type ColumnDef, + type ColumnFiltersState, + type PaginationState, + type SortingState, + type VisibilityState, + getCoreRowModel, + getFacetedRowModel, + getFacetedUniqueValues, + getFilteredRowModel, + getPaginationRowModel, + getSortedRowModel, + useReactTable, +} from "@tanstack/react-table"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; -import { useAuth } from "@/lib/auth-context" +import { useAuth } from "@/lib/auth-context"; import { + addNetworkLexicon, deleteLexicon, + deleteNetworkLexicon, getLexicon, getLexicons, uploadLexicon, type LexiconDetail, type LexiconSummary, -} from "@/lib/api" -import { SiteHeader } from "@/components/site-header" -import { Badge } from "@/components/ui/badge" -import { Button } from "@/components/ui/button" +} from "@/lib/api"; +import { DataTable } from "@/components/data-table/data-table"; +import { DataTableColumnHeader } from "@/components/data-table/data-table-column-header"; +import { DataTableToolbar } from "@/components/data-table/data-table-toolbar"; +import { SiteHeader } from "@/components/site-header"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; import { Dialog, DialogClose, @@ -23,122 +42,240 @@ DialogFooter, DialogHeader, DialogTitle, DialogTrigger, -} from "@/components/ui/dialog" -import { Input } from "@/components/ui/input" -import { Label } from "@/components/ui/label" +} from "@/components/ui/dialog"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue, -} from "@/components/ui/select" -import { Switch } from "@/components/ui/switch" -import { - Table, - TableBody, - TableCell, - TableHead, - TableHeader, - TableRow, -} from "@/components/ui/table" -import { Textarea } from "@/components/ui/textarea" +} from "@/components/ui/select"; +import { Switch } from "@/components/ui/switch"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { Textarea } from "@/components/ui/textarea"; export default function LexiconsPage() { - const { getToken } = useAuth() - const [lexicons, setLexicons] = useState([]) - const [error, setError] = useState(null) - const [viewLexicon, setViewLexicon] = useState(null) + const { getToken } = useAuth(); + const [lexicons, setLexicons] = useState([]); + const [error, setError] = useState(null); + const [viewLexicon, setViewLexicon] = useState(null); const load = useCallback(() => { - getLexicons(getToken).then(setLexicons).catch((e) => setError(e.message)) - }, [getToken]) + getLexicons(getToken) + .then(setLexicons) + .catch((e) => setError(e.message)); + }, [getToken]); useEffect(() => { - load() - }, [load]) + load(); + }, [load]); async function handleView(id: string) { try { - const detail = await getLexicon(getToken, id) - setViewLexicon(detail) + const detail = await getLexicon(getToken, id); + setViewLexicon(detail); } catch (e: unknown) { - setError(e instanceof Error ? e.message : String(e)) + setError(e instanceof Error ? e.message : String(e)); } } - async function handleDelete(id: string) { + async function handleDelete(lex: LexiconSummary) { try { - await deleteLexicon(getToken, id) - load() + if (lex.source === "network") { + await deleteNetworkLexicon(getToken, lex.id); + } else { + await deleteLexicon(getToken, lex.id); + } + load(); } catch (e: unknown) { - setError(e instanceof Error ? e.message : String(e)) + setError(e instanceof Error ? e.message : String(e)); } } + const columns = useMemo[]>( + () => [ + { + id: "id", + accessorKey: "id", + header: ({ column }) => ( + + ), + cell: ({ row }) => ( + {row.original.id} + ), + filterFn: "includesString", + enableColumnFilter: true, + enableSorting: true, + enableHiding: false, + meta: { + label: "ID", + placeholder: "Filter by ID...", + variant: "text", + }, + }, + { + id: "lexicon_type", + accessorKey: "lexicon_type", + header: ({ column }) => ( + + ), + cell: ({ row }) => ( + {row.original.lexicon_type} + ), + filterFn: (row, columnId, filterValue) => { + if (!Array.isArray(filterValue) || filterValue.length === 0) + return true; + return filterValue.includes(row.getValue(columnId)); + }, + enableColumnFilter: true, + enableSorting: true, + meta: { + label: "Type", + variant: "multiSelect", + options: [ + { label: "Record", value: "record" }, + { label: "Query", value: "query" }, + { label: "Procedure", value: "procedure" }, + ], + }, + }, + { + id: "source", + accessorKey: "source", + header: ({ column }) => ( + + ), + cell: ({ row }) => ( + + {row.original.source} + + ), + filterFn: (row, columnId, filterValue) => { + if (!Array.isArray(filterValue) || filterValue.length === 0) + return true; + return filterValue.includes(row.getValue(columnId)); + }, + enableColumnFilter: true, + enableSorting: true, + meta: { + label: "Source", + variant: "select", + options: [ + { label: "Manual", value: "manual" }, + { label: "Network", value: "network" }, + ], + }, + }, + { + id: "action", + accessorKey: "action", + header: ({ column }) => ( + + ), + cell: ({ row }) => row.original.action ?? "--", + enableSorting: true, + }, + { + id: "backfill", + accessorKey: "backfill", + header: ({ column }) => ( + + ), + cell: ({ row }) => (row.original.backfill ? "Yes" : "No"), + enableSorting: true, + }, + { + id: "revision", + accessorKey: "revision", + header: ({ column }) => ( + + ), + cell: ({ row }) => ( + {row.original.revision} + ), + enableSorting: true, + }, + { + id: "actions", + header: () => Actions, + cell: ({ row }) => ( +
+ + +
+ ), + enableSorting: false, + enableHiding: false, + }, + ], + // eslint-disable-next-line react-hooks/exhaustive-deps + [getToken], + ); + + const [sorting, setSorting] = useState([ + { id: "id", desc: false }, + ]); + const [columnFilters, setColumnFilters] = useState([]); + const [columnVisibility, setColumnVisibility] = useState({}); + const [pagination, setPagination] = useState({ + pageIndex: 0, + pageSize: 20, + }); + + const table = useReactTable({ + data: lexicons, + columns, + state: { + sorting, + columnFilters, + columnVisibility, + pagination, + }, + defaultColumn: { + enableColumnFilter: false, + }, + onSortingChange: setSorting, + onColumnFiltersChange: setColumnFilters, + onColumnVisibilityChange: setColumnVisibility, + onPaginationChange: setPagination, + getCoreRowModel: getCoreRowModel(), + getFilteredRowModel: getFilteredRowModel(), + getSortedRowModel: getSortedRowModel(), + getPaginationRowModel: getPaginationRowModel(), + getFacetedRowModel: getFacetedRowModel(), + getFacetedUniqueValues: getFacetedUniqueValues(), + getRowId: (row) => row.id, + }); + return ( <>
{error &&

{error}

} -
-

Uploaded Lexicons

- -
- -
- - - - ID - Type - Action - Backfill - Revision - Actions - - - - {lexicons.length === 0 && ( - - - No lexicons uploaded yet. - - - )} - {lexicons.map((lex) => ( - - {lex.id} - - {lex.lexicon_type} - - {lex.action ?? "--"} - {lex.backfill ? "Yes" : "No"} - {lex.revision} - -
- - -
- - - ))} - -
-
+ + + + + {viewLexicon && ( setViewLexicon(null)}> @@ -146,7 +283,8 @@ {viewLexicon.id} - Revision {viewLexicon.revision} · {viewLexicon.lexicon_type} + Revision {viewLexicon.revision} ·{" "} + {viewLexicon.lexicon_type} 
@@ -157,128 +295,334 @@
)}
- ) + ); } -function UploadDialog({ +// --------------------------------------------------------------------------- +// Unified Add Lexicon dialog +// --------------------------------------------------------------------------- + +function AddLexiconDialog({ getToken, onSuccess, }: { - getToken: () => Promise - onSuccess: () => void + getToken: () => Promise; + onSuccess: () => void; }) { - const [json, setJson] = useState("") - const [targetCollection, setTargetCollection] = useState("") - const [action, setAction] = useState("") - const [backfill, setBackfill] = useState(true) - const [error, setError] = useState(null) - const [open, setOpen] = useState(false) + const [open, setOpen] = useState(false); + const [error, setError] = useState(null); - const mainType = useMemo(() => { + // Local state + const [json, setJson] = useState(""); + const [localTargetCollection, setLocalTargetCollection] = useState(""); + const [action, setAction] = useState(""); + const [backfill, setBackfill] = useState(true); + + // Network state + const [nsid, setNsid] = useState(""); + const [networkTargetCollection, setNetworkTargetCollection] = useState(""); + const [mainType, setMainType] = useState(); + const [resolving, setResolving] = useState(false); + const abortRef = useRef(null); + + const localMainType = useMemo(() => { try { - const parsed = JSON.parse(json) - return parsed?.defs?.main?.type as string | undefined + const parsed = JSON.parse(json); + return parsed?.defs?.main?.type as string | undefined; } catch { - return undefined + return undefined; } - }, [json]) + }, [json]); + + const showLocalTargetCollection = + localMainType === "query" || localMainType === "procedure"; + const showAction = localMainType === "procedure"; + + // Debounced NSID resolution + useEffect(() => { + abortRef.current?.abort(); + setMainType(undefined); + + if (nsid.split(".").length < 3) return; + + const debounce = setTimeout(() => { + const controller = new AbortController(); + abortRef.current = controller; + setResolving(true); - const showTargetCollection = mainType === "query" || mainType === "procedure" - const showAction = mainType === "procedure" + resolveNsidType(nsid, controller.signal) + .then((type) => { + if (!controller.signal.aborted) setMainType(type); + }) + .finally(() => { + if (!controller.signal.aborted) setResolving(false); + }); + }, 500); + + return () => clearTimeout(debounce); + }, [nsid]); + + const showNetworkTargetCollection = + mainType === "query" || mainType === "procedure"; + + function reset() { + setError(null); + setJson(""); + setLocalTargetCollection(""); + setAction(""); + setBackfill(true); + setNsid(""); + setNetworkTargetCollection(""); + setMainType(undefined); + } - async function handleUpload() { - setError(null) + async function handleUploadLocal() { + setError(null); try { - const lexiconJson = JSON.parse(json) + const lexiconJson = JSON.parse(json); await uploadLexicon(getToken, { lexicon_json: lexiconJson, backfill, - target_collection: showTargetCollection - ? targetCollection || undefined + target_collection: showLocalTargetCollection + ? localTargetCollection || undefined : undefined, action: showAction ? action || undefined : undefined, - }) - setJson("") - setTargetCollection("") - setAction("") - setBackfill(true) - setOpen(false) - onSuccess() + }); + reset(); + setOpen(false); + onSuccess(); } catch (e: unknown) { - setError(e instanceof Error ? e.message : String(e)) + setError(e instanceof Error ? e.message : String(e)); + } + } + + async function handleAddNetwork() { + setError(null); + try { + await addNetworkLexicon(getToken, { + nsid, + target_collection: showNetworkTargetCollection + ? networkTargetCollection || undefined + : undefined, + }); + reset(); + setOpen(false); + onSuccess(); + } catch (e: unknown) { + setError(e instanceof Error ? e.message : String(e)); } } return ( - + { + setOpen(v); + if (!v) reset(); + }} + > - + - Upload Lexicon + Add Lexicon - Paste the lexicon JSON document below. + Upload a local lexicon JSON document or track one from the network. -
- {error &&

{error}

} -
- -