From 143c93358c7e863fdb1cf005b606d85c3cf96973 Mon Sep 17 00:00:00 2001 From: Lino Le Van Date: Sun, 3 May 2026 16:11:45 -0700 Subject: [PATCH] fix: the problems with the tap setup --- .gitignore | 1 + api/package.json | 1 + api/src/db/drizzle/0005_fix_raw_records.sql | 3 ++ api/src/db/drizzle/meta/0005_snapshot.json | 1 + api/src/db/drizzle/meta/_journal.json | 7 +++ api/src/db/tables/raw_records.ts | 18 +++++-- api/src/ingest/tap.ts | 56 ++++++++++++++++++--- pnpm-lock.yaml | 36 +++++++++++++ tap/dev.sh | 4 +- 9 files changed, 116 insertions(+), 11 deletions(-) create mode 100644 api/src/db/drizzle/0005_fix_raw_records.sql create mode 100644 api/src/db/drizzle/meta/0005_snapshot.json diff --git a/.gitignore b/.gitignore index 33f1c04..fa0b3b4 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ node_modules tap.db tap.db-shm tap.db-wal +tap.log db/internal www/src/routeTree.gen.ts www/.tanstack diff --git a/api/package.json b/api/package.json index eda8b10..1b78176 100644 --- a/api/package.json +++ b/api/package.json @@ -17,6 +17,7 @@ "build:lexicons": "rm -rf ./src/lexicons && lex build --lexicons ./lexicons --import-ext \".ts\" --out ./src/lexicons" }, "dependencies": { + "@atproto/api": "^0.19.11", "@atproto/identity": "^0.4.12", "@atproto/lex": "^0.0.25", "@atproto/lexicon": "^0.6.2", diff --git a/api/src/db/drizzle/0005_fix_raw_records.sql b/api/src/db/drizzle/0005_fix_raw_records.sql new file mode 100644 index 0000000..ce2aa27 --- /dev/null +++ b/api/src/db/drizzle/0005_fix_raw_records.sql @@ -0,0 +1,3 @@ +ALTER TABLE "raw_records" DROP CONSTRAINT "raw_records_tap_id_unique";--> statement-breakpoint +DROP INDEX "raw_records_did_collection_rkey_index";--> statement-breakpoint +CREATE UNIQUE INDEX "raw_records_did_collection_rkey_rev_index" ON "raw_records" USING btree ("did","collection","rkey","rev"); \ No newline at end of file diff --git a/api/src/db/drizzle/meta/0005_snapshot.json b/api/src/db/drizzle/meta/0005_snapshot.json new file mode 100644 index 0000000..f3418d6 --- /dev/null +++ b/api/src/db/drizzle/meta/0005_snapshot.json @@ -0,0 +1 @@ +{"id":"0c74eb7b-314d-4167-ad96-4e3121c2c949","prevId":"3cf2ee14-564d-4bfe-8b7d-da8a86b8e6bf","version":"7","dialect":"postgresql","tables":{"public.bsky_profile":{"name":"bsky_profile","schema":"","columns":{"did":{"name":"did","type":"text","primaryKey":true,"notNull":true},"rev":{"name":"rev","type":"text","primaryKey":false,"notNull":true},"cid":{"name":"cid","type":"text","primaryKey":false,"notNull":false},"deleted":{"name":"deleted","type":"boolean","primaryKey":false,"notNull":true},"display_name":{"name":"display_name","type":"text","primaryKey":false,"notNull":false},"description":{"name":"description","type":"text","primaryKey":false,"notNull":false},"pronouns":{"name":"pronouns","type":"text","primaryKey":false,"notNull":false},"website":{"name":"website","type":"text","primaryKey":false,"notNull":false},"avatar":{"name":"avatar","type":"text","primaryKey":false,"notNull":false},"banner":{"name":"banner","type":"text","primaryKey":false,"notNull":false}},"indexes":{},"foreignKeys":{},"compositePrimaryKeys":{},"uniqueConstraints":{},"policies":{},"checkConstraints":{"bsky_profile_cid_only_null_if_deleted":{"name":"bsky_profile_cid_only_null_if_deleted","value":"(\"bsky_profile\".\"cid\" is not null or \"bsky_profile\".\"deleted\" = true)"}},"isRLSEnabled":false},"public.raw_records":{"name":"raw_records","schema":"","columns":{"id":{"name":"id","type":"uuidv7","primaryKey":true,"notNull":true},"tap_id":{"name":"tap_id","type":"bigint","primaryKey":false,"notNull":true},"received_at":{"name":"received_at","type":"timestamp with time zone","primaryKey":false,"notNull":true,"default":"now()"},"did":{"name":"did","type":"text","primaryKey":false,"notNull":true},"rev":{"name":"rev","type":"text","primaryKey":false,"notNull":true},"collection":{"name":"collection","type":"text","primaryKey":false,"notNull":true},"rkey":{"name":"rkey","type":"text","primaryKey":false,"notNull":true},"action":{"name":"action","type":"raw_action","typeSchema":"public","primaryKey":false,"notNull":true},"cid":{"name":"cid","type":"text","primaryKey":false,"notNull":false},"record":{"name":"record","type":"jsonb","primaryKey":false,"notNull":false}},"indexes":{"raw_records_did_collection_rkey_rev_index":{"name":"raw_records_did_collection_rkey_rev_index","columns":[{"expression":"did","isExpression":false,"asc":true,"nulls":"last"},{"expression":"collection","isExpression":false,"asc":true,"nulls":"last"},{"expression":"rkey","isExpression":false,"asc":true,"nulls":"last"},{"expression":"rev","isExpression":false,"asc":true,"nulls":"last"}],"isUnique":true,"concurrently":false,"method":"btree","with":{}},"raw_records_collection_index":{"name":"raw_records_collection_index","columns":[{"expression":"collection","isExpression":false,"asc":true,"nulls":"last"}],"isUnique":false,"concurrently":false,"method":"btree","with":{}}},"foreignKeys":{},"compositePrimaryKeys":{},"uniqueConstraints":{},"policies":{},"checkConstraints":{"raw_records_not_null_unless_delete":{"name":"raw_records_not_null_unless_delete","value":"(\"raw_records\".\"action\" = 'delete' or (\"raw_records\".\"cid\" is not null and \"raw_records\".\"record\" is not null))"}},"isRLSEnabled":false},"public.tangled_artifact":{"name":"tangled_artifact","schema":"","columns":{"did":{"name":"did","type":"text","primaryKey":false,"notNull":true},"rev":{"name":"rev","type":"text","primaryKey":false,"notNull":true},"rkey":{"name":"rkey","type":"text","primaryKey":false,"notNull":true},"cid":{"name":"cid","type":"text","primaryKey":false,"notNull":false},"deleted":{"name":"deleted","type":"boolean","primaryKey":false,"notNull":true},"name":{"name":"name","type":"text","primaryKey":false,"notNull":false},"repo":{"name":"repo","type":"text","primaryKey":false,"notNull":false},"repo_did":{"name":"repo_did","type":"text","primaryKey":false,"notNull":false},"tag":{"name":"tag","type":"text","primaryKey":false,"notNull":false},"created_at":{"name":"created_at","type":"timestamp with time zone","primaryKey":false,"notNull":false},"artifact_ref":{"name":"artifact_ref","type":"text","primaryKey":false,"notNull":false},"artifact_size":{"name":"artifact_size","type":"integer","primaryKey":false,"notNull":false},"artifact_mime_type":{"name":"artifact_mime_type","type":"text","primaryKey":false,"notNull":false}},"indexes":{},"foreignKeys":{},"compositePrimaryKeys":{"tangled_artifact_did_rkey_pk":{"name":"tangled_artifact_did_rkey_pk","columns":["did","rkey"]}},"uniqueConstraints":{},"policies":{},"checkConstraints":{"tangled_artifact_cid_only_null_if_deleted":{"name":"tangled_artifact_cid_only_null_if_deleted","value":"(\"tangled_artifact\".\"cid\" is not null or \"tangled_artifact\".\"deleted\" = true)"},"tangled_artifact_non_null_fields":{"name":"tangled_artifact_non_null_fields","value":"(\"tangled_artifact\".\"deleted\" = true or (\"tangled_artifact\".\"name\" is not null and \"tangled_artifact\".\"tag\" is not null and \"tangled_artifact\".\"created_at\" is not null and \"tangled_artifact\".\"artifact_ref\" is not null and \"tangled_artifact\".\"artifact_size\" is not null and \"tangled_artifact\".\"artifact_mime_type\" is not null))"}},"isRLSEnabled":false},"public.tangled_collaborator":{"name":"tangled_collaborator","schema":"","columns":{"did":{"name":"did","type":"text","primaryKey":false,"notNull":true},"rev":{"name":"rev","type":"text","primaryKey":false,"notNull":true},"rkey":{"name":"rkey","type":"text","primaryKey":false,"notNull":true},"cid":{"name":"cid","type":"text","primaryKey":false,"notNull":false},"deleted":{"name":"deleted","type":"boolean","primaryKey":false,"notNull":true},"subject":{"name":"subject","type":"text","primaryKey":false,"notNull":false},"repo":{"name":"repo","type":"text","primaryKey":false,"notNull":false},"repo_did":{"name":"repo_did","type":"text","primaryKey":false,"notNull":false},"created_at":{"name":"created_at","type":"timestamp with time zone","primaryKey":false,"notNull":false}},"indexes":{},"foreignKeys":{},"compositePrimaryKeys":{"tangled_collaborator_did_rkey_pk":{"name":"tangled_collaborator_did_rkey_pk","columns":["did","rkey"]}},"uniqueConstraints":{},"policies":{},"checkConstraints":{"tangled_collaborator_cid_only_null_if_deleted":{"name":"tangled_collaborator_cid_only_null_if_deleted","value":"(\"tangled_collaborator\".\"cid\" is not null or \"tangled_collaborator\".\"deleted\" = true)"},"tangled_collaborator_non_null_fields":{"name":"tangled_collaborator_non_null_fields","value":"(\"tangled_collaborator\".\"deleted\" = true or (\"tangled_collaborator\".\"subject\" is not null and \"tangled_collaborator\".\"created_at\" is not null))"}},"isRLSEnabled":false},"public.tangled_follow":{"name":"tangled_follow","schema":"","columns":{"did":{"name":"did","type":"text","primaryKey":false,"notNull":true},"rev":{"name":"rev","type":"text","primaryKey":false,"notNull":true},"rkey":{"name":"rkey","type":"text","primaryKey":false,"notNull":true},"cid":{"name":"cid","type":"text","primaryKey":false,"notNull":false},"deleted":{"name":"deleted","type":"boolean","primaryKey":false,"notNull":true},"subject":{"name":"subject","type":"text","primaryKey":false,"notNull":false},"created_at":{"name":"created_at","type":"timestamp with time zone","primaryKey":false,"notNull":false}},"indexes":{},"foreignKeys":{},"compositePrimaryKeys":{"tangled_follow_did_rkey_pk":{"name":"tangled_follow_did_rkey_pk","columns":["did","rkey"]}},"uniqueConstraints":{},"policies":{},"checkConstraints":{"tangled_follow_cid_only_null_if_deleted":{"name":"tangled_follow_cid_only_null_if_deleted","value":"(\"tangled_follow\".\"cid\" is not null or \"tangled_follow\".\"deleted\" = true)"},"tangled_follow_non_null_fields":{"name":"tangled_follow_non_null_fields","value":"(\"tangled_follow\".\"deleted\" = true or (\"tangled_follow\".\"subject\" is not null and \"tangled_follow\".\"created_at\" is not null))"}},"isRLSEnabled":false},"public.tangled_issue":{"name":"tangled_issue","schema":"","columns":{"did":{"name":"did","type":"text","primaryKey":false,"notNull":true},"rev":{"name":"rev","type":"text","primaryKey":false,"notNull":true},"rkey":{"name":"rkey","type":"text","primaryKey":false,"notNull":true},"cid":{"name":"cid","type":"text","primaryKey":false,"notNull":false},"deleted":{"name":"deleted","type":"boolean","primaryKey":false,"notNull":true},"repo":{"name":"repo","type":"text","primaryKey":false,"notNull":false},"repo_did":{"name":"repo_did","type":"text","primaryKey":false,"notNull":false},"title":{"name":"title","type":"text","primaryKey":false,"notNull":false},"body":{"name":"body","type":"text","primaryKey":false,"notNull":false},"created_at":{"name":"created_at","type":"timestamp with time zone","primaryKey":false,"notNull":false},"mentions":{"name":"mentions","type":"text[]","primaryKey":false,"notNull":false},"references":{"name":"references","type":"text[]","primaryKey":false,"notNull":false}},"indexes":{},"foreignKeys":{},"compositePrimaryKeys":{"tangled_issue_did_rkey_pk":{"name":"tangled_issue_did_rkey_pk","columns":["did","rkey"]}},"uniqueConstraints":{},"policies":{},"checkConstraints":{"tangled_issue_cid_only_null_if_deleted":{"name":"tangled_issue_cid_only_null_if_deleted","value":"(\"tangled_issue\".\"cid\" is not null or \"tangled_issue\".\"deleted\" = true)"},"tangled_issue_non_null_fields":{"name":"tangled_issue_non_null_fields","value":"(\"tangled_issue\".\"deleted\" = true or (\"tangled_issue\".\"title\" is not null and \"tangled_issue\".\"created_at\" is not null))"}},"isRLSEnabled":false},"public.tangled_issue_comment":{"name":"tangled_issue_comment","schema":"","columns":{"did":{"name":"did","type":"text","primaryKey":false,"notNull":true},"rev":{"name":"rev","type":"text","primaryKey":false,"notNull":true},"rkey":{"name":"rkey","type":"text","primaryKey":false,"notNull":true},"cid":{"name":"cid","type":"text","primaryKey":false,"notNull":false},"deleted":{"name":"deleted","type":"boolean","primaryKey":false,"notNull":true},"issue":{"name":"issue","type":"text","primaryKey":false,"notNull":false},"body":{"name":"body","type":"text","primaryKey":false,"notNull":false},"created_at":{"name":"created_at","type":"timestamp with time zone","primaryKey":false,"notNull":false},"reply_to":{"name":"reply_to","type":"text","primaryKey":false,"notNull":false},"mentions":{"name":"mentions","type":"text[]","primaryKey":false,"notNull":false},"references":{"name":"references","type":"text[]","primaryKey":false,"notNull":false}},"indexes":{},"foreignKeys":{},"compositePrimaryKeys":{"tangled_issue_comment_did_rkey_pk":{"name":"tangled_issue_comment_did_rkey_pk","columns":["did","rkey"]}},"uniqueConstraints":{},"policies":{},"checkConstraints":{"tangled_issue_comment_cid_only_null_if_deleted":{"name":"tangled_issue_comment_cid_only_null_if_deleted","value":"(\"tangled_issue_comment\".\"cid\" is not null or \"tangled_issue_comment\".\"deleted\" = true)"},"tangled_issue_comment_non_null_fields":{"name":"tangled_issue_comment_non_null_fields","value":"(\"tangled_issue_comment\".\"deleted\" = true or (\"tangled_issue_comment\".\"issue\" is not null and \"tangled_issue_comment\".\"body\" is not null and \"tangled_issue_comment\".\"created_at\" is not null))"}},"isRLSEnabled":false},"public.tangled_issue_state":{"name":"tangled_issue_state","schema":"","columns":{"did":{"name":"did","type":"text","primaryKey":false,"notNull":true},"rev":{"name":"rev","type":"text","primaryKey":false,"notNull":true},"rkey":{"name":"rkey","type":"text","primaryKey":false,"notNull":true},"cid":{"name":"cid","type":"text","primaryKey":false,"notNull":false},"deleted":{"name":"deleted","type":"boolean","primaryKey":false,"notNull":true},"issue":{"name":"issue","type":"text","primaryKey":false,"notNull":false},"state":{"name":"state","type":"text","primaryKey":false,"notNull":false}},"indexes":{},"foreignKeys":{},"compositePrimaryKeys":{"tangled_issue_state_did_rkey_pk":{"name":"tangled_issue_state_did_rkey_pk","columns":["did","rkey"]}},"uniqueConstraints":{},"policies":{},"checkConstraints":{"tangled_issue_state_cid_only_null_if_deleted":{"name":"tangled_issue_state_cid_only_null_if_deleted","value":"(\"tangled_issue_state\".\"cid\" is not null or \"tangled_issue_state\".\"deleted\" = true)"},"tangled_issue_state_non_null_fields":{"name":"tangled_issue_state_non_null_fields","value":"(\"tangled_issue_state\".\"deleted\" = true or (\"tangled_issue_state\".\"issue\" is not null and \"tangled_issue_state\".\"state\" is not null))"}},"isRLSEnabled":false},"public.tangled_knot":{"name":"tangled_knot","schema":"","columns":{"did":{"name":"did","type":"text","primaryKey":false,"notNull":true},"rev":{"name":"rev","type":"text","primaryKey":false,"notNull":true},"rkey":{"name":"rkey","type":"text","primaryKey":false,"notNull":true},"cid":{"name":"cid","type":"text","primaryKey":false,"notNull":false},"deleted":{"name":"deleted","type":"boolean","primaryKey":false,"notNull":true},"created_at":{"name":"created_at","type":"timestamp with time zone","primaryKey":false,"notNull":false}},"indexes":{},"foreignKeys":{},"compositePrimaryKeys":{"tangled_knot_did_rkey_pk":{"name":"tangled_knot_did_rkey_pk","columns":["did","rkey"]}},"uniqueConstraints":{},"policies":{},"checkConstraints":{"tangled_knot_cid_only_null_if_deleted":{"name":"tangled_knot_cid_only_null_if_deleted","value":"(\"tangled_knot\".\"cid\" is not null or \"tangled_knot\".\"deleted\" = true)"},"tangled_knot_non_null_fields":{"name":"tangled_knot_non_null_fields","value":"(\"tangled_knot\".\"deleted\" = true or \"tangled_knot\".\"created_at\" is not null)"}},"isRLSEnabled":false},"public.tangled_knot_member":{"name":"tangled_knot_member","schema":"","columns":{"did":{"name":"did","type":"text","primaryKey":false,"notNull":true},"rev":{"name":"rev","type":"text","primaryKey":false,"notNull":true},"rkey":{"name":"rkey","type":"text","primaryKey":false,"notNull":true},"cid":{"name":"cid","type":"text","primaryKey":false,"notNull":false},"deleted":{"name":"deleted","type":"boolean","primaryKey":false,"notNull":true},"subject":{"name":"subject","type":"text","primaryKey":false,"notNull":false},"domain":{"name":"domain","type":"text","primaryKey":false,"notNull":false},"created_at":{"name":"created_at","type":"timestamp with time zone","primaryKey":false,"notNull":false}},"indexes":{},"foreignKeys":{},"compositePrimaryKeys":{"tangled_knot_member_did_rkey_pk":{"name":"tangled_knot_member_did_rkey_pk","columns":["did","rkey"]}},"uniqueConstraints":{},"policies":{},"checkConstraints":{"tangled_knot_member_cid_only_null_if_deleted":{"name":"tangled_knot_member_cid_only_null_if_deleted","value":"(\"tangled_knot_member\".\"cid\" is not null or \"tangled_knot_member\".\"deleted\" = true)"},"tangled_knot_member_non_null_fields":{"name":"tangled_knot_member_non_null_fields","value":"(\"tangled_knot_member\".\"deleted\" = true or (\"tangled_knot_member\".\"subject\" is not null and \"tangled_knot_member\".\"domain\" is not null and \"tangled_knot_member\".\"created_at\" is not null))"}},"isRLSEnabled":false},"public.tangled_label_definition":{"name":"tangled_label_definition","schema":"","columns":{"did":{"name":"did","type":"text","primaryKey":false,"notNull":true},"rev":{"name":"rev","type":"text","primaryKey":false,"notNull":true},"rkey":{"name":"rkey","type":"text","primaryKey":false,"notNull":true},"cid":{"name":"cid","type":"text","primaryKey":false,"notNull":false},"deleted":{"name":"deleted","type":"boolean","primaryKey":false,"notNull":true},"name":{"name":"name","type":"text","primaryKey":false,"notNull":false},"value_type":{"name":"value_type","type":"text","primaryKey":false,"notNull":false},"value_format":{"name":"value_format","type":"text","primaryKey":false,"notNull":false},"value_enum":{"name":"value_enum","type":"text[]","primaryKey":false,"notNull":false},"scope":{"name":"scope","type":"text[]","primaryKey":false,"notNull":false},"color":{"name":"color","type":"text","primaryKey":false,"notNull":false},"created_at":{"name":"created_at","type":"timestamp with time zone","primaryKey":false,"notNull":false},"multiple":{"name":"multiple","type":"boolean","primaryKey":false,"notNull":false}},"indexes":{},"foreignKeys":{},"compositePrimaryKeys":{"tangled_label_definition_did_rkey_pk":{"name":"tangled_label_definition_did_rkey_pk","columns":["did","rkey"]}},"uniqueConstraints":{},"policies":{},"checkConstraints":{"tangled_label_definition_cid_only_null_if_deleted":{"name":"tangled_label_definition_cid_only_null_if_deleted","value":"(\"tangled_label_definition\".\"cid\" is not null or \"tangled_label_definition\".\"deleted\" = true)"},"tangled_label_definition_non_null_fields":{"name":"tangled_label_definition_non_null_fields","value":"(\"tangled_label_definition\".\"deleted\" = true or (\"tangled_label_definition\".\"name\" is not null and \"tangled_label_definition\".\"value_type\" is not null and \"tangled_label_definition\".\"value_format\" is not null and \"tangled_label_definition\".\"scope\" is not null and \"tangled_label_definition\".\"created_at\" is not null))"}},"isRLSEnabled":false},"public.tangled_label_op":{"name":"tangled_label_op","schema":"","columns":{"did":{"name":"did","type":"text","primaryKey":false,"notNull":true},"rev":{"name":"rev","type":"text","primaryKey":false,"notNull":true},"rkey":{"name":"rkey","type":"text","primaryKey":false,"notNull":true},"cid":{"name":"cid","type":"text","primaryKey":false,"notNull":false},"deleted":{"name":"deleted","type":"boolean","primaryKey":false,"notNull":true},"subject":{"name":"subject","type":"text","primaryKey":false,"notNull":false},"performed_at":{"name":"performed_at","type":"timestamp with time zone","primaryKey":false,"notNull":false},"add":{"name":"add","type":"jsonb","primaryKey":false,"notNull":false},"delete":{"name":"delete","type":"jsonb","primaryKey":false,"notNull":false}},"indexes":{},"foreignKeys":{},"compositePrimaryKeys":{"tangled_label_op_did_rkey_pk":{"name":"tangled_label_op_did_rkey_pk","columns":["did","rkey"]}},"uniqueConstraints":{},"policies":{},"checkConstraints":{"tangled_label_op_cid_only_null_if_deleted":{"name":"tangled_label_op_cid_only_null_if_deleted","value":"(\"tangled_label_op\".\"cid\" is not null or \"tangled_label_op\".\"deleted\" = true)"},"tangled_label_op_non_null_fields":{"name":"tangled_label_op_non_null_fields","value":"(\"tangled_label_op\".\"deleted\" = true or (\"tangled_label_op\".\"subject\" is not null and \"tangled_label_op\".\"add\" is not null and \"tangled_label_op\".\"delete\" is not null and \"tangled_label_op\".\"performed_at\" is not null))"}},"isRLSEnabled":false},"public.tangled_pipeline":{"name":"tangled_pipeline","schema":"","columns":{"did":{"name":"did","type":"text","primaryKey":false,"notNull":true},"rev":{"name":"rev","type":"text","primaryKey":false,"notNull":true},"rkey":{"name":"rkey","type":"text","primaryKey":false,"notNull":true},"cid":{"name":"cid","type":"text","primaryKey":false,"notNull":false},"deleted":{"name":"deleted","type":"boolean","primaryKey":false,"notNull":true},"trigger_metadata_kind":{"name":"trigger_metadata_kind","type":"tangled_pipeline_trigger_metadata_kind","typeSchema":"public","primaryKey":false,"notNull":false},"trigger_metadata_repo_knot":{"name":"trigger_metadata_repo_knot","type":"text","primaryKey":false,"notNull":false},"trigger_metadata_repo_did":{"name":"trigger_metadata_repo_did","type":"text","primaryKey":false,"notNull":false},"trigger_metadata_repo_repo_did":{"name":"trigger_metadata_repo_repo_did","type":"text","primaryKey":false,"notNull":false},"trigger_metadata_repo_repo":{"name":"trigger_metadata_repo_repo","type":"text","primaryKey":false,"notNull":false},"trigger_metadata_repo_default_branch":{"name":"trigger_metadata_repo_default_branch","type":"text","primaryKey":false,"notNull":false},"trigger_metadata_push_ref":{"name":"trigger_metadata_push_ref","type":"text","primaryKey":false,"notNull":false},"trigger_metadata_push_new_sha":{"name":"trigger_metadata_push_new_sha","type":"text","primaryKey":false,"notNull":false},"trigger_metadata_push_old_sha":{"name":"trigger_metadata_push_old_sha","type":"text","primaryKey":false,"notNull":false},"trigger_metadata_pull_source_branch":{"name":"trigger_metadata_pull_source_branch","type":"text","primaryKey":false,"notNull":false},"trigger_metadata_pull_target_branch":{"name":"trigger_metadata_pull_target_branch","type":"text","primaryKey":false,"notNull":false},"trigger_metadata_pull_source_sha":{"name":"trigger_metadata_pull_source_sha","type":"text","primaryKey":false,"notNull":false},"trigger_metadata_pull_action":{"name":"trigger_metadata_pull_action","type":"text","primaryKey":false,"notNull":false},"trigger_metadata_manual":{"name":"trigger_metadata_manual","type":"jsonb","primaryKey":false,"notNull":false},"workflows":{"name":"workflows","type":"jsonb","primaryKey":false,"notNull":false}},"indexes":{},"foreignKeys":{},"compositePrimaryKeys":{"tangled_pipeline_did_rkey_pk":{"name":"tangled_pipeline_did_rkey_pk","columns":["did","rkey"]}},"uniqueConstraints":{},"policies":{},"checkConstraints":{"tangled_pipeline_cid_only_null_if_deleted":{"name":"tangled_pipeline_cid_only_null_if_deleted","value":"(\"tangled_pipeline\".\"cid\" is not null or \"tangled_pipeline\".\"deleted\" = true)"},"tangled_pipeline_non_null_fields":{"name":"tangled_pipeline_non_null_fields","value":"(\"tangled_pipeline\".\"deleted\" = true or (\"tangled_pipeline\".\"trigger_metadata_kind\" is not null and \"tangled_pipeline\".\"trigger_metadata_repo_knot\" is not null and \"tangled_pipeline\".\"trigger_metadata_repo_did\" is not null and \"tangled_pipeline\".\"trigger_metadata_repo_default_branch\" is not null and \"tangled_pipeline\".\"workflows\" is not null))"}},"isRLSEnabled":false},"public.tangled_pipeline_status":{"name":"tangled_pipeline_status","schema":"","columns":{"did":{"name":"did","type":"text","primaryKey":false,"notNull":true},"rev":{"name":"rev","type":"text","primaryKey":false,"notNull":true},"rkey":{"name":"rkey","type":"text","primaryKey":false,"notNull":true},"cid":{"name":"cid","type":"text","primaryKey":false,"notNull":false},"deleted":{"name":"deleted","type":"boolean","primaryKey":false,"notNull":true},"pipeline":{"name":"pipeline","type":"text","primaryKey":false,"notNull":false},"workflow":{"name":"workflow","type":"text","primaryKey":false,"notNull":false},"status":{"name":"status","type":"tangled_pipeline_status_status","typeSchema":"public","primaryKey":false,"notNull":false},"created_at":{"name":"created_at","type":"timestamp with time zone","primaryKey":false,"notNull":false},"error":{"name":"error","type":"text","primaryKey":false,"notNull":false},"exit_code":{"name":"exit_code","type":"integer","primaryKey":false,"notNull":false}},"indexes":{},"foreignKeys":{},"compositePrimaryKeys":{"tangled_pipeline_status_did_rkey_pk":{"name":"tangled_pipeline_status_did_rkey_pk","columns":["did","rkey"]}},"uniqueConstraints":{},"policies":{},"checkConstraints":{"tangled_pipeline_status_cid_only_null_if_deleted":{"name":"tangled_pipeline_status_cid_only_null_if_deleted","value":"(\"tangled_pipeline_status\".\"cid\" is not null or \"tangled_pipeline_status\".\"deleted\" = true)"},"tangled_pipeline_status_non_null_fields":{"name":"tangled_pipeline_status_non_null_fields","value":"(\"tangled_pipeline_status\".\"deleted\" = true or (\"tangled_pipeline_status\".\"pipeline\" is not null and \"tangled_pipeline_status\".\"workflow\" is not null and \"tangled_pipeline_status\".\"status\" is not null and \"tangled_pipeline_status\".\"created_at\" is not null))"}},"isRLSEnabled":false},"public.tangled_profile":{"name":"tangled_profile","schema":"","columns":{"did":{"name":"did","type":"text","primaryKey":true,"notNull":true},"rev":{"name":"rev","type":"text","primaryKey":false,"notNull":true},"cid":{"name":"cid","type":"text","primaryKey":false,"notNull":false},"deleted":{"name":"deleted","type":"boolean","primaryKey":false,"notNull":true},"avatar":{"name":"avatar","type":"text","primaryKey":false,"notNull":false},"description":{"name":"description","type":"text","primaryKey":false,"notNull":false},"links":{"name":"links","type":"text[]","primaryKey":false,"notNull":false},"stats":{"name":"stats","type":"text[]","primaryKey":false,"notNull":false},"bluesky":{"name":"bluesky","type":"boolean","primaryKey":false,"notNull":false},"location":{"name":"location","type":"text","primaryKey":false,"notNull":false},"pinned_repositories":{"name":"pinned_repositories","type":"text[]","primaryKey":false,"notNull":false},"pronouns":{"name":"pronouns","type":"text","primaryKey":false,"notNull":false},"preferred_handle":{"name":"preferred_handle","type":"text","primaryKey":false,"notNull":false}},"indexes":{},"foreignKeys":{},"compositePrimaryKeys":{},"uniqueConstraints":{},"policies":{},"checkConstraints":{"tangled_profile_cid_only_null_if_deleted":{"name":"tangled_profile_cid_only_null_if_deleted","value":"(\"tangled_profile\".\"cid\" is not null or \"tangled_profile\".\"deleted\" = true)"},"tangled_profile_non_null_fields":{"name":"tangled_profile_non_null_fields","value":"(\"tangled_profile\".\"deleted\" = true or \"tangled_profile\".\"bluesky\" is not null)"}},"isRLSEnabled":false},"public.tangled_public_key":{"name":"tangled_public_key","schema":"","columns":{"did":{"name":"did","type":"text","primaryKey":false,"notNull":true},"rev":{"name":"rev","type":"text","primaryKey":false,"notNull":true},"rkey":{"name":"rkey","type":"text","primaryKey":false,"notNull":true},"cid":{"name":"cid","type":"text","primaryKey":false,"notNull":false},"deleted":{"name":"deleted","type":"boolean","primaryKey":false,"notNull":true},"key":{"name":"key","type":"text","primaryKey":false,"notNull":false},"name":{"name":"name","type":"text","primaryKey":false,"notNull":false},"created_at":{"name":"created_at","type":"timestamp with time zone","primaryKey":false,"notNull":false}},"indexes":{},"foreignKeys":{},"compositePrimaryKeys":{"tangled_public_key_did_rkey_pk":{"name":"tangled_public_key_did_rkey_pk","columns":["did","rkey"]}},"uniqueConstraints":{},"policies":{},"checkConstraints":{"tangled_public_key_cid_only_null_if_deleted":{"name":"tangled_public_key_cid_only_null_if_deleted","value":"(\"tangled_public_key\".\"cid\" is not null or \"tangled_public_key\".\"deleted\" = true)"},"tangled_public_key_non_null_fields":{"name":"tangled_public_key_non_null_fields","value":"(\"tangled_public_key\".\"deleted\" = true or (\"tangled_public_key\".\"key\" is not null and \"tangled_public_key\".\"name\" is not null and \"tangled_public_key\".\"created_at\" is not null))"}},"isRLSEnabled":false},"public.tangled_pull":{"name":"tangled_pull","schema":"","columns":{"did":{"name":"did","type":"text","primaryKey":false,"notNull":true},"rev":{"name":"rev","type":"text","primaryKey":false,"notNull":true},"rkey":{"name":"rkey","type":"text","primaryKey":false,"notNull":true},"cid":{"name":"cid","type":"text","primaryKey":false,"notNull":false},"deleted":{"name":"deleted","type":"boolean","primaryKey":false,"notNull":true},"title":{"name":"title","type":"text","primaryKey":false,"notNull":false},"body":{"name":"body","type":"text","primaryKey":false,"notNull":false},"rounds":{"name":"rounds","type":"jsonb","primaryKey":false,"notNull":false},"source_branch":{"name":"source_branch","type":"text","primaryKey":false,"notNull":false},"source_repo":{"name":"source_repo","type":"text","primaryKey":false,"notNull":false},"source_repo_did":{"name":"source_repo_did","type":"text","primaryKey":false,"notNull":false},"target_branch":{"name":"target_branch","type":"text","primaryKey":false,"notNull":false},"target_repo":{"name":"target_repo","type":"text","primaryKey":false,"notNull":false},"target_repo_did":{"name":"target_repo_did","type":"text","primaryKey":false,"notNull":false},"created_at":{"name":"created_at","type":"timestamp with time zone","primaryKey":false,"notNull":false},"mentions":{"name":"mentions","type":"text[]","primaryKey":false,"notNull":false},"references":{"name":"references","type":"text[]","primaryKey":false,"notNull":false},"dependent_on":{"name":"dependent_on","type":"text","primaryKey":false,"notNull":false}},"indexes":{},"foreignKeys":{},"compositePrimaryKeys":{"tangled_pull_did_rkey_pk":{"name":"tangled_pull_did_rkey_pk","columns":["did","rkey"]}},"uniqueConstraints":{},"policies":{},"checkConstraints":{"tangled_pull_cid_only_null_if_deleted":{"name":"tangled_pull_cid_only_null_if_deleted","value":"(\"tangled_pull\".\"cid\" is not null or \"tangled_pull\".\"deleted\" = true)"},"tangled_pull_non_null_fields":{"name":"tangled_pull_non_null_fields","value":"(\"tangled_pull\".\"deleted\" = true or (\"tangled_pull\".\"target_branch\" is not null and \"tangled_pull\".\"title\" is not null and \"tangled_pull\".\"created_at\" is not null and \"tangled_pull\".\"rounds\" is not null))"}},"isRLSEnabled":false},"public.tangled_pull_comment":{"name":"tangled_pull_comment","schema":"","columns":{"did":{"name":"did","type":"text","primaryKey":false,"notNull":true},"rev":{"name":"rev","type":"text","primaryKey":false,"notNull":true},"rkey":{"name":"rkey","type":"text","primaryKey":false,"notNull":true},"cid":{"name":"cid","type":"text","primaryKey":false,"notNull":false},"deleted":{"name":"deleted","type":"boolean","primaryKey":false,"notNull":true},"pull":{"name":"pull","type":"text","primaryKey":false,"notNull":false},"body":{"name":"body","type":"text","primaryKey":false,"notNull":false},"created_at":{"name":"created_at","type":"timestamp with time zone","primaryKey":false,"notNull":false},"mentions":{"name":"mentions","type":"text[]","primaryKey":false,"notNull":false},"references":{"name":"references","type":"text[]","primaryKey":false,"notNull":false}},"indexes":{},"foreignKeys":{},"compositePrimaryKeys":{"tangled_pull_comment_did_rkey_pk":{"name":"tangled_pull_comment_did_rkey_pk","columns":["did","rkey"]}},"uniqueConstraints":{},"policies":{},"checkConstraints":{"tangled_pull_comment_cid_only_null_if_deleted":{"name":"tangled_pull_comment_cid_only_null_if_deleted","value":"(\"tangled_pull_comment\".\"cid\" is not null or \"tangled_pull_comment\".\"deleted\" = true)"},"tangled_pull_comment_non_null_fields":{"name":"tangled_pull_comment_non_null_fields","value":"(\"tangled_pull_comment\".\"deleted\" = true or (\"tangled_pull_comment\".\"pull\" is not null and \"tangled_pull_comment\".\"body\" is not null and \"tangled_pull_comment\".\"created_at\" is not null))"}},"isRLSEnabled":false},"public.tangled_pull_status":{"name":"tangled_pull_status","schema":"","columns":{"did":{"name":"did","type":"text","primaryKey":false,"notNull":true},"rev":{"name":"rev","type":"text","primaryKey":false,"notNull":true},"rkey":{"name":"rkey","type":"text","primaryKey":false,"notNull":true},"cid":{"name":"cid","type":"text","primaryKey":false,"notNull":false},"deleted":{"name":"deleted","type":"boolean","primaryKey":false,"notNull":true},"pull":{"name":"pull","type":"text","primaryKey":false,"notNull":false},"status":{"name":"status","type":"text","primaryKey":false,"notNull":false}},"indexes":{},"foreignKeys":{},"compositePrimaryKeys":{"tangled_pull_status_did_rkey_pk":{"name":"tangled_pull_status_did_rkey_pk","columns":["did","rkey"]}},"uniqueConstraints":{},"policies":{},"checkConstraints":{"tangled_pull_status_cid_only_null_if_deleted":{"name":"tangled_pull_status_cid_only_null_if_deleted","value":"(\"tangled_pull_status\".\"cid\" is not null or \"tangled_pull_status\".\"deleted\" = true)"},"tangled_pull_status_non_null_fields":{"name":"tangled_pull_status_non_null_fields","value":"(\"tangled_pull_status\".\"deleted\" = true or (\"tangled_pull_status\".\"pull\" is not null and \"tangled_pull_status\".\"status\" is not null))"}},"isRLSEnabled":false},"public.tangled_reaction":{"name":"tangled_reaction","schema":"","columns":{"did":{"name":"did","type":"text","primaryKey":false,"notNull":true},"rev":{"name":"rev","type":"text","primaryKey":false,"notNull":true},"rkey":{"name":"rkey","type":"text","primaryKey":false,"notNull":true},"cid":{"name":"cid","type":"text","primaryKey":false,"notNull":false},"deleted":{"name":"deleted","type":"boolean","primaryKey":false,"notNull":true},"subject":{"name":"subject","type":"text","primaryKey":false,"notNull":false},"reaction":{"name":"reaction","type":"text","primaryKey":false,"notNull":false},"created_at":{"name":"created_at","type":"timestamp with time zone","primaryKey":false,"notNull":false}},"indexes":{},"foreignKeys":{},"compositePrimaryKeys":{"tangled_reaction_did_rkey_pk":{"name":"tangled_reaction_did_rkey_pk","columns":["did","rkey"]}},"uniqueConstraints":{},"policies":{},"checkConstraints":{"tangled_reaction_cid_only_null_if_deleted":{"name":"tangled_reaction_cid_only_null_if_deleted","value":"(\"tangled_reaction\".\"cid\" is not null or \"tangled_reaction\".\"deleted\" = true)"},"tangled_reaction_non_null_fields":{"name":"tangled_reaction_non_null_fields","value":"(\"tangled_reaction\".\"deleted\" = true or (\"tangled_reaction\".\"subject\" is not null and \"tangled_reaction\".\"reaction\" is not null and \"tangled_reaction\".\"created_at\" is not null))"}},"isRLSEnabled":false},"public.tangled_ref_update":{"name":"tangled_ref_update","schema":"","columns":{"did":{"name":"did","type":"text","primaryKey":false,"notNull":true},"rev":{"name":"rev","type":"text","primaryKey":false,"notNull":true},"rkey":{"name":"rkey","type":"text","primaryKey":false,"notNull":true},"cid":{"name":"cid","type":"text","primaryKey":false,"notNull":false},"deleted":{"name":"deleted","type":"boolean","primaryKey":false,"notNull":true},"ref":{"name":"ref","type":"text","primaryKey":false,"notNull":false},"committer_did":{"name":"committer_did","type":"text","primaryKey":false,"notNull":false},"owner_did":{"name":"owner_did","type":"text","primaryKey":false,"notNull":false},"repo_did":{"name":"repo_did","type":"text","primaryKey":false,"notNull":false},"repo_name":{"name":"repo_name","type":"text","primaryKey":false,"notNull":false},"old_sha":{"name":"old_sha","type":"text","primaryKey":false,"notNull":false},"new_sha":{"name":"new_sha","type":"text","primaryKey":false,"notNull":false},"meta_is_default_ref":{"name":"meta_is_default_ref","type":"boolean","primaryKey":false,"notNull":false},"meta_lang_breakdown":{"name":"meta_lang_breakdown","type":"jsonb","primaryKey":false,"notNull":false},"meta_commit_count":{"name":"meta_commit_count","type":"jsonb","primaryKey":false,"notNull":false}},"indexes":{},"foreignKeys":{},"compositePrimaryKeys":{"tangled_ref_update_did_rkey_pk":{"name":"tangled_ref_update_did_rkey_pk","columns":["did","rkey"]}},"uniqueConstraints":{},"policies":{},"checkConstraints":{"tangled_ref_update_cid_only_null_if_deleted":{"name":"tangled_ref_update_cid_only_null_if_deleted","value":"(\"tangled_ref_update\".\"cid\" is not null or \"tangled_ref_update\".\"deleted\" = true)"},"tangled_ref_update_non_null_fields":{"name":"tangled_ref_update_non_null_fields","value":"(\"tangled_ref_update\".\"deleted\" = true or (\"tangled_ref_update\".\"ref\" is not null and \"tangled_ref_update\".\"committer_did\" is not null and \"tangled_ref_update\".\"repo_name\" is not null and \"tangled_ref_update\".\"old_sha\" is not null and \"tangled_ref_update\".\"new_sha\" is not null and \"tangled_ref_update\".\"meta_is_default_ref\" is not null and \"tangled_ref_update\".\"meta_commit_count\" is not null))"}},"isRLSEnabled":false},"public.tangled_repo":{"name":"tangled_repo","schema":"","columns":{"did":{"name":"did","type":"text","primaryKey":false,"notNull":true},"rev":{"name":"rev","type":"text","primaryKey":false,"notNull":true},"rkey":{"name":"rkey","type":"text","primaryKey":false,"notNull":true},"cid":{"name":"cid","type":"text","primaryKey":false,"notNull":false},"deleted":{"name":"deleted","type":"boolean","primaryKey":false,"notNull":true},"name":{"name":"name","type":"text","primaryKey":false,"notNull":false},"knot":{"name":"knot","type":"text","primaryKey":false,"notNull":false},"spindle":{"name":"spindle","type":"text","primaryKey":false,"notNull":false},"description":{"name":"description","type":"text","primaryKey":false,"notNull":false},"website":{"name":"website","type":"text","primaryKey":false,"notNull":false},"topics":{"name":"topics","type":"text[]","primaryKey":false,"notNull":false},"source":{"name":"source","type":"text","primaryKey":false,"notNull":false},"labels":{"name":"labels","type":"text[]","primaryKey":false,"notNull":false},"repo_did":{"name":"repo_did","type":"text","primaryKey":false,"notNull":false},"created_at":{"name":"created_at","type":"timestamp with time zone","primaryKey":false,"notNull":false}},"indexes":{},"foreignKeys":{},"compositePrimaryKeys":{"tangled_repo_did_rkey_pk":{"name":"tangled_repo_did_rkey_pk","columns":["did","rkey"]}},"uniqueConstraints":{},"policies":{},"checkConstraints":{"tangled_repo_cid_only_null_if_deleted":{"name":"tangled_repo_cid_only_null_if_deleted","value":"(\"tangled_repo\".\"cid\" is not null or \"tangled_repo\".\"deleted\" = true)"},"tangled_repo_non_null_fields":{"name":"tangled_repo_non_null_fields","value":"(\"tangled_repo\".\"deleted\" = true or (\"tangled_repo\".\"name\" is not null and \"tangled_repo\".\"knot\" is not null and \"tangled_repo\".\"created_at\" is not null))"}},"isRLSEnabled":false},"public.tangled_spindle":{"name":"tangled_spindle","schema":"","columns":{"did":{"name":"did","type":"text","primaryKey":false,"notNull":true},"rev":{"name":"rev","type":"text","primaryKey":false,"notNull":true},"rkey":{"name":"rkey","type":"text","primaryKey":false,"notNull":true},"cid":{"name":"cid","type":"text","primaryKey":false,"notNull":false},"deleted":{"name":"deleted","type":"boolean","primaryKey":false,"notNull":true},"created_at":{"name":"created_at","type":"timestamp with time zone","primaryKey":false,"notNull":false}},"indexes":{},"foreignKeys":{},"compositePrimaryKeys":{"tangled_spindle_did_rkey_pk":{"name":"tangled_spindle_did_rkey_pk","columns":["did","rkey"]}},"uniqueConstraints":{},"policies":{},"checkConstraints":{"tangled_spindle_cid_only_null_if_deleted":{"name":"tangled_spindle_cid_only_null_if_deleted","value":"(\"tangled_spindle\".\"cid\" is not null or \"tangled_spindle\".\"deleted\" = true)"},"tangled_spindle_non_null_fields":{"name":"tangled_spindle_non_null_fields","value":"(\"tangled_spindle\".\"deleted\" = true or \"tangled_spindle\".\"created_at\" is not null)"}},"isRLSEnabled":false},"public.tangled_spindle_member":{"name":"tangled_spindle_member","schema":"","columns":{"did":{"name":"did","type":"text","primaryKey":false,"notNull":true},"rev":{"name":"rev","type":"text","primaryKey":false,"notNull":true},"rkey":{"name":"rkey","type":"text","primaryKey":false,"notNull":true},"cid":{"name":"cid","type":"text","primaryKey":false,"notNull":false},"deleted":{"name":"deleted","type":"boolean","primaryKey":false,"notNull":true},"subject":{"name":"subject","type":"text","primaryKey":false,"notNull":false},"instance":{"name":"instance","type":"text","primaryKey":false,"notNull":false},"created_at":{"name":"created_at","type":"timestamp with time zone","primaryKey":false,"notNull":false}},"indexes":{},"foreignKeys":{},"compositePrimaryKeys":{"tangled_spindle_member_did_rkey_pk":{"name":"tangled_spindle_member_did_rkey_pk","columns":["did","rkey"]}},"uniqueConstraints":{},"policies":{},"checkConstraints":{"tangled_spindle_member_cid_only_null_if_deleted":{"name":"tangled_spindle_member_cid_only_null_if_deleted","value":"(\"tangled_spindle_member\".\"cid\" is not null or \"tangled_spindle_member\".\"deleted\" = true)"},"tangled_spindle_member_non_null_fields":{"name":"tangled_spindle_member_non_null_fields","value":"(\"tangled_spindle_member\".\"deleted\" = true or (\"tangled_spindle_member\".\"subject\" is not null and \"tangled_spindle_member\".\"instance\" is not null and \"tangled_spindle_member\".\"created_at\" is not null))"}},"isRLSEnabled":false},"public.tangled_star":{"name":"tangled_star","schema":"","columns":{"did":{"name":"did","type":"text","primaryKey":false,"notNull":true},"rev":{"name":"rev","type":"text","primaryKey":false,"notNull":true},"rkey":{"name":"rkey","type":"text","primaryKey":false,"notNull":true},"cid":{"name":"cid","type":"text","primaryKey":false,"notNull":false},"deleted":{"name":"deleted","type":"boolean","primaryKey":false,"notNull":true},"subject":{"name":"subject","type":"text","primaryKey":false,"notNull":false},"subject_did":{"name":"subject_did","type":"text","primaryKey":false,"notNull":false},"created_at":{"name":"created_at","type":"timestamp with time zone","primaryKey":false,"notNull":false}},"indexes":{},"foreignKeys":{},"compositePrimaryKeys":{"tangled_star_did_rkey_pk":{"name":"tangled_star_did_rkey_pk","columns":["did","rkey"]}},"uniqueConstraints":{},"policies":{},"checkConstraints":{"tangled_star_cid_only_null_if_deleted":{"name":"tangled_star_cid_only_null_if_deleted","value":"(\"tangled_star\".\"cid\" is not null or \"tangled_star\".\"deleted\" = true)"},"tangled_star_non_null_fields":{"name":"tangled_star_non_null_fields","value":"(\"tangled_star\".\"deleted\" = true or \"tangled_star\".\"created_at\" is not null)"}},"isRLSEnabled":false},"public.tangled_string":{"name":"tangled_string","schema":"","columns":{"did":{"name":"did","type":"text","primaryKey":false,"notNull":true},"rev":{"name":"rev","type":"text","primaryKey":false,"notNull":true},"rkey":{"name":"rkey","type":"text","primaryKey":false,"notNull":true},"cid":{"name":"cid","type":"text","primaryKey":false,"notNull":false},"deleted":{"name":"deleted","type":"boolean","primaryKey":false,"notNull":true},"filename":{"name":"filename","type":"text","primaryKey":false,"notNull":false},"description":{"name":"description","type":"text","primaryKey":false,"notNull":false},"created_at":{"name":"created_at","type":"timestamp with time zone","primaryKey":false,"notNull":false},"contents":{"name":"contents","type":"text","primaryKey":false,"notNull":false}},"indexes":{},"foreignKeys":{},"compositePrimaryKeys":{"tangled_string_did_rkey_pk":{"name":"tangled_string_did_rkey_pk","columns":["did","rkey"]}},"uniqueConstraints":{},"policies":{},"checkConstraints":{"tangled_string_cid_only_null_if_deleted":{"name":"tangled_string_cid_only_null_if_deleted","value":"(\"tangled_string\".\"cid\" is not null or \"tangled_string\".\"deleted\" = true)"},"tangled_string_non_null_fields":{"name":"tangled_string_non_null_fields","value":"(\"tangled_string\".\"deleted\" = true or (\"tangled_string\".\"filename\" is not null and \"tangled_string\".\"description\" is not null and \"tangled_string\".\"created_at\" is not null and \"tangled_string\".\"contents\" is not null))"}},"isRLSEnabled":false},"public.tangled_vouch":{"name":"tangled_vouch","schema":"","columns":{"did":{"name":"did","type":"text","primaryKey":false,"notNull":true},"rev":{"name":"rev","type":"text","primaryKey":false,"notNull":true},"rkey":{"name":"rkey","type":"text","primaryKey":false,"notNull":true},"cid":{"name":"cid","type":"text","primaryKey":false,"notNull":false},"deleted":{"name":"deleted","type":"boolean","primaryKey":false,"notNull":true},"kind":{"name":"kind","type":"tangled_vouch_kind","typeSchema":"public","primaryKey":false,"notNull":false},"reason":{"name":"reason","type":"text","primaryKey":false,"notNull":false},"created_at":{"name":"created_at","type":"timestamp with time zone","primaryKey":false,"notNull":false}},"indexes":{},"foreignKeys":{},"compositePrimaryKeys":{"tangled_vouch_did_rkey_pk":{"name":"tangled_vouch_did_rkey_pk","columns":["did","rkey"]}},"uniqueConstraints":{},"policies":{},"checkConstraints":{"tangled_vouch_cid_only_null_if_deleted":{"name":"tangled_vouch_cid_only_null_if_deleted","value":"(\"tangled_vouch\".\"cid\" is not null or \"tangled_vouch\".\"deleted\" = true)"},"tangled_vouch_non_null_fields":{"name":"tangled_vouch_non_null_fields","value":"(\"tangled_vouch\".\"deleted\" = true or (\"tangled_vouch\".\"kind\" is not null and \"tangled_vouch\".\"created_at\" is not null))"}},"isRLSEnabled":false}},"enums":{"public.raw_action":{"name":"raw_action","schema":"public","values":["create","update","delete"]},"public.tangled_pipeline_status_status":{"name":"tangled_pipeline_status_status","schema":"public","values":["pending","running","failed","timeout","cancelled","success"]},"public.tangled_pipeline_trigger_metadata_kind":{"name":"tangled_pipeline_trigger_metadata_kind","schema":"public","values":["push","pull_request","manual"]},"public.tangled_vouch_kind":{"name":"tangled_vouch_kind","schema":"public","values":["vouch","denounce"]}},"schemas":{},"sequences":{},"roles":{},"policies":{},"views":{},"_meta":{"columns":{},"schemas":{},"tables":{}}} diff --git a/api/src/db/drizzle/meta/_journal.json b/api/src/db/drizzle/meta/_journal.json index 010a6c6..5a61ce3 100644 --- a/api/src/db/drizzle/meta/_journal.json +++ b/api/src/db/drizzle/meta/_journal.json @@ -36,6 +36,13 @@ "when": 1777842176815, "tag": "0004_bsky_profile", "breakpoints": true + }, + { + "idx": 5, + "version": "7", + "when": 1777847015982, + "tag": "0005_fix_raw_records", + "breakpoints": true } ] } \ No newline at end of file diff --git a/api/src/db/tables/raw_records.ts b/api/src/db/tables/raw_records.ts index d87fa17..6b308da 100644 --- a/api/src/db/tables/raw_records.ts +++ b/api/src/db/tables/raw_records.ts @@ -1,4 +1,14 @@ -import { bigint, check, index, jsonb, pgEnum, pgTable, text, timestamp } from "drizzle-orm/pg-core"; +import { + bigint, + check, + index, + jsonb, + pgEnum, + pgTable, + text, + timestamp, + uniqueIndex, +} from "drizzle-orm/pg-core"; import { uuidv7WithDefault } from "../helpers.ts"; import { and, isNotNull, or, sql } from "drizzle-orm"; @@ -8,22 +18,24 @@ export const rawRecords = pgTable( "raw_records", { id: uuidv7WithDefault().primaryKey(), - tap_id: bigint({ mode: "bigint" }).notNull().unique(), + tap_id: bigint({ mode: "bigint" }).notNull(), received_at: timestamp({ withTimezone: true }).notNull().defaultNow(), + did: text().notNull(), rev: text().notNull(), collection: text().notNull(), rkey: text().notNull(), + action: actionEnum().notNull(), cid: text(), // nullable on deletes record: jsonb(), // nullable on deletes }, (table) => [ + uniqueIndex().on(table.did, table.collection, table.rkey, table.rev), check( "raw_records_not_null_unless_delete", or(sql`${table.action} = 'delete'`, and(isNotNull(table.cid), isNotNull(table.record)))!, ), index().on(table.collection), - index().on(table.did, table.collection, table.rkey), ], ); diff --git a/api/src/ingest/tap.ts b/api/src/ingest/tap.ts index 99e3294..9ac6980 100644 --- a/api/src/ingest/tap.ts +++ b/api/src/ingest/tap.ts @@ -1,5 +1,6 @@ import { Tap, type RecordEvent, type TapHandler } from "@atproto/tap"; -import { jsonToLex, type JsonValue, lexToJson } from "@atproto/lex"; +import { type JsonValue, lexToJson } from "@atproto/lex"; +import { Agent, CredentialSession } from "@atproto/api"; import { db } from "../db/index.ts"; import { rawRecords } from "../db/tables/raw_records.ts"; import { TAP_URL } from "../lib/constants.ts"; @@ -118,15 +119,14 @@ async function flushRows(batch: [RecordEvent, () => Promise][]) { const insertedRows = await db .insert(rawRecords) .values(mappedBatch) - .onConflictDoNothing({ target: rawRecords.tap_id }) + .onConflictDoNothing({ + target: [rawRecords.did, rawRecords.collection, rawRecords.rkey, rawRecords.rev], + }) .returning(); - const insertedTapIds = new Set(insertedRows.map((row) => row.tap_id)); - console.log(`Inserted ${insertedTapIds.size} rows.`); - - const insertedRecords = mappedBatch.filter(({ tap_id }) => insertedTapIds.has(tap_id)); + console.log(`Inserted ${insertedRows.length} rows.`); // kinda sus, but trust bro - syncLevelTwoTables(insertedRecords as CollectionHandlerArg[]); + syncLevelTwoTables(mappedBatch as CollectionHandlerArg[]); await Promise.all(batch.map(([_, ack]) => ack())); } catch (err) { @@ -184,6 +184,48 @@ console.log("Started Tap Importer..."); export { channel }; +// Add non-profile repos to collection list +async function seedTap() { + try { + const session = new CredentialSession(new URL("https://relay1.us-west.bsky.network")); + const agent = new Agent(session); + const collections = Object.keys(COLLECTION_HANDLERS).filter((c) => c.startsWith("sh.tangled")); + + let repos: string[] = []; + + for (const collection of collections) { + console.log("Getting repos from", collection, "to seed"); + + let cursor: string | undefined = undefined; + + while (true) { + const repoPage = await agent.com.atproto.sync.listReposByCollection({ + collection, + limit: 1000, + cursor, + }); + const repoDids = repoPage.data.repos.map((r) => r.did); + cursor = repoPage.data.cursor; + repos.push(...repoDids); + if (repoDids.length === 0 || !cursor) break; + } + } + + // dedup + repos = [...new Set(repos)]; + + // push to tap, since it's so lazy + for (let i = 0; i < repos.length; i += 500) { + await tap.addRepos(repos.slice(i, i + 500)); + } + console.log("Seeded", repos.length, "into the tap"); + } catch (err) { + console.error("Oops tap seeding failed", err); + } +} +seedTap(); +setInterval(seedTap, 1000 * 60 * 10); + // (async () => { // const fullIngest = await db.select().from(rawRecords); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e5f007f..026a67f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -17,6 +17,9 @@ importers: api: dependencies: + '@atproto/api': + specifier: ^0.19.11 + version: 0.19.11 '@atproto/identity': specifier: ^0.4.12 version: 0.4.12 @@ -131,6 +134,9 @@ packages: '@atproto-labs/simple-store@0.3.0': resolution: {integrity: sha512-nOb6ONKBRJHRlukW1sVawUkBqReLlLx6hT35VS3imaNPwiXDxLnTK7lxw3Lrl9k5yugSBDQAkZAq3MPTEFSUBQ==, tarball: https://registry.npmjs.org/@atproto-labs/simple-store/-/simple-store-0.3.0.tgz} + '@atproto/api@0.19.11': + resolution: {integrity: sha512-7V4Sg6hcv/UxoXobjfvy/Ox2ioKQtZ3DzbsiFndYCcBfsZ5GO8rNEroHPq3hT0CFBJK1NAD6JfOtTBN2z267Xg==, tarball: https://registry.npmjs.org/@atproto/api/-/api-0.19.11.tgz} + '@atproto/common-web@0.4.21': resolution: {integrity: sha512-Odq+wdk3YNasGCjjlpl3bCIPvqYHige5DLfMkIffNv/2PI/iIj5ZvAvMvJlJ59OhReKSxtpI0invx5UQPc3+fw==, tarball: https://registry.npmjs.org/@atproto/common-web/-/common-web-0.4.21.tgz} @@ -198,6 +204,9 @@ packages: resolution: {integrity: sha512-dox1XIymuC7/ZRhUqKezIGgooZS45C6vHCfu0PnWjfvsLCK2kAlnvX4IBkA/WpcoijDhQ9ejChnFbo/sLmgvAg==, tarball: https://registry.npmjs.org/@atproto/ws-client/-/ws-client-0.0.4.tgz} engines: {node: '>=18.7.0'} + '@atproto/xrpc@0.7.7': + resolution: {integrity: sha512-K1ZyO/BU8JNtXX5dmPp7b5UrkLMMqpsIa/Lrj5D3Su+j1Xwq1m6QJ2XJ1AgjEjkI1v4Muzm7klianLE6XGxtmA==, tarball: https://registry.npmjs.org/@atproto/xrpc/-/xrpc-0.7.7.tgz} + '@babel/code-frame@7.27.1': resolution: {integrity: sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==, tarball: https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz} engines: {node: '>=6.9.0'} @@ -1338,6 +1347,9 @@ packages: resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==, tarball: https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz} engines: {node: '>=8.0.0'} + await-lock@2.2.2: + resolution: {integrity: sha512-aDczADvlvTGajTDjcjpJMqRkOF6Qdz3YbPZm/PyW6tKPkx2hlYBzxMhEywM/tU72HrVZjgl5VCdRuMlA7pZ8Gw==, tarball: https://registry.npmjs.org/await-lock/-/await-lock-2.2.2.tgz} + babel-dead-code-elimination@1.0.12: resolution: {integrity: sha512-GERT7L2TiYcYDtYk1IpD+ASAYXjKbLTDPhBtYj7X1NuRMDTMtAx9kyBenub1Ev41lo91OHCKdmP+egTDmfQ7Ig==, tarball: https://registry.npmjs.org/babel-dead-code-elimination/-/babel-dead-code-elimination-1.0.12.tgz} @@ -2069,6 +2081,10 @@ packages: resolution: {integrity: sha512-Pugqs6M0m7Lv1I7FtxN4aoyToKg1C4tu+/381vH35y8oENM/Ai7f7C4StcoK4/+BSw9ebcS8jRiVrORFKCALLw==, tarball: https://registry.npmjs.org/tinypool/-/tinypool-2.1.0.tgz} engines: {node: ^20.0.0 || >=22.0.0} + tlds@1.261.0: + resolution: {integrity: sha512-QXqwfEl9ddlGBaRFXIvNKK6OhipSiLXuRuLJX5DErz0o0Q0rYxulWLdFryTkV5PkdZct5iMInwYEGe/eR++1AA==, tarball: https://registry.npmjs.org/tlds/-/tlds-1.261.0.tgz} + hasBin: true + to-regex-range@5.0.1: resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==, tarball: https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz} engines: {node: '>=8.0'} @@ -2255,6 +2271,17 @@ snapshots: '@atproto-labs/simple-store@0.3.0': {} + '@atproto/api@0.19.11': + dependencies: + '@atproto/common-web': 0.4.21 + '@atproto/lexicon': 0.6.2 + '@atproto/syntax': 0.5.4 + '@atproto/xrpc': 0.7.7 + await-lock: 2.2.2 + multiformats: 9.9.0 + tlds: 1.261.0 + zod: 3.25.76 + '@atproto/common-web@0.4.21': dependencies: '@atproto/lex-data': 0.0.15 @@ -2407,6 +2434,11 @@ snapshots: - bufferutil - utf-8-validate + '@atproto/xrpc@0.7.7': + dependencies: + '@atproto/lexicon': 0.6.2 + zod: 3.25.76 + '@babel/code-frame@7.27.1': dependencies: '@babel/helper-validator-identifier': 7.28.5 @@ -3278,6 +3310,8 @@ snapshots: atomic-sleep@1.0.0: {} + await-lock@2.2.2: {} + babel-dead-code-elimination@1.0.12: dependencies: '@babel/core': 7.29.0 @@ -3939,6 +3973,8 @@ snapshots: tinypool@2.1.0: {} + tlds@1.261.0: {} + to-regex-range@5.0.1: dependencies: is-number: 7.0.0 diff --git a/tap/dev.sh b/tap/dev.sh index ad96522..d04f906 100755 --- a/tap/dev.sh +++ b/tap/dev.sh @@ -5,5 +5,7 @@ export TAP_COLLECTION_FILTERS="sh.tangled.*,app.bsky.actor.profile" export TAP_IDENT_CACHE_SIZE="20000" export TAP_RELAY_URL="https://relay1.us-west.bsky.network" export TAP_SIGNAL_COLLECTION="sh.tangled.actor.profile" +export TAP_RESYNC_PARALLELISM="20" -tap run +touch ./tap.log +tap run >> ./tap.log -- 2.51.2