From 7864b962d230d8003444d03c22c629339e246752 Mon Sep 17 00:00:00 2001
From: afterlifepro <vielle.dev@proton.me>
Date: Wed, 19 Nov 2025 17:47:34 +0000
Subject: [PATCH] tidy up some indexer logic

- remove sqlx cache bc i aint usin it rn (will add it back l8r for docker build)
- document how im going to do backfill/etc
- add meta table with a rev column. this is to store repo rev for diffing/etc
---
 ...7dbcfdaf6b7bd0ef74f632d007b0f8eb797c97ca.json | 12 ------------
 ...64e9d74b28d8ff2619da9a5000b05f6106194d6b.json | 12 ------------
 ...5c2c38cb2911efff5432ae454210d23b70ee0ae2.json | 12 ------------
 sqlx.sh                                          |  2 --
 src/db.rs                                        | 16 +++++++++++++++-
 src/main.rs                                      | 14 ++++++++++++++
 6 files changed, 29 insertions(+), 39 deletions(-)
 delete mode 100644 .sqlx/query-214eb02cdf9fbf46e9344f4d7dbcfdaf6b7bd0ef74f632d007b0f8eb797c97ca.json
 delete mode 100644 .sqlx/query-2663586971e5e67cd7f5452f64e9d74b28d8ff2619da9a5000b05f6106194d6b.json
 delete mode 100644 .sqlx/query-94ea3e0cdd96672b408e4a445c2c38cb2911efff5432ae454210d23b70ee0ae2.json
 delete mode 100755 sqlx.sh

diff --git a/.sqlx/query-214eb02cdf9fbf46e9344f4d7dbcfdaf6b7bd0ef74f632d007b0f8eb797c97ca.json b/.sqlx/query-214eb02cdf9fbf46e9344f4d7dbcfdaf6b7bd0ef74f632d007b0f8eb797c97ca.json
deleted file mode 100644
index 0d7d619..0000000
--- a/.sqlx/query-214eb02cdf9fbf46e9344f4d7dbcfdaf6b7bd0ef74f632d007b0f8eb797c97ca.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "CREATE TABLE IF NOT EXISTS foreign_records (\n            did TEXT,\n            collection TEXT,\n            rkey TEXT,\n            record JSON NOT NULL,\n            PRIMARY KEY (did, collection, rkey)\n        );",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": []
-  },
-  "hash": "214eb02cdf9fbf46e9344f4d7dbcfdaf6b7bd0ef74f632d007b0f8eb797c97ca"
-}
diff --git a/.sqlx/query-2663586971e5e67cd7f5452f64e9d74b28d8ff2619da9a5000b05f6106194d6b.json b/.sqlx/query-2663586971e5e67cd7f5452f64e9d74b28d8ff2619da9a5000b05f6106194d6b.json
deleted file mode 100644
index 25434a4..0000000
--- a/.sqlx/query-2663586971e5e67cd7f5452f64e9d74b28d8ff2619da9a5000b05f6106194d6b.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "CREATE TABLE IF NOT EXISTS blobs (\n            did TEXT,\n            cid TEXT,\n            blob bytea NOT NULL,\n            PRIMARY KEY (did, cid)\n        )",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": []
-  },
-  "hash": "2663586971e5e67cd7f5452f64e9d74b28d8ff2619da9a5000b05f6106194d6b"
-}
diff --git a/.sqlx/query-94ea3e0cdd96672b408e4a445c2c38cb2911efff5432ae454210d23b70ee0ae2.json b/.sqlx/query-94ea3e0cdd96672b408e4a445c2c38cb2911efff5432ae454210d23b70ee0ae2.json
deleted file mode 100644
index c6cdc5d..0000000
--- a/.sqlx/query-94ea3e0cdd96672b408e4a445c2c38cb2911efff5432ae454210d23b70ee0ae2.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "CREATE TABLE IF NOT EXISTS records (\n            collection TEXT,\n            rkey TEXT,\n            record JSON NOT NULL,\n            PRIMARY KEY (collection, rkey)\n        );",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": []
-  },
-  "hash": "94ea3e0cdd96672b408e4a445c2c38cb2911efff5432ae454210d23b70ee0ae2"
-}
diff --git a/sqlx.sh b/sqlx.sh
deleted file mode 100755
index d799e2b..0000000
--- a/sqlx.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-docker compose up db -d
-DATABASE_URL=postgres://user:password@localhost:5432/user cargo sqlx prepare
\ No newline at end of file
diff --git a/src/db.rs b/src/db.rs
index f580736..260ccfb 100644
--- a/src/db.rs
+++ b/src/db.rs
@@ -58,5 +58,19 @@ pub async fn init() -> Pool<Postgres> {
         panic!();
     };
 
-    return conn;
+    if let Err(err) = query!(
+        "CREATE TABLE IF NOT EXISTS meta (
+            did TEXT,
+            rev TEXT,
+            PRIMARY KEY (did)
+        );"
+    )
+    .execute(&conn)
+    .await
+    {
+        println!("Creating table `meta`: \n{err}");
+        panic!();
+    };
+
+    conn
 }
diff --git a/src/main.rs b/src/main.rs
index 20a5e80..cd4b657 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -2,6 +2,20 @@ mod config;
 mod db;
 mod resolver;
 
+/// backfill works as follows (https://docs.bsky.app/docs/advanced-guides/backfill)
+///
+/// 1. resolve did -> pds
+/// 2. stream com.atproto.sync.subscribeRepos to a buffer
+/// 3. get a car file from com.atproto.sync.getRepo (diff if a rev is stored in database)
+/// 4. apply car file diff to database (incl rev)
+/// 5. start playing events from buffer
+///     a. drop all events from other users
+///     b. drop all events with a lower rev than current rev
+///     c. apply event & update rev
+///     d. (non blocking) get blobs if missing
+///     e. (non blocking) parse for strongref and store strongrefs
+///     f. (non blocking) trigger garbage collection of blobs and strongref
+/// 6. once buffer is empty, parse events live
 #[tokio::main]
 async fn main() {
     println!("User: {}", *config::USER);
-- 
2.51.2