From 6a56d7b9e67b2b008748b399d63acaa5d4122fcc Mon Sep 17 00:00:00 2001 From: afterlifepro Date: Wed, 12 Nov 2025 18:30:04 +0000 Subject: [PATCH] initialise indexer database we use `cargo sqlx prepare` with offline mode as the database can be offline when docker starts, which leads to builds failing --- ...cfdaf6b7bd0ef74f632d007b0f8eb797c97ca.json | 12 ++++ ...9d74b28d8ff2619da9a5000b05f6106194d6b.json | 12 ++++ ...c38cb2911efff5432ae454210d23b70ee0ae2.json | 12 ++++ Cargo.lock | 14 +++++ Cargo.toml | 2 +- Dockerfile | 19 ++++++ sqlx.sh | 2 + src/config.rs | 31 +++++++++ src/db.rs | 63 +++++++++++++++++++ src/main.rs | 13 +++- 10 files changed, 177 insertions(+), 3 deletions(-) create mode 100644 .sqlx/query-214eb02cdf9fbf46e9344f4d7dbcfdaf6b7bd0ef74f632d007b0f8eb797c97ca.json create mode 100644 .sqlx/query-2663586971e5e67cd7f5452f64e9d74b28d8ff2619da9a5000b05f6106194d6b.json create mode 100644 .sqlx/query-94ea3e0cdd96672b408e4a445c2c38cb2911efff5432ae454210d23b70ee0ae2.json create mode 100644 Dockerfile create mode 100755 sqlx.sh create mode 100644 src/config.rs create mode 100644 src/db.rs diff --git a/.sqlx/query-214eb02cdf9fbf46e9344f4d7dbcfdaf6b7bd0ef74f632d007b0f8eb797c97ca.json b/.sqlx/query-214eb02cdf9fbf46e9344f4d7dbcfdaf6b7bd0ef74f632d007b0f8eb797c97ca.json new file mode 100644 index 0000000..0d7d619 --- /dev/null +++ b/.sqlx/query-214eb02cdf9fbf46e9344f4d7dbcfdaf6b7bd0ef74f632d007b0f8eb797c97ca.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "CREATE TABLE IF NOT EXISTS foreign_records (\n did TEXT,\n collection TEXT,\n rkey TEXT,\n record JSON NOT NULL,\n PRIMARY KEY (did, collection, rkey)\n );", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "214eb02cdf9fbf46e9344f4d7dbcfdaf6b7bd0ef74f632d007b0f8eb797c97ca" +} diff --git a/.sqlx/query-2663586971e5e67cd7f5452f64e9d74b28d8ff2619da9a5000b05f6106194d6b.json b/.sqlx/query-2663586971e5e67cd7f5452f64e9d74b28d8ff2619da9a5000b05f6106194d6b.json new file mode 100644 index 0000000..25434a4 --- /dev/null +++ b/.sqlx/query-2663586971e5e67cd7f5452f64e9d74b28d8ff2619da9a5000b05f6106194d6b.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "CREATE TABLE IF NOT EXISTS blobs (\n did TEXT,\n cid TEXT,\n blob bytea NOT NULL,\n PRIMARY KEY (did, cid)\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "2663586971e5e67cd7f5452f64e9d74b28d8ff2619da9a5000b05f6106194d6b" +} diff --git a/.sqlx/query-94ea3e0cdd96672b408e4a445c2c38cb2911efff5432ae454210d23b70ee0ae2.json b/.sqlx/query-94ea3e0cdd96672b408e4a445c2c38cb2911efff5432ae454210d23b70ee0ae2.json new file mode 100644 index 0000000..c6cdc5d --- /dev/null +++ b/.sqlx/query-94ea3e0cdd96672b408e4a445c2c38cb2911efff5432ae454210d23b70ee0ae2.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "CREATE TABLE IF NOT EXISTS records (\n collection TEXT,\n rkey TEXT,\n record JSON NOT NULL,\n PRIMARY KEY (collection, rkey)\n );", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "94ea3e0cdd96672b408e4a445c2c38cb2911efff5432ae454210d23b70ee0ae2" +} diff --git a/Cargo.lock b/Cargo.lock index d711b82..4572ce0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3335,6 +3335,8 @@ dependencies = [ "sha2", "smallvec", "thiserror 2.0.17", + "tokio", + "tokio-stream", "tracing", "url", ] @@ -3373,6 +3375,7 @@ dependencies = [ "sqlx-postgres", "sqlx-sqlite", "syn 2.0.109", + "tokio", "url", ] @@ -3807,6 +3810,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.17" diff --git a/Cargo.toml b/Cargo.toml index 0205f8d..14b6c81 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,5 +5,5 @@ edition = "2024" [dependencies] jacquard = "0.9.0" -sqlx = "0.8.6" +sqlx = { version = "0.8.6", features = ["runtime-tokio", "postgres"] } tokio = "1.48.0" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6ac874f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +FROM rust +WORKDIR /app + +# copying just cargo config and making a stub main +# this means docker can cache building deps +# and just build the main program in watch mode +# much quicker and nicer on my cpu, network, and ram lol +COPY ./Cargo.toml ./Cargo.toml +COPY ./Cargo.lock ./Cargo.lock +RUN mkdir ./src && echo 'fn main() { println!("Using cache build!!"); loop { println!("DOCKERFILE failed..."); } }' > ./src/main.rs +RUN cargo build --locked +RUN rm -f ./target/debug/index + +COPY ./src ./src +COPY ./.sqlx ./.sqlx +RUN cargo build --locked + +RUN cp ./target/debug/index ./index +CMD ["/app/index"] \ No newline at end of file diff --git a/sqlx.sh b/sqlx.sh new file mode 100755 index 0000000..d799e2b --- /dev/null +++ b/sqlx.sh @@ -0,0 +1,2 @@ +docker compose up db -d +DATABASE_URL=postgres://user:password@localhost:5432/user cargo sqlx prepare \ No newline at end of file diff --git a/src/config.rs b/src/config.rs new file mode 100644 index 0000000..4d69b72 --- /dev/null +++ b/src/config.rs @@ -0,0 +1,31 @@ +use jacquard::types::string::Did; +use std::env; +use std::sync::LazyLock; + +// this should be loaded before the program starts any threads +// if this panics threads that access it will be poisoned +pub static USER: LazyLock> = LazyLock::new(|| { + let Ok(env) = env::var("INDEX_USER") else { + panic!("INDEX_USER not set"); + }; + + let Ok(did) = Did::new_owned(env) else { + panic!("INDEX_USER was not a valid did") + }; + + did +}); + +pub static POSTGRES_URL: LazyLock = LazyLock::new(|| { + let Ok(user) = env::var("POSTGRES_USER") else { + panic!("POSTGRES_USER was not set. Cannot connect to database") + }; + let Ok(password) = env::var("POSTGRES_PASSWORD") else { + panic!("POSTGRES_PASSWORD was not set. Cannot connect to database") + }; + let Ok(host) = env::var("POSTGRES_HOST") else { + panic!("POSTGRES_HOST was not set. Cannot connect to database") + }; + + format!("postgres://{}:{}@{}/{}", user, password, host, user) +}); diff --git a/src/db.rs b/src/db.rs new file mode 100644 index 0000000..f47ad47 --- /dev/null +++ b/src/db.rs @@ -0,0 +1,63 @@ +use crate::config; +use sqlx::{Pool, Postgres, postgres::PgPool, query}; + +pub async fn init() -> Pool { + let conn = PgPool::connect(&config::POSTGRES_URL).await; + let conn = match conn { + Ok(val) => val, + Err(err) => { + println!("Could not connect to the database. Got error {err}"); + panic!() + } + }; + println!("Connected to db with pool: {:?}", conn); + + // initialise db tables + if let Err(err) = query!( + "CREATE TABLE IF NOT EXISTS records ( + collection TEXT, + rkey TEXT, + record JSON NOT NULL, + PRIMARY KEY (collection, rkey) + );" + ) + .execute(&conn) + .await + { + println!("Creating table `records`: \n{err}"); + panic!("Could not instantiate db"); + }; + + if let Err(err) = query!( + "CREATE TABLE IF NOT EXISTS foreign_records ( + did TEXT, + collection TEXT, + rkey TEXT, + record JSON NOT NULL, + PRIMARY KEY (did, collection, rkey) + );" + ) + .execute(&conn) + .await + { + println!("Creating table `foreign_records`: \n{err}"); + panic!(); + }; + + if let Err(err) = query!( + "CREATE TABLE IF NOT EXISTS blobs ( + did TEXT, + cid TEXT, + blob bytea NOT NULL, + PRIMARY KEY (did, cid) + )" + ) + .execute(&conn) + .await + { + println!("Creating table `blobs`: \n{err}"); + panic!(); + }; + + return conn; +} diff --git a/src/main.rs b/src/main.rs index e7a11a9..593577a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,12 @@ -fn main() { - println!("Hello, world!"); +mod config; +mod db; + +#[tokio::main] +async fn main() { + println!("Indexing {}", *config::USER); + let conn = db::init().await; + println!("Database established"); + + // to stop docker whinging + loop {} } -- 2.51.2