From 64aed01bd2c0bc2ea8c97166d235399d1dea1385 Mon Sep 17 00:00:00 2001 From: Jack Platten Date: Mon, 20 Jan 2025 18:25:48 -0800 Subject: [PATCH] we did it --- .dockerignore | 4 ++ .gitignore | 1 + Cargo.lock | 1 + Cargo.toml | 3 +- Dockerfile | 41 +++++++++++++++++++ .../2025-01-21-005557_add_time_us/down.sql | 2 + .../2025-01-21-005557_add_time_us/up.sql | 2 + .../2025-01-21-010225_rename_table/down.sql | 2 + .../2025-01-21-010225_rename_table/up.sql | 2 + .../down.sql | 2 + .../up.sql | 2 + src/db/mod.rs | 20 +++++++-- src/models.rs | 4 +- src/post_listener/mod.rs | 18 +++++--- src/schema.rs | 1 + 15 files changed, 95 insertions(+), 10 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 migrations/2025-01-21-005557_add_time_us/down.sql create mode 100644 migrations/2025-01-21-005557_add_time_us/up.sql create mode 100644 migrations/2025-01-21-010225_rename_table/down.sql create mode 100644 migrations/2025-01-21-010225_rename_table/up.sql create mode 100644 migrations/2025-01-21-011113_change_timestamp_to_text/down.sql create mode 100644 migrations/2025-01-21-011113_change_timestamp_to_text/up.sql diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..ee724af --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +target +.cargo +**/*.sh +**/*.tar.gz \ No newline at end of file diff --git a/.gitignore b/.gitignore index ea8c4bf..0b745e2 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /target +.env \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 402537c..7ca1e29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -79,6 +79,7 @@ name = "at-comments" version = "0.1.0" dependencies = [ "atrium-api", + "chrono", "diesel", "dotenvy", "jetstream-oxide", diff --git a/Cargo.toml b/Cargo.toml index fcbf77d..5b4d70b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,4 +8,5 @@ rocket = {version = "0.5.1", features = ["json"]} diesel = { version = "2.2.0", features = ["postgres"] } dotenvy = "0.15" jetstream-oxide = "0.1.0" -atrium-api ="0.24.5" \ No newline at end of file +atrium-api ="0.24.5" +chrono = { version = "0.4", features = ["serde"] } \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..646216f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,41 @@ +FROM docker.io/rust:1.84.0 AS build + +## cargo package name: customize here or provide via --build-arg +ARG pkg=at-comments + +RUN apt-get update -y && \ + apt-get install -y pkgconf make g++ libssl-dev + +WORKDIR /build + +COPY . . + +RUN --mount=type=cache,target=/build/target \ + --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + set -eux; \ + cargo build --release; \ + objcopy --compress-debug-sections target/release/$pkg ./main + +################################################################################ + +FROM docker.io/debian:bookworm-slim + +RUN apt-get update -y && \ + apt-get install -y libpq5 ca-certificates + +WORKDIR /app + +## copy the main binary +COPY --from=build /build/main ./ + +## copy runtime assets which may or may not exist +COPY --from=build /build/Rocket.tom[l] ./static +COPY --from=build /build/stati[c] ./static +COPY --from=build /build/template[s] ./templates + +## ensure the container listens globally on port 8080 +ENV ROCKET_ADDRESS=0.0.0.0 +ENV ROCKET_PORT=8080 + +CMD ./main \ No newline at end of file diff --git a/migrations/2025-01-21-005557_add_time_us/down.sql b/migrations/2025-01-21-005557_add_time_us/down.sql new file mode 100644 index 0000000..cc14793 --- /dev/null +++ b/migrations/2025-01-21-005557_add_time_us/down.sql @@ -0,0 +1,2 @@ +-- This file should undo anything in `up.sql` +ALTER TABLE posts DROP COLUMN time_us; \ No newline at end of file diff --git a/migrations/2025-01-21-005557_add_time_us/up.sql b/migrations/2025-01-21-005557_add_time_us/up.sql new file mode 100644 index 0000000..fe3d714 --- /dev/null +++ b/migrations/2025-01-21-005557_add_time_us/up.sql @@ -0,0 +1,2 @@ +-- Your SQL goes here +ALTER TABLE posts ADD COLUMN time_us TIMESTAMP; \ No newline at end of file diff --git a/migrations/2025-01-21-010225_rename_table/down.sql b/migrations/2025-01-21-010225_rename_table/down.sql new file mode 100644 index 0000000..ce859ef --- /dev/null +++ b/migrations/2025-01-21-010225_rename_table/down.sql @@ -0,0 +1,2 @@ +-- This file should undo anything in `up.sql` +ALTER TABLE slug_dids RENAME TO posts; \ No newline at end of file diff --git a/migrations/2025-01-21-010225_rename_table/up.sql b/migrations/2025-01-21-010225_rename_table/up.sql new file mode 100644 index 0000000..1b60c4e --- /dev/null +++ b/migrations/2025-01-21-010225_rename_table/up.sql @@ -0,0 +1,2 @@ +-- Your SQL goes here +ALTER TABLE posts RENAME TO slug_dids; \ No newline at end of file diff --git a/migrations/2025-01-21-011113_change_timestamp_to_text/down.sql b/migrations/2025-01-21-011113_change_timestamp_to_text/down.sql new file mode 100644 index 0000000..fb98d9f --- /dev/null +++ b/migrations/2025-01-21-011113_change_timestamp_to_text/down.sql @@ -0,0 +1,2 @@ +-- This file should undo anything in `up.sql` +ALTER TABLE posts ALTER COLUMN time_us TYPE TIMESTAMP; \ No newline at end of file diff --git a/migrations/2025-01-21-011113_change_timestamp_to_text/up.sql b/migrations/2025-01-21-011113_change_timestamp_to_text/up.sql new file mode 100644 index 0000000..1a10ff6 --- /dev/null +++ b/migrations/2025-01-21-011113_change_timestamp_to_text/up.sql @@ -0,0 +1,2 @@ +-- Your SQL goes here +ALTER TABLE posts ALTER COLUMN time_us TYPE TEXT; \ No newline at end of file diff --git a/src/db/mod.rs b/src/db/mod.rs index 7b532ae..02c453b 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -10,18 +10,20 @@ pub fn establish_connection() -> PgConnection { PgConnection::establish(&database_url).expect(&format!("Error connecting to {}", database_url)) } -pub fn insert_post_rkey<'a>(slug: &'a str, rkey: &'a str) -> Post { +pub fn insert_post_rkey<'a>(slug: &'a str, rkey: &'a str, time_us: &'a str) -> Result { use crate::schema::posts; - let new_post = NewPost { slug, rkey }; + let new_post = NewPost { slug, rkey, time_us }; let connection = &mut establish_connection(); + diesel::insert_into(posts::table) .values(&new_post) + .on_conflict(posts::slug) + .do_nothing() .returning(Post::as_returning()) .get_result(connection) - .expect("Error saving new post") } pub fn get_post_rkey(post_slug: &str) -> Result { @@ -35,3 +37,15 @@ pub fn get_post_rkey(post_slug: &str) -> Result { post } + +pub fn get_latest_time_us() -> Result { + use super::schema::posts::dsl::*; + + let connection = &mut establish_connection(); + let time_stamp = posts + .order(id.desc()) + .select(time_us) + .first(connection); + + time_stamp +} \ No newline at end of file diff --git a/src/models.rs b/src/models.rs index 75dd822..b2ef12d 100644 --- a/src/models.rs +++ b/src/models.rs @@ -10,11 +10,13 @@ pub struct Post { pub id: i32, pub slug: String, pub rkey: String, + pub time_us: String, } -#[derive(Insertable)] +#[derive(Insertable, Debug)] #[diesel(table_name = posts)] pub struct NewPost<'a> { pub slug: &'a str, pub rkey: &'a str, + pub time_us: &'a str, } diff --git a/src/post_listener/mod.rs b/src/post_listener/mod.rs index 138a721..18b8e46 100644 --- a/src/post_listener/mod.rs +++ b/src/post_listener/mod.rs @@ -7,8 +7,8 @@ use jetstream_oxide::{ DefaultJetstreamEndpoints, JetstreamCompression, JetstreamConfig, JetstreamConnector, }; use std::env; - -use crate::db::insert_post_rkey; +use crate::db::{get_latest_time_us, insert_post_rkey}; +use chrono; pub async fn subscribe_posts() { dotenv().ok(); @@ -17,11 +17,17 @@ pub async fn subscribe_posts() { let did = vec![Did::new(did_string.to_string()).unwrap()]; + let cursor = match get_latest_time_us() { + Ok(time) => chrono::DateTime::from_timestamp_micros(time.parse::().unwrap()), + Err(_) => None, + + }; + let config = JetstreamConfig { endpoint: DefaultJetstreamEndpoints::USWestOne.into(), wanted_dids: did, compression: JetstreamCompression::Zstd, - cursor: None, + cursor: cursor, wanted_collections: nsid, }; @@ -32,7 +38,7 @@ pub async fn subscribe_posts() { while let Ok(event) = receiver.recv_async().await { if let Commit(commit) = event { match commit { - CommitEvent::Create { info: _, commit } => { + CommitEvent::Create { info, commit } => { if let AppBskyFeedPost(record) = commit.record { // check and see if this post is what we're looking for @@ -50,10 +56,12 @@ pub async fn subscribe_posts() { // get the slug let uri_parts: Vec<&str> = uri.split('/').collect(); let slug = *uri_parts.last().unwrap(); + let time_us_string = info.time_us.to_string(); + let time_us = time_us_string.as_str(); // insert into db // should probably insert cursor too - insert_post_rkey(slug, &rkey); + let _ = insert_post_rkey(slug, &rkey, time_us); } _ => {} // ick } diff --git a/src/schema.rs b/src/schema.rs index 75c9d8d..34ef125 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -5,5 +5,6 @@ diesel::table! { id -> Int4, slug -> Text, rkey -> Text, + time_us -> Text, } } -- 2.51.2