From aa02314d1bb8dd3e3feb8f55468331c3bbba91ea Mon Sep 17 00:00:00 2001 From: Jack Platten Date: Mon, 9 Mar 2026 22:40:20 -0700 Subject: [PATCH] Refactor project to replace Jetstream listener with RSS polling - Updated dependencies in Cargo.toml to include reqwest and rss. - Modified Dockerfile to use the latest Rust image. - Replaced post_listener module with rss_poller for fetching and processing Bluesky RSS feeds. - Updated Rocket.toml configuration to use poster_handle and blog_domain. - Removed unused migration files and cleaned up the migrations directory. - Added a new script for running local Postgres with schema creation and seeding. - Updated README.md to reflect changes in functionality and configuration. --- Cargo.lock | 1123 ++++++++--------- Cargo.toml | 5 +- Dockerfile | 12 +- README.md | 66 +- Rocket.toml | 6 +- migrations/.keep | 0 .../down.sql | 6 - .../up.sql | 36 - .../2025-01-20-053547_create_posts/down.sql | 2 - .../2025-01-20-053547_create_posts/up.sql | 7 - .../down.sql | 2 - .../2025-01-20-054932_remove_timestamp/up.sql | 2 - .../down.sql | 2 - .../up.sql | 2 - .../2025-01-21-005557_add_time_us/down.sql | 2 - .../2025-01-21-005557_add_time_us/up.sql | 2 - .../2025-01-21-010225_rename_table/down.sql | 2 - .../2025-01-21-010225_rename_table/up.sql | 2 - .../down.sql | 3 - .../up.sql | 2 - .../2025-01-29-055814_fix_title/down.sql | 2 - migrations/2025-01-29-055814_fix_title/up.sql | 2 - mise.toml | 125 ++ scripts/run-local-postgres.sh | 86 ++ src/main.rs | 12 +- src/post_listener.rs | 259 ---- src/rss_poller.rs | 178 +++ 27 files changed, 983 insertions(+), 965 deletions(-) delete mode 100644 migrations/.keep delete mode 100644 migrations/00000000000000_diesel_initial_setup/down.sql delete mode 100644 migrations/00000000000000_diesel_initial_setup/up.sql delete mode 100644 migrations/2025-01-20-053547_create_posts/down.sql delete mode 100644 migrations/2025-01-20-053547_create_posts/up.sql delete mode 100644 migrations/2025-01-20-054932_remove_timestamp/down.sql delete mode 100644 migrations/2025-01-20-054932_remove_timestamp/up.sql delete mode 100644 migrations/2025-01-20-061742_change_did_to_rkey/down.sql delete mode 100644 migrations/2025-01-20-061742_change_did_to_rkey/up.sql delete mode 100644 migrations/2025-01-21-005557_add_time_us/down.sql delete mode 100644 migrations/2025-01-21-005557_add_time_us/up.sql delete mode 100644 migrations/2025-01-21-010225_rename_table/down.sql delete mode 100644 migrations/2025-01-21-010225_rename_table/up.sql delete mode 100644 migrations/2025-01-21-011113_change_timestamp_to_text/down.sql delete mode 100644 migrations/2025-01-21-011113_change_timestamp_to_text/up.sql delete mode 100644 migrations/2025-01-29-055814_fix_title/down.sql delete mode 100644 migrations/2025-01-29-055814_fix_title/up.sql create mode 100755 scripts/run-local-postgres.sh delete mode 100644 src/post_listener.rs create mode 100644 src/rss_poller.rs diff --git a/Cargo.lock b/Cargo.lock index 191a67f..d9707e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -54,17 +54,6 @@ dependencies = [ "libc", ] -[[package]] -name = "async-lock" -version = "3.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" -dependencies = [ - "event-listener 5.4.0", - "event-listener-strategy", - "pin-project-lite", -] - [[package]] name = "async-stream" version = "0.3.6" @@ -102,15 +91,14 @@ dependencies = [ name = "at-comments" version = "0.4.7" dependencies = [ - "atrium-api", "chrono", - "jetstream-oxide", "log", + "reqwest", "rocket", "rocket_db_pools", + "rss", "serde", "tokio", - "url", ] [[package]] @@ -122,6 +110,19 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atom_syndication" +version = "0.12.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f68d23e2cb4fd958c705b91a6b4c80ceeaf27a9e11651272a8389d5ce1a4a3" +dependencies = [ + "chrono", + "derive_builder", + "diligent-date-parser", + "never", + "quick-xml", +] + [[package]] name = "atomic" version = "0.5.3" @@ -138,54 +139,10 @@ dependencies = [ ] [[package]] -name = "atrium-api" -version = "0.25.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46355d3245edc7b3160b2a45fe55d09a6963ebd3eee0252feb6b72fb0eb71463" -dependencies = [ - "atrium-common", - "atrium-xrpc", - "chrono", - "http 1.3.1", - "ipld-core", - "langtag", - "regex", - "serde", - "serde_bytes", - "serde_json", - "thiserror 1.0.69", - "tokio", - "trait-variant", -] - -[[package]] -name = "atrium-common" -version = "0.1.2" +name = "atomic-waker" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed5610654043faa396a5a15afac0ac646d76aebe45aebd7cef4f8b96b0ab7f4" -dependencies = [ - "dashmap", - "lru", - "moka", - "thiserror 1.0.69", - "tokio", - "trait-variant", - "web-time", -] - -[[package]] -name = "atrium-xrpc" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0216ad50ce34e9ff982e171c3659e65dedaa2ed5ac2994524debdc9a9647ffa8" -dependencies = [ - "http 1.3.1", - "serde", - "serde_html_form", - "serde_json", - "thiserror 1.0.69", - "trait-variant", -] +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" @@ -209,16 +166,16 @@ dependencies = [ ] [[package]] -name = "base-x" -version = "0.2.11" +name = "base64" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "base64" -version = "0.21.7" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "binascii" @@ -271,8 +228,6 @@ version = "1.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fcb57c740ae1daf453ae85f16e37396f672b039e00d9d866e07ddb24e328e3a" dependencies = [ - "jobserver", - "libc", "shlex", ] @@ -291,32 +246,8 @@ dependencies = [ "iana-time-zone", "js-sys", "num-traits", - "serde", "wasm-bindgen", - "windows-link", -] - -[[package]] -name = "cid" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3147d8272e8fa0ccd29ce51194dd98f79ddfb8191ba9e3409884e751798acf3a" -dependencies = [ - "core2", - "multibase", - "multihash", - "serde", - "serde_bytes", - "unsigned-varint", -] - -[[package]] -name = "concurrent-queue" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" -dependencies = [ - "crossbeam-utils", + "windows-link 0.2.0", ] [[package]] @@ -346,15 +277,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "core2" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" -dependencies = [ - "memchr", -] - [[package]] name = "cpufeatures" version = "0.2.17" @@ -379,24 +301,6 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" -[[package]] -name = "crossbeam-channel" -version = "0.5.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "crossbeam-queue" version = "0.3.12" @@ -423,43 +327,38 @@ dependencies = [ ] [[package]] -name = "dashmap" -version = "6.1.0" +name = "darling" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ - "cfg-if", - "crossbeam-utils", - "hashbrown 0.14.5", - "lock_api", - "once_cell", - "parking_lot_core", + "darling_core", + "darling_macro", ] [[package]] -name = "data-encoding" -version = "2.8.0" +name = "darling_core" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "575f75dfd25738df5b91b8e43e14d44bda14637a58fae779fd2b064f8bf3e010" - -[[package]] -name = "data-encoding-macro" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f9724adfcf41f45bf652b3995837669d73c4d49a1b5ac1ff82905ac7d9b5558" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" dependencies = [ - "data-encoding", - "data-encoding-macro-internal", + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.100", ] [[package]] -name = "data-encoding-macro-internal" -version = "0.1.15" +name = "darling_macro" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18e4fdb82bd54a12e42fb58a800dcae6b9e13982238ce2296dc3570b92148e1f" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ - "data-encoding", - "syn 1.0.109", + "darling_core", + "quote", + "syn 2.0.100", ] [[package]] @@ -471,6 +370,37 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.100", +] + [[package]] name = "devise" version = "0.4.2" @@ -515,6 +445,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "diligent-date-parser" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ede7d79366f419921e2e2f67889c12125726692a313bffb474bd5f37a581e9" +dependencies = [ + "chrono", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -583,27 +522,6 @@ version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" -[[package]] -name = "event-listener" -version = "5.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite", -] - -[[package]] -name = "event-listener-strategy" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c3e4e0dd3673c1139bf041f3008816d9cf2946bbfac2945c09e523b8d7b05b2" -dependencies = [ - "event-listener 5.4.0", - "pin-project-lite", -] - [[package]] name = "fastrand" version = "2.3.0" @@ -624,30 +542,12 @@ dependencies = [ "version_check", ] -[[package]] -name = "flume" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" -dependencies = [ - "futures-core", - "futures-sink", - "nanorand", - "spin", -] - [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foldhash" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" - [[package]] name = "foreign-types" version = "0.3.2" @@ -719,17 +619,6 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" -[[package]] -name = "futures-macro" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - [[package]] name = "futures-sink" version = "0.3.31" @@ -751,7 +640,6 @@ dependencies = [ "futures-channel", "futures-core", "futures-io", - "futures-macro", "futures-sink", "futures-task", "memchr", @@ -770,20 +658,7 @@ dependencies = [ "libc", "log", "rustversion", - "windows 0.48.0", -] - -[[package]] -name = "generator" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc6bd114ceda131d3b1d665eba35788690ad37f5916457286b32ab6fd3c438dd" -dependencies = [ - "cfg-if", - "libc", - "log", - "rustversion", - "windows 0.58.0", + "windows", ] [[package]] @@ -803,10 +678,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", - "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", - "wasm-bindgen", ] [[package]] @@ -852,6 +725,25 @@ dependencies = [ "tracing", ] +[[package]] +name = "h2" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.3.1", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -867,11 +759,6 @@ name = "hashbrown" version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash", -] [[package]] name = "hashlink" @@ -969,6 +856,29 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http 1.3.1", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http 1.3.1", + "http-body 1.0.1", + "pin-project-lite", +] + [[package]] name = "httparse" version = "1.10.1" @@ -991,9 +901,9 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", + "h2 0.3.26", "http 0.2.12", - "http-body", + "http-body 0.4.6", "httparse", "httpdate", "itoa", @@ -1005,6 +915,79 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2 0.4.13", + "http 1.3.1", + "http-body 1.0.1", + "httparse", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http 1.3.1", + "hyper 1.8.1", + "hyper-util", + "rustls 0.23.37", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.8.1", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.3.1", + "http-body 1.0.1", + "hyper 1.8.1", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + [[package]] name = "iana-time-zone" version = "0.1.62" @@ -1017,7 +1000,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.52.0", + "windows-core", ] [[package]] @@ -1147,6 +1130,12 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.0.3" @@ -1186,15 +1175,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" [[package]] -name = "ipld-core" -version = "0.4.2" +name = "ipnet" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "104718b1cc124d92a6d01ca9c9258a7df311405debb3408c445a36452f9bf8db" -dependencies = [ - "cid", - "serde", - "serde_bytes", -] +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] name = "is-terminal" @@ -1213,37 +1197,6 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" -[[package]] -name = "jetstream-oxide" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a267b4f381f1db945e9eb6b9ff0f3070950cc01521fef6265556b6a683e41e13" -dependencies = [ - "async-trait", - "atrium-api", - "chrono", - "flume", - "futures-util", - "log", - "serde", - "serde_json", - "thiserror 2.0.12", - "tokio", - "tokio-tungstenite", - "tokio-util", - "url", - "zstd", -] - -[[package]] -name = "jobserver" -version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" -dependencies = [ - "libc", -] - [[package]] name = "js-sys" version = "0.3.77" @@ -1254,15 +1207,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "langtag" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed60c85f254d6ae8450cec15eedd921efbc4d1bdf6fcf6202b9a58b403f6f805" -dependencies = [ - "serde", -] - [[package]] name = "lazy_static" version = "1.5.0" @@ -1310,7 +1254,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff50ecb28bb86013e935fb6683ab1f6d3a20016f123c76fd4c27470076ac30f5" dependencies = [ "cfg-if", - "generator 0.7.5", + "generator", "scoped-tls", "serde", "serde_json", @@ -1318,28 +1262,6 @@ dependencies = [ "tracing-subscriber", ] -[[package]] -name = "loom" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" -dependencies = [ - "cfg-if", - "generator 0.8.4", - "scoped-tls", - "tracing", - "tracing-subscriber", -] - -[[package]] -name = "lru" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" -dependencies = [ - "hashbrown 0.15.2", -] - [[package]] name = "matchers" version = "0.2.0" @@ -1397,28 +1319,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "moka" -version = "0.12.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9321642ca94a4282428e6ea4af8cc2ca4eac48ac7a6a4ea8f33f76d0ce70926" -dependencies = [ - "async-lock", - "crossbeam-channel", - "crossbeam-epoch", - "crossbeam-utils", - "event-listener 5.4.0", - "futures-util", - "loom 0.7.2", - "parking_lot", - "portable-atomic", - "rustc_version", - "smallvec", - "tagptr", - "thiserror 1.0.69", - "uuid", -] - [[package]] name = "multer" version = "3.1.0" @@ -1438,37 +1338,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "multibase" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b3539ec3c1f04ac9748a260728e855f261b4977f5c3406612c884564f329404" -dependencies = [ - "base-x", - "data-encoding", - "data-encoding-macro", -] - -[[package]] -name = "multihash" -version = "0.19.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b430e7953c29dd6a09afc29ff0bb69c6e306329ee6794700aee27b76a1aea8d" -dependencies = [ - "core2", - "serde", - "unsigned-varint", -] - -[[package]] -name = "nanorand" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" -dependencies = [ - "getrandom 0.2.15", -] - [[package]] name = "native-tls" version = "0.2.14" @@ -1486,6 +1355,12 @@ dependencies = [ "tempfile", ] +[[package]] +name = "never" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96aba5aa877601bb3f6dd6a63a969e1f82e60646e81e71b14496995e9853c91" + [[package]] name = "nom" version = "7.1.3" @@ -1577,15 +1452,6 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" -[[package]] -name = "openssl-src" -version = "300.4.2+3.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168ce4e058f975fe43e89d9ccf78ca668601887ae736090aacc23ae353c298e2" -dependencies = [ - "cc", -] - [[package]] name = "openssl-sys" version = "0.9.107" @@ -1594,17 +1460,10 @@ checksum = "8288979acd84749c744a9014b4382d42b8f7b2592847b5afb2ed29e5d16ede07" dependencies = [ "cc", "libc", - "openssl-src", "pkg-config", "vcpkg", ] -[[package]] -name = "parking" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" - [[package]] name = "parking_lot" version = "0.12.3" @@ -1681,12 +1540,6 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" -[[package]] -name = "portable-atomic" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" - [[package]] name = "powerfmt" version = "0.2.0" @@ -1724,6 +1577,16 @@ dependencies = [ "yansi", ] +[[package]] +name = "quick-xml" +version = "0.37.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +dependencies = [ + "encoding_rs", + "memchr", +] + [[package]] name = "quote" version = "1.0.40" @@ -1798,18 +1661,6 @@ dependencies = [ "syn 2.0.100", ] -[[package]] -name = "regex" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - [[package]] name = "regex-automata" version = "0.4.9" @@ -1827,6 +1678,50 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "reqwest" +version = "0.12.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb" +dependencies = [ + "base64 0.22.1", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2 0.4.13", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "hyper 1.8.1", + "hyper-rustls", + "hyper-tls", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile 2.2.0", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows-registry", +] + [[package]] name = "ring" version = "0.17.14" @@ -1928,7 +1823,7 @@ dependencies = [ "either", "futures", "http 0.2.12", - "hyper", + "hyper 0.14.32", "indexmap", "log", "memchr", @@ -1946,19 +1841,22 @@ dependencies = [ ] [[package]] -name = "rustc-demangle" -version = "0.1.24" +name = "rss" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +checksum = "b2107738f003660f0a91f56fd3e3bd3ab5d918b2ddaf1e1ec2136fb1c46f71bf" +dependencies = [ + "atom_syndication", + "derive_builder", + "never", + "quick-xml", +] [[package]] -name = "rustc_version" -version = "0.4.1" +name = "rustc-demangle" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" -dependencies = [ - "semver", -] +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustix" @@ -1980,26 +1878,68 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "ring", - "rustls-webpki", + "rustls-webpki 0.101.7", "sct", ] +[[package]] +name = "rustls" +version = "0.23.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +dependencies = [ + "once_cell", + "rustls-pki-types", + "rustls-webpki 0.103.9", + "subtle", + "zeroize", +] + [[package]] name = "rustls-pemfile" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "base64", + "ring", + "untrusted", ] [[package]] name = "rustls-webpki" -version = "0.101.7" +version = "0.103.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" dependencies = [ "ring", + "rustls-pki-types", "untrusted", ] @@ -2069,12 +2009,6 @@ dependencies = [ "libc", ] -[[package]] -name = "semver" -version = "1.0.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" - [[package]] name = "serde" version = "1.0.219" @@ -2084,15 +2018,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde_bytes" -version = "0.11.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8437fd221bde2d4ca316d61b90e337e9e702b3820b87d63caa9ba6c02bd06d96" -dependencies = [ - "serde", -] - [[package]] name = "serde_derive" version = "1.0.219" @@ -2104,19 +2029,6 @@ dependencies = [ "syn 2.0.100", ] -[[package]] -name = "serde_html_form" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d2de91cf02bbc07cde38891769ccd5d4f073d22a40683aa4bc7a95781aaa2c4" -dependencies = [ - "form_urlencoded", - "indexmap", - "itoa", - "ryu", - "serde", -] - [[package]] name = "serde_json" version = "1.0.140" @@ -2139,14 +2051,15 @@ dependencies = [ ] [[package]] -name = "sha1" -version = "0.10.6" +name = "serde_urlencoded" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ - "cfg-if", - "cpufeatures", - "digest", + "form_urlencoded", + "itoa", + "ryu", + "serde", ] [[package]] @@ -2214,9 +2127,6 @@ name = "spin" version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" -dependencies = [ - "lock_api", -] [[package]] name = "sqlformat" @@ -2252,7 +2162,7 @@ dependencies = [ "crc", "crossbeam-queue", "either", - "event-listener 2.5.3", + "event-listener", "futures-channel", "futures-core", "futures-intrusive", @@ -2266,14 +2176,14 @@ dependencies = [ "once_cell", "paste", "percent-encoding", - "rustls", - "rustls-pemfile", + "rustls 0.21.12", + "rustls-pemfile 1.0.4", "serde", "serde_json", "sha2", "smallvec", "sqlformat", - "thiserror 1.0.69", + "thiserror", "tokio", "tokio-stream", "tracing", @@ -2325,7 +2235,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c824eb80b894f926f89a0b9da0c7f435d27cdd35b8c655b114e58223918577e" dependencies = [ "atoi", - "base64", + "base64 0.21.7", "bitflags", "byteorder", "crc", @@ -2351,7 +2261,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 1.0.69", + "thiserror", "tracing", "whoami", ] @@ -2377,7 +2287,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b8c4a4445d81357df8b1a650d0d0d6fbbbfe99d064aa5e02f3e4022061476d8" dependencies = [ - "loom 0.5.6", + "loom", ] [[package]] @@ -2391,6 +2301,12 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.6.1" @@ -2419,6 +2335,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + [[package]] name = "synstructure" version = "0.13.1" @@ -2431,10 +2356,25 @@ dependencies = [ ] [[package]] -name = "tagptr" -version = "0.2.0" +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] [[package]] name = "tempfile" @@ -2455,16 +2395,7 @@ version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl 1.0.69", -] - -[[package]] -name = "thiserror" -version = "2.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" -dependencies = [ - "thiserror-impl 2.0.12", + "thiserror-impl", ] [[package]] @@ -2478,17 +2409,6 @@ dependencies = [ "syn 2.0.100", ] -[[package]] -name = "thiserror-impl" -version = "2.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - [[package]] name = "thread_local" version = "1.1.8" @@ -2565,7 +2485,6 @@ dependencies = [ "bytes", "libc", "mio", - "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", @@ -2595,28 +2514,24 @@ dependencies = [ ] [[package]] -name = "tokio-stream" -version = "0.1.17" +name = "tokio-rustls" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "futures-core", - "pin-project-lite", + "rustls 0.23.37", "tokio", ] [[package]] -name = "tokio-tungstenite" -version = "0.24.0" +name = "tokio-stream" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" dependencies = [ - "futures-util", - "log", - "native-tls", + "futures-core", + "pin-project-lite", "tokio", - "tokio-native-tls", - "tungstenite", ] [[package]] @@ -2666,6 +2581,27 @@ dependencies = [ "winnow", ] +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + [[package]] name = "tower-service" version = "0.3.3" @@ -2734,43 +2670,12 @@ dependencies = [ "tracing-log", ] -[[package]] -name = "trait-variant" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - [[package]] name = "try-lock" version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" -[[package]] -name = "tungstenite" -version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" -dependencies = [ - "byteorder", - "bytes", - "data-encoding", - "http 1.3.1", - "httparse", - "log", - "native-tls", - "rand", - "sha1", - "thiserror 1.0.69", - "url", - "utf-8", -] - [[package]] name = "typenum" version = "1.18.0" @@ -2841,12 +2746,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" -[[package]] -name = "unsigned-varint" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" - [[package]] name = "untrusted" version = "0.9.0" @@ -2864,12 +2763,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "utf-8" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" - [[package]] name = "utf16_iter" version = "1.0.5" @@ -2882,15 +2775,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" -[[package]] -name = "uuid" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" -dependencies = [ - "getrandom 0.3.2", -] - [[package]] name = "valuable" version = "0.1.1" @@ -2965,6 +2849,19 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.100" @@ -2998,10 +2895,10 @@ dependencies = [ ] [[package]] -name = "web-time" -version = "1.1.0" +name = "web-sys" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" dependencies = [ "js-sys", "wasm-bindgen", @@ -3032,16 +2929,6 @@ dependencies = [ "windows-targets 0.48.5", ] -[[package]] -name = "windows" -version = "0.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6" -dependencies = [ - "windows-core 0.58.0", - "windows-targets 0.52.6", -] - [[package]] name = "windows-core" version = "0.52.0" @@ -3052,63 +2939,44 @@ dependencies = [ ] [[package]] -name = "windows-core" -version = "0.58.0" +name = "windows-link" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-result", - "windows-strings", - "windows-targets 0.52.6", -] +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" [[package]] -name = "windows-implement" -version = "0.58.0" +name = "windows-link" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] +checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65" [[package]] -name = "windows-interface" -version = "0.58.0" +name = "windows-registry" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" +checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", + "windows-result", + "windows-strings", + "windows-targets 0.53.4", ] -[[package]] -name = "windows-link" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65" - [[package]] name = "windows-result" -version = "0.2.0" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" dependencies = [ - "windows-targets 0.52.6", + "windows-link 0.1.3", ] [[package]] name = "windows-strings" -version = "0.1.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" dependencies = [ - "windows-result", - "windows-targets 0.52.6", + "windows-link 0.1.3", ] [[package]] @@ -3162,13 +3030,30 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d42b7b7f66d2a06854650af09cfdf8713e427a439c97ad65a6375318033ac4b" +dependencies = [ + "windows-link 0.2.0", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -3181,6 +3066,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -3193,6 +3084,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -3205,12 +3102,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -3223,6 +3132,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -3235,6 +3150,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -3247,6 +3168,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -3259,6 +3186,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + [[package]] name = "winnow" version = "0.7.4" @@ -3383,6 +3316,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + [[package]] name = "zerovec" version = "0.10.4" @@ -3404,31 +3343,3 @@ dependencies = [ "quote", "syn 2.0.100", ] - -[[package]] -name = "zstd" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "7.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" -dependencies = [ - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.15+zstd.1.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" -dependencies = [ - "cc", - "pkg-config", -] diff --git a/Cargo.toml b/Cargo.toml index c7d941f..c92963f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,9 +7,8 @@ license = "MIT" [dependencies] rocket = { version = "0.5.1", features = ["json"] } tokio = { version = "1", features = ["rt-multi-thread", "macros", "time"] } -url = "2" -jetstream-oxide = "0.1.2" -atrium-api = "0.25.4" +reqwest = "0.12" +rss = "2.0" log = "0.4.28" serde = "1.0.217" chrono = "0.4.42" diff --git a/Dockerfile b/Dockerfile index 6dd4d9c..f2dbadc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/rust:1.84.0 AS build +FROM docker.io/rust:latest AS build ## cargo package name: customize here or provide via --build-arg ARG pkg=at-comments @@ -11,11 +11,11 @@ WORKDIR /build COPY . . RUN --mount=type=cache,target=/build/target \ - --mount=type=cache,target=/usr/local/cargo/registry \ - --mount=type=cache,target=/usr/local/cargo/git \ - set -eux; \ - cargo build --release; \ - objcopy --compress-debug-sections target/release/$pkg ./main + --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + set -eux; \ + cargo build --release; \ + objcopy --compress-debug-sections target/release/$pkg ./main ################################################################################ diff --git a/README.md b/README.md index 557feed..9937ce0 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ const getPostAndThreadData = async ( slug: string, setThread, setUri, - setError + setError, ) => { const agent = new Agent("https://public.api.bsky.app"); const assembledUrl = "https://jack.is/posts/" + slug.toString(); @@ -41,13 +41,65 @@ const getPostAndThreadData = async ( ``` "oh no" was my reaction, so I decided to put together a little database project -that uses the Bluesky jetstream and persists all the relevant post rkeys in a -database, so that then my clientside code can just make a request to the API -server. +that polls my Bluesky RSS feed every 15 minutes and persists all the relevant post +rkeys in a database, so that my clientside code can just make a request to the API server. + +## How It Works + +The service polls the Bluesky RSS feed at `https://bsky.app/profile/{handle}/rss` every 15 minutes. +It looks for posts starting with 📝 that contain links to the configured blog domain, extracts the slug +from the URL and the rkey from the post's AT-URI, then stores the mapping in a PostgreSQL database. + +The API endpoint `GET /slug/` returns the post metadata (rkey, time_us) which the client-side +code uses to fetch the full post thread from Bluesky. + +For immediate indexing without waiting for the 15-minute polling interval, the blog's client-side +code includes fallback logic to check the RSS feed directly if a post isn't found in the database. + +## Configuration + +Configure in `Rocket.toml`: + +- `poster_handle`: Your Bluesky handle (e.g., "jack.is") +- `target_emoji`: Emoji prefix to identify blog posts (e.g., "📝") +- `blog_domain`: Your blog's domain to match URLs (e.g., "jack.is") + +## Local Postgres Runner + +If you want to run the app locally without a DB sidecar setup, use: + +```bash +mise run db-setup +``` + +This task: + +- Starts a Postgres container on `localhost:54329` +- Creates the `posts` table directly (ignores migrations) +- Seeds sample rows +- Uses `ROCKET_DATABASES` from `mise.toml` + +Then start the app: + +```bash +mise run dev +``` + +Quick test: + +```bash +curl http://127.0.0.1:4321/slug/hello-world +``` + +To stop/remove the local Postgres container: + +```bash +mise run db-down +``` ## Caveats This work so far makes a whole lot of assumptions. It only tracks a single user -currently. You could probably add more fields and DID handling to allow for -multiple separate Bluesky users to be tracked, but given that I'm probably the -only one that will be using it, I'm not too worried about that limitation. +currently. You could probably add more fields and handle multiple Bluesky users +to be tracked, but given that I'm probably the only one that will be using it, +I'm not too worried about that limitation. diff --git a/Rocket.toml b/Rocket.toml index d301291..c858dea 100644 --- a/Rocket.toml +++ b/Rocket.toml @@ -1,10 +1,12 @@ [debug] -poster_did = "did:plc:gaiatc34nb7yhkgoxrf44gpl" +poster_handle = "jack.is" target_emoji = "📝" +blog_domain = "jack.is" [release] -poster_did = "did:plc:cwdkf4xxjpznceembuuspt3d" +poster_handle = "jack.is" target_emoji = "📝" +blog_domain = "jack.is" [default] port = 4321 diff --git a/migrations/.keep b/migrations/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/migrations/00000000000000_diesel_initial_setup/down.sql b/migrations/00000000000000_diesel_initial_setup/down.sql deleted file mode 100644 index a9f5260..0000000 --- a/migrations/00000000000000_diesel_initial_setup/down.sql +++ /dev/null @@ -1,6 +0,0 @@ --- This file was automatically created by Diesel to setup helper functions --- and other internal bookkeeping. This file is safe to edit, any future --- changes will be added to existing projects as new migrations. - -DROP FUNCTION IF EXISTS diesel_manage_updated_at(_tbl regclass); -DROP FUNCTION IF EXISTS diesel_set_updated_at(); diff --git a/migrations/00000000000000_diesel_initial_setup/up.sql b/migrations/00000000000000_diesel_initial_setup/up.sql deleted file mode 100644 index d68895b..0000000 --- a/migrations/00000000000000_diesel_initial_setup/up.sql +++ /dev/null @@ -1,36 +0,0 @@ --- This file was automatically created by Diesel to setup helper functions --- and other internal bookkeeping. This file is safe to edit, any future --- changes will be added to existing projects as new migrations. - - - - --- Sets up a trigger for the given table to automatically set a column called --- `updated_at` whenever the row is modified (unless `updated_at` was included --- in the modified columns) --- --- # Example --- --- ```sql --- CREATE TABLE users (id SERIAL PRIMARY KEY, updated_at TIMESTAMP NOT NULL DEFAULT NOW()); --- --- SELECT diesel_manage_updated_at('users'); --- ``` -CREATE OR REPLACE FUNCTION diesel_manage_updated_at(_tbl regclass) RETURNS VOID AS $$ -BEGIN - EXECUTE format('CREATE TRIGGER set_updated_at BEFORE UPDATE ON %s - FOR EACH ROW EXECUTE PROCEDURE diesel_set_updated_at()', _tbl); -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION diesel_set_updated_at() RETURNS trigger AS $$ -BEGIN - IF ( - NEW IS DISTINCT FROM OLD AND - NEW.updated_at IS NOT DISTINCT FROM OLD.updated_at - ) THEN - NEW.updated_at := current_timestamp; - END IF; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; diff --git a/migrations/2025-01-20-053547_create_posts/down.sql b/migrations/2025-01-20-053547_create_posts/down.sql deleted file mode 100644 index fe33dfa..0000000 --- a/migrations/2025-01-20-053547_create_posts/down.sql +++ /dev/null @@ -1,2 +0,0 @@ --- This file should undo anything in `up.sql` -DROP TABLE posts; \ No newline at end of file diff --git a/migrations/2025-01-20-053547_create_posts/up.sql b/migrations/2025-01-20-053547_create_posts/up.sql deleted file mode 100644 index 997f4ed..0000000 --- a/migrations/2025-01-20-053547_create_posts/up.sql +++ /dev/null @@ -1,7 +0,0 @@ --- Your SQL goes here -CREATE TABLE posts ( - id SERIAL PRIMARY KEY, - slug TEXT UNIQUE NOT NULL, - post_did TEXT NOT NULL, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP -); \ No newline at end of file diff --git a/migrations/2025-01-20-054932_remove_timestamp/down.sql b/migrations/2025-01-20-054932_remove_timestamp/down.sql deleted file mode 100644 index 4feba5b..0000000 --- a/migrations/2025-01-20-054932_remove_timestamp/down.sql +++ /dev/null @@ -1,2 +0,0 @@ --- This file should undo anything in `up.sql` -ALTER TABLE posts ADD COLUMN created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP; \ No newline at end of file diff --git a/migrations/2025-01-20-054932_remove_timestamp/up.sql b/migrations/2025-01-20-054932_remove_timestamp/up.sql deleted file mode 100644 index b6498ae..0000000 --- a/migrations/2025-01-20-054932_remove_timestamp/up.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Your SQL goes here -ALTER TABLE posts DROP COLUMN created_at; \ No newline at end of file diff --git a/migrations/2025-01-20-061742_change_did_to_rkey/down.sql b/migrations/2025-01-20-061742_change_did_to_rkey/down.sql deleted file mode 100644 index 96f6e9b..0000000 --- a/migrations/2025-01-20-061742_change_did_to_rkey/down.sql +++ /dev/null @@ -1,2 +0,0 @@ --- This file should undo anything in `up.sql` -ALTER TABLE posts RENAME COLUMN rkey TO post_did; \ No newline at end of file diff --git a/migrations/2025-01-20-061742_change_did_to_rkey/up.sql b/migrations/2025-01-20-061742_change_did_to_rkey/up.sql deleted file mode 100644 index f39d2e1..0000000 --- a/migrations/2025-01-20-061742_change_did_to_rkey/up.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Your SQL goes here -ALTER TABLE posts RENAME COLUMN post_did TO rkey; \ No newline at end of file diff --git a/migrations/2025-01-21-005557_add_time_us/down.sql b/migrations/2025-01-21-005557_add_time_us/down.sql deleted file mode 100644 index cc14793..0000000 --- a/migrations/2025-01-21-005557_add_time_us/down.sql +++ /dev/null @@ -1,2 +0,0 @@ --- This file should undo anything in `up.sql` -ALTER TABLE posts DROP COLUMN time_us; \ No newline at end of file diff --git a/migrations/2025-01-21-005557_add_time_us/up.sql b/migrations/2025-01-21-005557_add_time_us/up.sql deleted file mode 100644 index fe3d714..0000000 --- a/migrations/2025-01-21-005557_add_time_us/up.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Your SQL goes here -ALTER TABLE posts ADD COLUMN time_us TIMESTAMP; \ No newline at end of file diff --git a/migrations/2025-01-21-010225_rename_table/down.sql b/migrations/2025-01-21-010225_rename_table/down.sql deleted file mode 100644 index ce859ef..0000000 --- a/migrations/2025-01-21-010225_rename_table/down.sql +++ /dev/null @@ -1,2 +0,0 @@ --- This file should undo anything in `up.sql` -ALTER TABLE slug_dids RENAME TO posts; \ No newline at end of file diff --git a/migrations/2025-01-21-010225_rename_table/up.sql b/migrations/2025-01-21-010225_rename_table/up.sql deleted file mode 100644 index 1b60c4e..0000000 --- a/migrations/2025-01-21-010225_rename_table/up.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Your SQL goes here -ALTER TABLE posts RENAME TO slug_dids; \ No newline at end of file diff --git a/migrations/2025-01-21-011113_change_timestamp_to_text/down.sql b/migrations/2025-01-21-011113_change_timestamp_to_text/down.sql deleted file mode 100644 index 08787f7..0000000 --- a/migrations/2025-01-21-011113_change_timestamp_to_text/down.sql +++ /dev/null @@ -1,3 +0,0 @@ --- This file should undo anything in `up.sql` -ALTER TABLE slug_dids -ALTER COLUMN time_us TYPE TIMESTAMP; diff --git a/migrations/2025-01-21-011113_change_timestamp_to_text/up.sql b/migrations/2025-01-21-011113_change_timestamp_to_text/up.sql deleted file mode 100644 index 1a10ff6..0000000 --- a/migrations/2025-01-21-011113_change_timestamp_to_text/up.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Your SQL goes here -ALTER TABLE posts ALTER COLUMN time_us TYPE TEXT; \ No newline at end of file diff --git a/migrations/2025-01-29-055814_fix_title/down.sql b/migrations/2025-01-29-055814_fix_title/down.sql deleted file mode 100644 index 94d397f..0000000 --- a/migrations/2025-01-29-055814_fix_title/down.sql +++ /dev/null @@ -1,2 +0,0 @@ --- This file should undo anything in `up.sql` -ALTER TABLE posts RENAME TO slug_dids; \ No newline at end of file diff --git a/migrations/2025-01-29-055814_fix_title/up.sql b/migrations/2025-01-29-055814_fix_title/up.sql deleted file mode 100644 index ce859ef..0000000 --- a/migrations/2025-01-29-055814_fix_title/up.sql +++ /dev/null @@ -1,2 +0,0 @@ --- This file should undo anything in `up.sql` -ALTER TABLE slug_dids RENAME TO posts; \ No newline at end of file diff --git a/mise.toml b/mise.toml index 3770239..4156615 100644 --- a/mise.toml +++ b/mise.toml @@ -1,2 +1,127 @@ [tools] rust = "1.92.0" + +[env] +POSTGRES_CONTAINER = "at-comments-postgres" +POSTGRES_DB = "at_comments" +POSTGRES_USER = "at_comments" +POSTGRES_PASSWORD = "at_comments" +POSTGRES_PORT = "54329" +ROCKET_DATABASES = '{bluesky_comments={url="postgres://at_comments:at_comments@localhost:54329/at_comments"}}' + +[tasks.db-up] +description = "Start local Postgres container" +run = ''' +set -euo pipefail + +if ! command -v docker >/dev/null 2>&1; then + echo "docker is required but not installed or not in PATH" + exit 1 +fi + +if docker ps -a --filter "name=^/${POSTGRES_CONTAINER}$" --quiet | grep -q .; then + if docker ps --filter "name=^/${POSTGRES_CONTAINER}$" --filter "status=running" --quiet | grep -q .; then + echo "Container '$POSTGRES_CONTAINER' is already running" + else + echo "Starting existing container '$POSTGRES_CONTAINER'" + docker start "$POSTGRES_CONTAINER" >/dev/null + fi +else + echo "Creating container '$POSTGRES_CONTAINER' on localhost:$POSTGRES_PORT" + docker run -d \ + --name "$POSTGRES_CONTAINER" \ + -e POSTGRES_DB="$POSTGRES_DB" \ + -e POSTGRES_USER="$POSTGRES_USER" \ + -e POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \ + -p "$POSTGRES_PORT:5432" \ + postgres:16-alpine >/dev/null +fi + +echo "Waiting for Postgres to become ready..." +for _ in $(seq 1 45); do + if docker exec "$POSTGRES_CONTAINER" pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB" >/dev/null 2>&1; then + echo "Postgres is ready" + exit 0 + fi + sleep 1 +done + +echo "Postgres did not become ready in time" +exit 1 +''' + +[tasks.db-init] +description = "Create posts table (no migrations)" +depends = ["db-up"] +run = ''' +set -euo pipefail + +docker exec -i "$POSTGRES_CONTAINER" psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" <<'SQL' +CREATE TABLE IF NOT EXISTS posts ( + id SERIAL PRIMARY KEY, + slug TEXT UNIQUE NOT NULL, + rkey TEXT NOT NULL, + time_us TEXT NOT NULL +); +SQL + +echo "Schema ready" +''' + +[tasks.db-seed] +description = "Seed sample post rows" +depends = ["db-init"] +run = ''' +set -euo pipefail + +docker exec -i "$POSTGRES_CONTAINER" psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" <<'SQL' +INSERT INTO posts (slug, rkey, time_us) VALUES + ('hello-world', '3lch7i6x4x22m', '1738411000000000'), + ('rust-and-rockets', '3lch7i6x4x22n', '1738414600000000'), + ('using-bluesky-comments', '3lch7i6x4x22o', '1738418200000000') +ON CONFLICT (slug) DO UPDATE SET + rkey = EXCLUDED.rkey, + time_us = EXCLUDED.time_us; +SQL + +echo "Seed data loaded" +''' + +[tasks.db-setup] +description = "Start local Postgres, create schema, and seed rows" +depends = ["db-seed"] +run = ''' +echo "Local Postgres is ready" +echo "Run the app with: mise run dev" +echo "Quick test: curl http://127.0.0.1:4321/slug/hello-world" +''' + +[tasks.db-down] +description = "Stop and remove local Postgres container" +run = ''' +set -euo pipefail + +if docker ps -a --filter "name=^/${POSTGRES_CONTAINER}$" --quiet | grep -q .; then + docker rm -f "$POSTGRES_CONTAINER" + echo "Removed container '$POSTGRES_CONTAINER'" +else + echo "Container '$POSTGRES_CONTAINER' does not exist" +fi +''' + +[tasks.db-connect] +description = "Open psql shell in local Postgres container" +depends = ["db-up"] +run = ''' +set -euo pipefail + +if [ -t 0 ] && [ -t 1 ]; then + exec docker exec -it "$POSTGRES_CONTAINER" psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" +else + exec docker exec -i "$POSTGRES_CONTAINER" psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" +fi +''' + +[tasks.dev] +description = "Run at-comments with local mise environment" +run = "cargo run" diff --git a/scripts/run-local-postgres.sh b/scripts/run-local-postgres.sh new file mode 100755 index 0000000..6000480 --- /dev/null +++ b/scripts/run-local-postgres.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +set -euo pipefail + +CONTAINER_NAME="${CONTAINER_NAME:-at-comments-postgres}" +POSTGRES_DB="${POSTGRES_DB:-at_comments}" +POSTGRES_USER="${POSTGRES_USER:-at_comments}" +POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-at_comments}" +POSTGRES_PORT="${POSTGRES_PORT:-54329}" +SEED_DATA="${SEED_DATA:-1}" + +if ! command -v docker >/dev/null 2>&1; then + echo "docker is required but not installed or not in PATH" + exit 1 +fi + +container_exists() { + docker ps -a --format '{{.Names}}' | grep -qx "$CONTAINER_NAME" +} + +container_running() { + [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER_NAME" 2>/dev/null || true)" = "true" ] +} + +if container_exists; then + if container_running; then + echo "Container '$CONTAINER_NAME' is already running" + else + echo "Starting existing container '$CONTAINER_NAME'" + docker start "$CONTAINER_NAME" >/dev/null + fi +else + echo "Creating container '$CONTAINER_NAME' on localhost:$POSTGRES_PORT" + docker run -d \ + --name "$CONTAINER_NAME" \ + -e POSTGRES_DB="$POSTGRES_DB" \ + -e POSTGRES_USER="$POSTGRES_USER" \ + -e POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \ + -p "$POSTGRES_PORT:5432" \ + postgres:16-alpine >/dev/null +fi + +echo "Waiting for Postgres to become ready..." +for _ in $(seq 1 45); do + if docker exec "$CONTAINER_NAME" pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB" >/dev/null 2>&1; then + break + fi + sleep 1 +done + +if ! docker exec "$CONTAINER_NAME" pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB" >/dev/null 2>&1; then + echo "Postgres did not become ready in time" + exit 1 +fi + +echo "Creating schema (no migrations)..." +docker exec -i "$CONTAINER_NAME" psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" <<'SQL' +CREATE TABLE IF NOT EXISTS posts ( + id SERIAL PRIMARY KEY, + slug TEXT UNIQUE NOT NULL, + rkey TEXT NOT NULL, + time_us TEXT NOT NULL +); +SQL + +if [ "$SEED_DATA" = "1" ]; then + echo "Seeding sample rows..." + docker exec -i "$CONTAINER_NAME" psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" <<'SQL' +INSERT INTO posts (slug, rkey, time_us) VALUES + ('hello-world', '3lch7i6x4x22m', '1738411000000000'), + ('rust-and-rockets', '3lch7i6x4x22n', '1738414600000000'), + ('using-bluesky-comments', '3lch7i6x4x22o', '1738418200000000') +ON CONFLICT (slug) DO UPDATE SET + rkey = EXCLUDED.rkey, + time_us = EXCLUDED.time_us; +SQL +fi + +echo +echo "Local Postgres is ready." +DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:${POSTGRES_PORT}/${POSTGRES_DB}" +printf "Run app with:\n" +printf "ROCKET_DATABASES='{bluesky_comments={url=\"%s\"}}' cargo run\n" "$DATABASE_URL" +printf "\nQuick test:\n" +printf "curl http://127.0.0.1:4321/slug/hello-world\n" +printf "\nTo stop/remove DB container later:\n" +printf "docker rm -f %s\n" "$CONTAINER_NAME" diff --git a/src/main.rs b/src/main.rs index 8060a90..33d4ff3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,7 +2,7 @@ extern crate rocket; pub mod models; -mod post_listener; +mod rss_poller; use models::Meta; use rocket::fairing::{self, Fairing}; @@ -16,13 +16,13 @@ use rocket_db_pools::{sqlx, Connection, Database}; #[database("bluesky_comments")] struct Comments(sqlx::PgPool); -struct ListenerFairing; +struct PollerFairing; #[rocket::async_trait] -impl Fairing for ListenerFairing { +impl Fairing for PollerFairing { fn info(&self) -> rocket::fairing::Info { rocket::fairing::Info { - name: "Jetstream listener", + name: "RSS Poller", kind: rocket::fairing::Kind::Ignite, } } @@ -32,7 +32,7 @@ impl Fairing for ListenerFairing { Some(pool) => pool.0.clone(), None => return Err(rocket), }; - rocket::tokio::task::spawn(post_listener::websocket_listener(pool)); + rocket::tokio::task::spawn(rss_poller::rss_polling_task(pool)); Ok(rocket) } } @@ -111,7 +111,7 @@ fn service_unavailable() -> Value { fn rocket() -> _ { rocket::build() .attach(Comments::init()) // init the database - .attach(ListenerFairing) + .attach(PollerFairing) .register("/", catchers![not_found]) .register("/slug", catchers![slug_not_found, service_unavailable]) .mount("/", routes![index, post_meta, slug_root]) diff --git a/src/post_listener.rs b/src/post_listener.rs deleted file mode 100644 index 7e044a3..0000000 --- a/src/post_listener.rs +++ /dev/null @@ -1,259 +0,0 @@ -use atrium_api::app::bsky::richtext::facet::MainFeaturesItem; -use atrium_api::record::KnownRecord::AppBskyFeedPost; -use jetstream_oxide::{ - events::{commit::CommitEvent, JetstreamEvent::Commit}, - exports::{Did, Nsid}, - DefaultJetstreamEndpoints, JetstreamCompression, JetstreamConfig, JetstreamConnector, -}; - -use rocket::Config; -use rocket_db_pools::sqlx::{self}; -use serde::Deserialize; -use sqlx::Row; - -use tokio::net::TcpStream; -use tokio::time::{sleep, timeout, Duration}; -use url::Url; - -use std::time::Instant; - -#[derive(Deserialize, Debug)] -struct ListenerConfig { - poster_did: String, - target_emoji: String, -} - -/// Quick TCP reachability check for a websocket `wss://` endpoint. -/// It attempts a TCP connect to the host:port within `timeout_ms` milliseconds. -async fn check_ws_endpoint_reachable(endpoint: &str, timeout_ms: u64) -> Result<(), String> { - let url = Url::parse(endpoint).map_err(|e| format!("Bad endpoint URL: {}", e))?; - let host = url - .host_str() - .ok_or_else(|| "No host in endpoint URL".to_string())?; - let port = url - .port_or_known_default() - .ok_or_else(|| "No port and no default".to_string())?; - - let addr = format!("{host}:{port}"); - let dur = Duration::from_millis(timeout_ms); - match timeout(dur, TcpStream::connect(&addr)).await { - Ok(Ok(_stream)) => Ok(()), - Ok(Err(e)) => Err(format!("TCP connect to {addr} failed: {}", e)), - Err(_) => Err(format!( - "TCP connect to {addr} timed out after {}ms", - timeout_ms - )), - } -} - -/// Consumer-side Jetstream listener with preflight + backoff + endpoint toggling. -/// This keeps the consumer API unchanged (no jetstream-oxide edits required). -pub async fn websocket_listener(pool: sqlx::Pool) { - let listener_config = Config::figment().extract::().unwrap(); - - let nsid = vec![Nsid::new("app.bsky.feed.post".to_string()).unwrap()]; - let did = vec![Did::new(listener_config.poster_did.clone()).unwrap()]; - - // Endpoints to rotate between on failures. - let endpoints: Vec = vec![ - DefaultJetstreamEndpoints::USWestOne.into(), - DefaultJetstreamEndpoints::USEastTwo.into(), - ]; - let mut endpoint_index: usize = 0; - - // Backoff parameters - let mut retry_attempt: u32 = 0; - let base_delay_ms: u64 = 1_000; // 1s - let max_delay_ms: u64 = 30_000; // 30s - let success_threshold_s: u64 = 15; // reset retries if connection lasted this long - - loop { - // Recompute last persisted cursor from DB before each attempt (so we resume where we left off). - let mut cursor = match sqlx::query("SELECT time_us FROM posts ORDER BY id DESC LIMIT 1") - .fetch_one(&pool) - .await - { - Ok(row) => { - let time = row.get::(0); - chrono::DateTime::from_timestamp_micros(time.parse::().unwrap()) - } - Err(_) => None, - }; - - // If the cursor is older than a day, drop it to avoid massive replay. - let diff = match cursor { - Some(c) => chrono::Utc::now() - c, - None => chrono::Duration::days(1), - }; - if diff.num_days() >= 1 { - log::warn!("[jetstream] Cursor is more than a day old, resetting cursor"); - cursor = None; - } - - // Preflight TCP check to avoid returning a connector when host is obviously unreachable. - let endpoint_str = endpoints[endpoint_index].as_str(); - match check_ws_endpoint_reachable(endpoint_str, 1500).await { - Ok(()) => { - log::warn!("[jetstream] Endpoint {} reachable (TCP)", endpoint_str); - } - Err(err) => { - log::warn!( - "[jetstream] Endpoint {} not reachable: {}. toggling endpoint and backing off", - endpoint_str, - err - ); - // Toggle endpoint and backoff - endpoint_index = (endpoint_index + 1) % endpoints.len(); - let delay = - (base_delay_ms * (2_u64.saturating_pow(retry_attempt))).min(max_delay_ms); - log::error!("[jetstream] Will retry connect in {}ms", delay); - sleep(Duration::from_millis(delay)).await; - retry_attempt = retry_attempt.saturating_add(1); - continue; - } - } - - let jetstream_config = JetstreamConfig { - endpoint: endpoints[endpoint_index].clone(), - wanted_dids: did.clone(), - compression: JetstreamCompression::Zstd, - cursor, - wanted_collections: nsid.clone(), - max_retries: 0, - ..Default::default() - }; - - let jetstream = match JetstreamConnector::new(jetstream_config) { - Ok(j) => j, - Err(e) => { - log::error!("[jetstream] Failed to create Jetstream connector: {}", e); - // Toggle endpoint and backoff - endpoint_index = (endpoint_index + 1) % endpoints.len(); - log::warn!( - "[jetstream] Toggling endpoint due to connector error. Next endpoint: {}", - endpoints[endpoint_index] - ); - let delay = - (base_delay_ms * (2_u64.saturating_pow(retry_attempt))).min(max_delay_ms); - log::error!("[jetstream] Will retry creating connector in {}ms", delay); - sleep(Duration::from_millis(delay)).await; - retry_attempt = retry_attempt.saturating_add(1); - continue; - } - }; - - // Use existing connector API (which spawns the background websocket task) and receive a channel. - let receiver = match jetstream.connect().await { - Ok(conn) => conn, - Err(e) => { - log::error!("[jetstream] Failed to connect to Jetstream: {}", e); - endpoint_index = (endpoint_index + 1) % endpoints.len(); - log::warn!( - "[jetstream] Toggling endpoint due to connect error. Next endpoint: {}", - endpoints[endpoint_index] - ); - let delay = - (base_delay_ms * (2_u64.saturating_pow(retry_attempt))).min(max_delay_ms); - log::error!("[jetstream] Will retry connect in {}ms", delay); - sleep(Duration::from_millis(delay)).await; - retry_attempt = retry_attempt.saturating_add(1); - continue; - } - }; - - log::warn!( - "[jetstream] Listening for: {} and emoji: {} on endpoint {}", - listener_config.poster_did, - listener_config.target_emoji, - endpoints[endpoint_index] - ); - - // Track connection health to reset backoff on healthy sessions. - let connected_at = Instant::now(); - let mut saw_event = false; - - // Consume events until the receiver is closed; then reconnect. - // Note: JetstreamReceiver's `recv_async` is used by existing consumer code in this repo. - while let Ok(event) = receiver.recv_async().await { - saw_event = true; - - if let Commit(CommitEvent::Create { info, commit }) = event { - if let AppBskyFeedPost(record) = commit.record { - // check and see if this post is what we're looking for - log::warn!("[jetstream] Checking record: {}", record.text); - if record.text.starts_with(&listener_config.target_emoji) { - if let Some(facets) = record.facets.clone() { - let features = facets - .iter() - .flat_map(|facet| &facet.features) - .filter_map(|feature| { - if let atrium_api::types::Union::Refs(MainFeaturesItem::Link( - link, - )) = feature - { - Some((commit.info.rkey.clone(), link.uri.clone())) - } else { - None - } - }) - .collect::>(); - - for (rkey, uri) in features { - let Some(slug) = uri.split_terminator('/').next_back() else { - log::error!( - "[jetstream] Failed to extract slug from URI: {}", - uri - ); - continue; - }; - - let time_us = info.time_us.to_string(); - - if let Err(e) = sqlx::query("INSERT INTO posts (slug, rkey, time_us) VALUES ($1, $2, $3) ON CONFLICT (slug) DO NOTHING") - .bind(slug) - .bind(&rkey) - .bind(&time_us) - .execute(&pool) - .await - { - log::error!("[jetstream] Failed to insert post: {}", e); - } else { - log::warn!("[jetstream] Inserted post"); - } - } - } - } - } - } - } - - // Receiver closed: treat as disconnection and attempt reconnect. - log::error!("[jetstream] Receiver closed; will attempt to reconnect"); - - // Toggle endpoint on unexpected disconnect so next attempt tries the other instance. - endpoint_index = (endpoint_index + 1) % endpoints.len(); - log::warn!( - "[jetstream] Toggling endpoint due to disconnect. Next endpoint: {}", - endpoints[endpoint_index] - ); - - // Reset retry_attempt if this connection was healthy (saw events or lasted past threshold). - if saw_event || connected_at.elapsed().as_secs() > success_threshold_s { - retry_attempt = 0; - log::warn!("[jetstream] Connection was healthy; resetting retry attempts"); - } else { - retry_attempt = retry_attempt.saturating_add(1); - } - - // Backoff before reconnecting. - let delay = (base_delay_ms * (2_u64.saturating_pow(retry_attempt))).min(max_delay_ms); - log::error!( - "[jetstream] Reconnecting in {}ms (attempt #{})", - delay, - retry_attempt - ); - sleep(Duration::from_millis(delay)).await; - - // Outer loop continues and will attempt the next endpoint / reconnect. - } -} diff --git a/src/rss_poller.rs b/src/rss_poller.rs new file mode 100644 index 0000000..d08765a --- /dev/null +++ b/src/rss_poller.rs @@ -0,0 +1,178 @@ +use rocket::Config; +use rocket_db_pools::sqlx::{self}; +use serde::Deserialize; +use tokio::time::{sleep, Duration}; + +#[derive(Deserialize, Debug)] +struct PollerConfig { + poster_handle: String, + target_emoji: String, + blog_domain: String, +} + +/// Fetch and parse RSS feed from Bluesky profile +async fn fetch_rss(handle: &str) -> Result { + let url = format!("https://bsky.app/profile/{}/rss", handle); + + let response = reqwest::get(&url) + .await + .map_err(|e| format!("Failed to fetch RSS: {}", e))?; + + let content = response + .text() + .await + .map_err(|e| format!("Failed to read RSS content: {}", e))?; + + rss::Channel::read_from(content.as_bytes()).map_err(|e| format!("Failed to parse RSS: {}", e)) +} + +/// Extract rkey from Bluesky post AT-URI +/// Format: at://did:plc:xxx/app.bsky.feed.post/rkey +fn extract_rkey(uri: &str) -> Option { + uri.split('/').next_back().map(|s| s.to_string()) +} + +/// Extract slug from blog URL +fn extract_slug_from_url(url: &str, blog_domain: &str) -> Option { + if !url.contains(blog_domain) { + return None; + } + + // Remove query/fragment and normalize trailing slash URLs. + let without_query = url.split('?').next().unwrap_or(url); + let without_fragment = without_query.split('#').next().unwrap_or(without_query); + let normalized = without_fragment.trim_end_matches('/'); + + normalized + .split('/') + .next_back() + .filter(|segment| !segment.is_empty()) + .map(|segment| segment.to_string()) +} + +/// Check post text for target emoji and extract blog URLs +fn find_blog_urls(description: &str, target_emoji: &str, blog_domain: &str) -> Vec { + if !description.starts_with(target_emoji) { + return vec![]; + } + + // Simple URL extraction - look for blog domain in text + description + .split_whitespace() + .filter(|word| word.contains(blog_domain)) + .map(|s| { + s.trim_matches(|c: char| { + !c.is_alphanumeric() && c != ':' && c != '/' && c != '.' && c != '-' && c != '_' + }) + }) + .filter(|s| s.starts_with("http")) + .map(|s| s.to_string()) + .collect() +} + +/// Poll RSS feed and update database +async fn poll_rss(pool: &sqlx::Pool, config: &PollerConfig) -> Result<(), String> { + log::info!("[rss] Polling RSS feed for {}", config.poster_handle); + + let channel = fetch_rss(&config.poster_handle).await?; + + let mut processed = 0; + + for item in channel.items() { + // Extract rkey from guid (contains at:// URI) + let guid = match item.guid() { + Some(g) => g.value(), + None => continue, + }; + + let rkey = match extract_rkey(guid) { + Some(r) => r, + None => { + log::warn!("[rss] Failed to extract rkey from guid: {}", guid); + continue; + } + }; + + // Get post description/content + let description = match item.description() { + Some(d) => d, + None => continue, + }; + + // Check for target emoji and blog URLs + let urls = find_blog_urls(description, &config.target_emoji, &config.blog_domain); + + if urls.is_empty() { + continue; + } + + // Extract timestamp from pub_date if available + let time_us = item + .pub_date() + .and_then(|date_str| chrono::DateTime::parse_from_rfc2822(date_str).ok()) + .map(|dt| dt.timestamp_micros().to_string()) + .unwrap_or_else(|| chrono::Utc::now().timestamp_micros().to_string()); + + // Process each blog URL found + for url in urls { + if let Some(slug) = extract_slug_from_url(&url, &config.blog_domain) { + match sqlx::query( + "INSERT INTO posts (slug, rkey, time_us) VALUES ($1, $2, $3) ON CONFLICT (slug) DO NOTHING" + ) + .bind(&slug) + .bind(&rkey) + .bind(&time_us) + .execute(pool) + .await + { + Ok(result) => { + if result.rows_affected() > 0 { + log::info!("[rss] Inserted new post: slug={}, rkey={}", slug, rkey); + processed += 1; + } + } + Err(e) => { + log::error!("[rss] Failed to insert post {}: {}", slug, e); + } + } + } + } + } + + log::info!("[rss] Poll complete, processed {} new posts", processed); + Ok(()) +} + +/// Background task that polls RSS every 15 minutes +pub async fn rss_polling_task(pool: sqlx::Pool) { + let config = match Config::figment().extract::() { + Ok(c) => c, + Err(e) => { + log::error!("[rss] Failed to load config: {}", e); + return; + } + }; + + log::info!( + "[rss] Starting RSS poller for {} (emoji: {}, domain: {})", + config.poster_handle, + config.target_emoji, + config.blog_domain + ); + + // Do initial poll immediately + if let Err(e) = poll_rss(&pool, &config).await { + log::error!("[rss] Initial poll failed: {}", e); + } + + // Poll every 15 minutes + let interval = Duration::from_secs(15 * 60); + + loop { + sleep(interval).await; + + if let Err(e) = poll_rss(&pool, &config).await { + log::error!("[rss] Poll failed: {}", e); + } + } +} -- 2.51.2