diff --git a/Cargo.lock b/Cargo.lock deleted file mode 100644 index 3f51793..0000000 --- a/Cargo.lock +++ /dev/null @@ -1,6591 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "addr2line" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler2" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" - -[[package]] -name = "ahash" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" -dependencies = [ - "cfg-if", - "getrandom 0.2.15", - "once_cell", - "version_check", - "zerocopy 0.7.35", -] - -[[package]] -name = "aho-corasick" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" -dependencies = [ - "memchr", -] - -[[package]] -name = "allocator-api2" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" - -[[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - -[[package]] -name = "anstream" -version = "0.6.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" - -[[package]] -name = "anstyle-parse" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" -dependencies = [ - "windows-sys 0.59.0", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" -dependencies = [ - "anstyle", - "once_cell", - "windows-sys 0.59.0", -] - -[[package]] -name = "anyhow" -version = "1.0.97" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" - -[[package]] -name = "arbitrary" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" - -[[package]] -name = "arc-swap" -version = "1.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" - -[[package]] -name = "arrayvec" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" - -[[package]] -name = "askama" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b79091df18a97caea757e28cd2d5fda49c6cd4bd01ddffd7ff01ace0c0ad2c28" -dependencies = [ - "askama_derive", - "askama_escape", - "humansize", - "num-traits", - "percent-encoding", - "serde", - "serde_json", -] - -[[package]] -name = "askama_derive" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19fe8d6cb13c4714962c072ea496f3392015f0989b1a2847bb4b2d9effd71d83" -dependencies = [ - "askama_parser", - "basic-toml", - "mime", - "mime_guess", - "proc-macro2", - "quote", - "serde", - "syn 2.0.103", -] - -[[package]] -name = "askama_escape" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "619743e34b5ba4e9703bba34deac3427c72507c7159f5fd030aea8cac0cfe341" - -[[package]] -name = "askama_parser" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acb1161c6b64d1c3d83108213c2a2533a342ac225aabd0bda218278c2ddb00c0" -dependencies = [ - "nom", -] - -[[package]] -name = "asn1-rs" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56624a96882bb8c26d61312ae18cb45868e5a9992ea73c58e45c3101e56a1e60" -dependencies = [ - "asn1-rs-derive", - "asn1-rs-impl", - "displaydoc", - "nom", - "num-traits", - "rusticata-macros", - "thiserror 2.0.12", - "time", -] - -[[package]] -name = "asn1-rs-derive" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", - "synstructure", -] - -[[package]] -name = "asn1-rs-impl" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "async-channel" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" -dependencies = [ - "concurrent-queue", - "event-listener-strategy", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-compression" -version = "0.4.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40f6024f3f856663b45fd0c9b6f2024034a702f453549449e0d84a305900dad4" -dependencies = [ - "flate2", - "futures-core", - "memchr", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "async-lock" -version = "3.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" -dependencies = [ - "event-listener", - "event-listener-strategy", - "pin-project-lite", -] - -[[package]] -name = "async-stream" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" -dependencies = [ - "async-stream-impl", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-stream-impl" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "async-task" -version = "4.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" - -[[package]] -name = "async-trait" -version = "0.1.88" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - -[[package]] -name = "atrium-api" -version = "0.25.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46355d3245edc7b3160b2a45fe55d09a6963ebd3eee0252feb6b72fb0eb71463" -dependencies = [ - "atrium-common", - "atrium-xrpc", - "chrono", - "http", - "ipld-core", - "langtag", - "regex", - "serde", - "serde_bytes", - "serde_json", - "thiserror 1.0.69", - "tokio", - "trait-variant", -] - -[[package]] -name = "atrium-common" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed5610654043faa396a5a15afac0ac646d76aebe45aebd7cef4f8b96b0ab7f4" -dependencies = [ - "dashmap", - "lru", - "moka", - "thiserror 1.0.69", - "tokio", - "trait-variant", - "web-time", -] - -[[package]] -name = "atrium-identity" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9e2d42bb4dbea038f4f5f45e3af2a89d61a9894a75f06aa550b74a60d2be380" -dependencies = [ - "atrium-api", - "atrium-common", - "atrium-xrpc", - "serde", - "serde_html_form", - "serde_json", - "thiserror 1.0.69", - "trait-variant", -] - -[[package]] -name = "atrium-oauth" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca22dc4eaf77fd9bf050b21192ac58cd654a437d28e000ec114ebd93a51d36f5" -dependencies = [ - "atrium-api", - "atrium-common", - "atrium-identity", - "atrium-xrpc", - "base64 0.22.1", - "chrono", - "dashmap", - "ecdsa", - "elliptic-curve", - "jose-jwa", - "jose-jwk", - "p256", - "rand 0.8.5", - "reqwest", - "serde", - "serde_html_form", - "serde_json", - "sha2", - "thiserror 1.0.69", - "tokio", - "trait-variant", -] - -[[package]] -name = "atrium-xrpc" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0216ad50ce34e9ff982e171c3659e65dedaa2ed5ac2994524debdc9a9647ffa8" -dependencies = [ - "http", - "serde", - "serde_html_form", - "serde_json", - "thiserror 1.0.69", - "trait-variant", -] - -[[package]] -name = "auto_enums" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c170965892137a3a9aeb000b4524aa3cc022a310e709d848b6e1cdce4ab4781" -dependencies = [ - "derive_utils", - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "autocfg" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" - -[[package]] -name = "aws-lc-rs" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fcc8f365936c834db5514fc45aee5b1202d677e6b40e48468aaaa8183ca8c7" -dependencies = [ - "aws-lc-sys", - "zeroize", -] - -[[package]] -name = "aws-lc-sys" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61b1d86e7705efe1be1b569bab41d4fa1e14e220b60a160f78de2db687add079" -dependencies = [ - "bindgen 0.69.5", - "cc", - "cmake", - "dunce", - "fs_extra", -] - -[[package]] -name = "axum" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "021e862c184ae977658b36c4500f7feac3221ca5da43e3f25bd04ab6c79a29b5" -dependencies = [ - "axum-core", - "bytes", - "form_urlencoded", - "futures-util", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-util", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "serde_json", - "serde_path_to_error", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tower", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "axum-core" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68464cd0412f486726fb3373129ef5d2993f90c34bc2bc1c1e9943b2f4fc7ca6" -dependencies = [ - "bytes", - "futures-core", - "http", - "http-body", - "http-body-util", - "mime", - "pin-project-lite", - "rustversion", - "sync_wrapper", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "axum-extra" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45bf463831f5131b7d3c756525b305d40f1185b688565648a92e1392ca35713d" -dependencies = [ - "axum", - "axum-core", - "bytes", - "cookie", - "futures-util", - "headers", - "http", - "http-body", - "http-body-util", - "mime", - "pin-project-lite", - "rustversion", - "serde", - "tower", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum-metrics" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5534db72cef9897272302f3eb28c43a1a3bbd20fb40b4956da8d32366d7c8c69" -dependencies = [ - "axum", - "bytes", - "http-body", - "metrics", - "pin-project-lite", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum-template" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3df50f7d669bfc3a8c348f08f536fe37e7acfbeded3cfdffd2ad3d76725fc40c" -dependencies = [ - "axum", - "handlebars", - "serde", - "thiserror 2.0.12", -] - -[[package]] -name = "backtrace" -version = "0.3.74" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "windows-targets 0.52.6", -] - -[[package]] -name = "base-x" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" - -[[package]] -name = "base16ct" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" - -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - -[[package]] -name = "base64ct" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" - -[[package]] -name = "basic-toml" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba62675e8242a4c4e806d12f11d136e626e6c8361d6b829310732241652a178a" -dependencies = [ - "serde", -] - -[[package]] -name = "bincode" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] - -[[package]] -name = "bincode" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" -dependencies = [ - "bincode_derive", - "serde", - "unty", -] - -[[package]] -name = "bincode_derive" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" -dependencies = [ - "virtue", -] - -[[package]] -name = "bindgen" -version = "0.69.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" -dependencies = [ - "bitflags", - "cexpr", - "clang-sys", - "itertools 0.12.1", - "lazy_static", - "lazycell", - "log", - "prettyplease", - "proc-macro2", - "quote", - "regex", - "rustc-hash 1.1.0", - "shlex", - "syn 2.0.103", - "which", -] - -[[package]] -name = "bindgen" -version = "0.70.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f" -dependencies = [ - "bitflags", - "cexpr", - "clang-sys", - "itertools 0.13.0", - "proc-macro2", - "quote", - "regex", - "rustc-hash 1.1.0", - "shlex", - "syn 2.0.103", -] - -[[package]] -name = "bindgen" -version = "0.71.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" -dependencies = [ - "bitflags", - "cexpr", - "clang-sys", - "itertools 0.13.0", - "proc-macro2", - "quote", - "regex", - "rustc-hash 2.1.1", - "shlex", - "syn 2.0.103", -] - -[[package]] -name = "bitflags" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" - -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - -[[package]] -name = "borrow-or-share" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eeab4423108c5d7c744f4d234de88d18d636100093ae04caf4825134b9c3a32" - -[[package]] -name = "bumpalo" -version = "3.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - -[[package]] -name = "bytes" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" - -[[package]] -name = "byteview" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6236364b88b9b6d0bc181ba374cf1ab55ba3ef97a1cb6f8cddad48a273767fb5" - -[[package]] -name = "bzip2-sys" -version = "0.1.13+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" -dependencies = [ - "cc", - "pkg-config", -] - -[[package]] -name = "camino" -version = "1.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3" -dependencies = [ - "serde", -] - -[[package]] -name = "cardinality-estimator" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ae5e12c435064f9e8ec53c5a782ca9a362702a4863fe1b6448f524ecede8fe3" -dependencies = [ - "enum_dispatch", - "serde", - "wyhash", -] - -[[package]] -name = "cardinality-estimator-safe" -version = "4.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc9887b4092040ea9a416fc3de9769ee7783e3cd5c168c941e6a8de69723b971" -dependencies = [ - "digest", - "enum_dispatch", - "serde", -] - -[[package]] -name = "cc" -version = "1.2.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "525046617d8376e3db1deffb079e91cef90a89fc3ca5c185bbf8c9ecdd15cd5c" -dependencies = [ - "jobserver", - "libc", - "shlex", -] - -[[package]] -name = "cexpr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" -dependencies = [ - "nom", -] - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "cfg_aliases" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" - -[[package]] -name = "chrono" -version = "0.4.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" -dependencies = [ - "android-tzdata", - "iana-time-zone", - "js-sys", - "num-traits", - "serde", - "wasm-bindgen", - "windows-link", -] - -[[package]] -name = "cid" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3147d8272e8fa0ccd29ce51194dd98f79ddfb8191ba9e3409884e751798acf3a" -dependencies = [ - "core2", - "multibase", - "multihash", - "serde", - "serde_bytes", - "unsigned-varint", -] - -[[package]] -name = "clang-sys" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" -dependencies = [ - "glob", - "libc", - "libloading", -] - -[[package]] -name = "clap" -version = "4.5.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be92d32e80243a54711e5d7ce823c35c41c9d929dc4ab58e1276f625841aadf9" -dependencies = [ - "clap_builder", - "clap_derive", -] - -[[package]] -name = "clap_builder" -version = "4.5.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707eab41e9622f9139419d573eca0900137718000c517d47da73045f54331c3d" -dependencies = [ - "anstream", - "anstyle", - "clap_lex", - "strsim 0.11.1", -] - -[[package]] -name = "clap_derive" -version = "4.5.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "clap_lex" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" - -[[package]] -name = "clocksource" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "129026dd5a8a9592d96916258f3a5379589e513ea5e86aeb0bd2530286e44e9e" -dependencies = [ - "libc", - "time", - "winapi", -] - -[[package]] -name = "cmake" -version = "0.1.54" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" -dependencies = [ - "cc", -] - -[[package]] -name = "cmsketch" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "553c840ee51da812c6cd621f9f7e07dfb00a49f91283a8e6380c78cba4f61aba" -dependencies = [ - "paste", -] - -[[package]] -name = "colorchoice" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" - -[[package]] -name = "compare" -version = "0.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea0095f6103c2a8b44acd6fd15960c801dafebf02e21940360833e0673f48ba7" - -[[package]] -name = "concurrent-queue" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "const-oid" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" - -[[package]] -name = "constellation" -version = "0.1.0" -dependencies = [ - "anyhow", - "askama", - "axum", - "axum-extra", - "axum-metrics", - "bincode 1.3.3", - "clap", - "ctrlc", - "flume", - "fs4 0.12.0", - "headers-accept", - "links", - "mediatype", - "metrics", - "metrics-exporter-prometheus 0.16.2", - "metrics-process", - "num-format", - "ratelimit", - "rocksdb", - "serde", - "serde_json", - "serde_with", - "tempfile", - "tinyjson", - "tokio", - "tokio-util", - "tower-http", - "tungstenite 0.26.2", - "zstd", -] - -[[package]] -name = "cookie" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" -dependencies = [ - "base64 0.22.1", - "hmac", - "percent-encoding", - "rand 0.8.5", - "sha2", - "subtle", - "time", - "version_check", -] - -[[package]] -name = "core-foundation" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - -[[package]] -name = "core2" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" -dependencies = [ - "memchr", -] - -[[package]] -name = "cpufeatures" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" -dependencies = [ - "libc", -] - -[[package]] -name = "crc32fast" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "critical-section" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" - -[[package]] -name = "crossbeam-channel" -version = "0.5.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-skiplist" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" - -[[package]] -name = "crypto-bigint" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" -dependencies = [ - "generic-array", - "rand_core 0.6.4", - "subtle", - "zeroize", -] - -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "ctrlc" -version = "3.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46f93780a459b7d656ef7f071fe699c4d3d2cb201c4b24d085b6ddc505276e73" -dependencies = [ - "nix", - "windows-sys 0.59.0", -] - -[[package]] -name = "darling" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" -dependencies = [ - "darling_core 0.14.4", - "darling_macro 0.14.4", -] - -[[package]] -name = "darling" -version = "0.20.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" -dependencies = [ - "darling_core 0.20.11", - "darling_macro 0.20.11", -] - -[[package]] -name = "darling_core" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim 0.10.0", - "syn 1.0.109", -] - -[[package]] -name = "darling_core" -version = "0.20.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim 0.11.1", - "syn 2.0.103", -] - -[[package]] -name = "darling_macro" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" -dependencies = [ - "darling_core 0.14.4", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "darling_macro" -version = "0.20.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" -dependencies = [ - "darling_core 0.20.11", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "dashmap" -version = "6.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" -dependencies = [ - "cfg-if", - "crossbeam-utils", - "hashbrown 0.14.5", - "lock_api", - "once_cell", - "parking_lot_core", -] - -[[package]] -name = "data-encoding" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "575f75dfd25738df5b91b8e43e14d44bda14637a58fae779fd2b064f8bf3e010" - -[[package]] -name = "data-encoding-macro" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f9724adfcf41f45bf652b3995837669d73c4d49a1b5ac1ff82905ac7d9b5558" -dependencies = [ - "data-encoding", - "data-encoding-macro-internal", -] - -[[package]] -name = "data-encoding-macro-internal" -version = "0.1.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18e4fdb82bd54a12e42fb58a800dcae6b9e13982238ce2296dc3570b92148e1f" -dependencies = [ - "data-encoding", - "syn 2.0.103", -] - -[[package]] -name = "debug-ignore" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffe7ed1d93f4553003e20b629abe9085e1e81b1429520f897f8f8860bc6dfc21" - -[[package]] -name = "der" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" -dependencies = [ - "const-oid", - "pem-rfc7468", - "zeroize", -] - -[[package]] -name = "der-parser" -version = "10.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" -dependencies = [ - "asn1-rs", - "displaydoc", - "nom", - "num-bigint", - "num-traits", - "rusticata-macros", -] - -[[package]] -name = "deranged" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" -dependencies = [ - "powerfmt", - "serde", -] - -[[package]] -name = "derive_builder" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" -dependencies = [ - "derive_builder_macro", -] - -[[package]] -name = "derive_builder_core" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" -dependencies = [ - "darling 0.20.11", - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "derive_builder_macro" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" -dependencies = [ - "derive_builder_core", - "syn 2.0.103", -] - -[[package]] -name = "derive_more" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "093242cf7570c207c83073cf82f79706fe7b8317e98620a47d5be7c3d8497678" -dependencies = [ - "derive_more-impl", -] - -[[package]] -name = "derive_more-impl" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", - "unicode-xid", -] - -[[package]] -name = "derive_utils" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccfae181bab5ab6c5478b2ccb69e4c68a02f8c3ec72f6616bfec9dbc599d2ee0" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", - "subtle", -] - -[[package]] -name = "dirs-next" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" -dependencies = [ - "cfg-if", - "dirs-sys-next", -] - -[[package]] -name = "dirs-sys-next" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" -dependencies = [ - "libc", - "redox_users", - "winapi", -] - -[[package]] -name = "displaydoc" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "double-ended-peekable" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57" - -[[package]] -name = "downcast-rs" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" - -[[package]] -name = "dropshot" -version = "0.16.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50e8fed669e35e757646ad10f97c4d26dd22cce3da689b307954f7000d2719d0" -dependencies = [ - "async-stream", - "async-trait", - "base64 0.22.1", - "bytes", - "camino", - "chrono", - "debug-ignore", - "dropshot_endpoint", - "form_urlencoded", - "futures", - "hostname 0.4.1", - "http", - "http-body-util", - "hyper", - "hyper-util", - "indexmap 2.9.0", - "multer", - "openapiv3", - "paste", - "percent-encoding", - "rustls 0.22.4", - "rustls-pemfile", - "schemars", - "scopeguard", - "semver", - "serde", - "serde_json", - "serde_path_to_error", - "serde_urlencoded", - "sha1", - "slog", - "slog-async", - "slog-bunyan", - "slog-json", - "slog-term", - "thiserror 2.0.12", - "tokio", - "tokio-rustls 0.25.0", - "toml", - "uuid", - "version_check", - "waitgroup", -] - -[[package]] -name = "dropshot_endpoint" -version = "0.16.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acebb687581abdeaa2c89fa448818a5f803b0e68e5d7e7a1cf585a8f3c5c57ac" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "semver", - "serde", - "serde_tokenstream", - "syn 2.0.103", -] - -[[package]] -name = "dunce" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" - -[[package]] -name = "dyn-clone" -version = "1.0.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c7a8fb8a9fbf66c1f703fe16184d10ca0ee9d23be5b4436400408ba54a95005" - -[[package]] -name = "ecdsa" -version = "0.16.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" -dependencies = [ - "der", - "digest", - "elliptic-curve", - "rfc6979", - "signature", - "spki", -] - -[[package]] -name = "either" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" - -[[package]] -name = "elliptic-curve" -version = "0.13.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" -dependencies = [ - "base16ct", - "crypto-bigint", - "digest", - "ff", - "generic-array", - "group", - "pem-rfc7468", - "pkcs8", - "rand_core 0.6.4", - "sec1", - "subtle", - "zeroize", -] - -[[package]] -name = "encoding_rs" -version = "0.8.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "enum-as-inner" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "enum_dispatch" -version = "0.3.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" -dependencies = [ - "once_cell", - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "env_filter" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" -dependencies = [ - "log", - "regex", -] - -[[package]] -name = "env_logger" -version = "0.11.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" -dependencies = [ - "anstream", - "anstyle", - "env_filter", - "jiff", - "log", -] - -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - -[[package]] -name = "errno" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" -dependencies = [ - "libc", - "windows-sys 0.59.0", -] - -[[package]] -name = "event-listener" -version = "5.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite", -] - -[[package]] -name = "event-listener-strategy" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" -dependencies = [ - "event-listener", - "pin-project-lite", -] - -[[package]] -name = "fastrand" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" - -[[package]] -name = "ff" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" -dependencies = [ - "rand_core 0.6.4", - "subtle", -] - -[[package]] -name = "fjall" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b2ced3483989a62b3533c9f99054d73b527c6c0045cf22b00fe87956f1a46f" -dependencies = [ - "byteorder", - "byteview", - "dashmap", - "log", - "lsm-tree", - "path-absolutize", - "std-semaphore", - "tempfile", - "xxhash-rust", -] - -[[package]] -name = "flate2" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" -dependencies = [ - "crc32fast", - "miniz_oxide", -] - -[[package]] -name = "fluent-uri" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1918b65d96df47d3591bed19c5cca17e3fa5d0707318e4b5ef2eae01764df7e5" -dependencies = [ - "borrow-or-share", - "ref-cast", -] - -[[package]] -name = "flume" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" -dependencies = [ - "futures-core", - "futures-sink", - "nanorand", - "spin", -] - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "foldhash" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" - -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - -[[package]] -name = "form_urlencoded" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "foyer" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b4d8e96374206ff1b4265f2e2e6e1f80bc3048957b2a1e7fdeef929d68f318f" -dependencies = [ - "equivalent", - "foyer-common", - "foyer-memory", - "foyer-storage", - "madsim-tokio", - "mixtrics", - "pin-project", - "serde", - "thiserror 2.0.12", - "tokio", - "tracing", -] - -[[package]] -name = "foyer-common" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "911b8e3f23d5fe55b0b240f75af1d2fa5cb7261d3f9b38ef1c57bbc9f0449317" -dependencies = [ - "bincode 1.3.3", - "bytes", - "cfg-if", - "itertools 0.14.0", - "madsim-tokio", - "mixtrics", - "parking_lot", - "pin-project", - "serde", - "thiserror 2.0.12", - "tokio", - "twox-hash", -] - -[[package]] -name = "foyer-intrusive-collections" -version = "0.10.0-dev" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4fee46bea69e0596130e3210e65d3424e0ac1e6df3bde6636304bdf1ca4a3b" -dependencies = [ - "memoffset", -] - -[[package]] -name = "foyer-memory" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "506883d5a8500dea1b1662f7180f3534bdcbfa718d3253db7179552ef83612fa" -dependencies = [ - "arc-swap", - "bitflags", - "cmsketch", - "equivalent", - "foyer-common", - "foyer-intrusive-collections", - "hashbrown 0.15.2", - "itertools 0.14.0", - "madsim-tokio", - "mixtrics", - "parking_lot", - "pin-project", - "serde", - "thiserror 2.0.12", - "tokio", - "tracing", -] - -[[package]] -name = "foyer-storage" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ba8403a54a2f2032fb647e49c442e5feeb33f3989f7024f1b178341a016f06d" -dependencies = [ - "allocator-api2", - "anyhow", - "auto_enums", - "bytes", - "equivalent", - "flume", - "foyer-common", - "foyer-memory", - "fs4 0.13.1", - "futures-core", - "futures-util", - "itertools 0.14.0", - "libc", - "lz4", - "madsim-tokio", - "ordered_hash_map", - "parking_lot", - "paste", - "pin-project", - "rand 0.9.1", - "serde", - "thiserror 2.0.12", - "tokio", - "tracing", - "twox-hash", - "zstd", -] - -[[package]] -name = "fs4" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c29c30684418547d476f0b48e84f4821639119c483b1eccd566c8cd0cd05f521" -dependencies = [ - "rustix 0.38.44", - "windows-sys 0.52.0", -] - -[[package]] -name = "fs4" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" -dependencies = [ - "rustix 1.0.5", - "windows-sys 0.59.0", -] - -[[package]] -name = "fs_extra" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" - -[[package]] -name = "futures" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" - -[[package]] -name = "futures-executor" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" - -[[package]] -name = "futures-macro" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "futures-sink" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" - -[[package]] -name = "futures-task" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" - -[[package]] -name = "futures-util" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "pin-utils", - "slab", -] - -[[package]] -name = "generator" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc6bd114ceda131d3b1d665eba35788690ad37f5916457286b32ab6fd3c438dd" -dependencies = [ - "cfg-if", - "libc", - "log", - "rustversion", - "windows", -] - -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", - "zeroize", -] - -[[package]] -name = "getrandom" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "wasi 0.11.0+wasi-snapshot-preview1", - "wasm-bindgen", -] - -[[package]] -name = "getrandom" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "r-efi", - "wasi 0.14.2+wasi-0.2.4", - "wasm-bindgen", -] - -[[package]] -name = "gimli" -version = "0.31.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" - -[[package]] -name = "glob" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" - -[[package]] -name = "group" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" -dependencies = [ - "ff", - "rand_core 0.6.4", - "subtle", -] - -[[package]] -name = "guardian" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17e2ac29387b1aa07a1e448f7bb4f35b500787971e965b02842b900afa5c8f6f" - -[[package]] -name = "h2" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5017294ff4bb30944501348f6f8e42e6ad28f42c8bbef7a74029aff064a4e3c2" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http", - "indexmap 2.9.0", - "slab", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "handlebars" -version = "6.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "759e2d5aea3287cb1190c8ec394f42866cb5bf74fcbf213f354e3c856ea26098" -dependencies = [ - "derive_builder", - "log", - "num-order", - "pest", - "pest_derive", - "serde", - "serde_json", - "thiserror 2.0.12", - "walkdir", -] - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - -[[package]] -name = "hashbrown" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" -dependencies = [ - "ahash", -] - -[[package]] -name = "hashbrown" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" - -[[package]] -name = "hashbrown" -version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash", -] - -[[package]] -name = "headers" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322106e6bd0cba2d5ead589ddb8150a13d7c4217cf80d7c4f682ca994ccc6aa9" -dependencies = [ - "base64 0.21.7", - "bytes", - "headers-core", - "http", - "httpdate", - "mime", - "sha1", -] - -[[package]] -name = "headers-accept" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "244c37aaa637e64fb73af1bb2a4f3f6f2b72f1f39f1cd6f4443f0cc21fe80280" -dependencies = [ - "headers-core", - "http", - "mediatype", -] - -[[package]] -name = "headers-core" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4" -dependencies = [ - "http", -] - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "hermit-abi" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbd780fe5cc30f81464441920d82ac8740e2e46b29a6fad543ddd075229ce37e" - -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - -[[package]] -name = "hickory-proto" -version = "0.25.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8a6fe56c0038198998a6f217ca4e7ef3a5e51f46163bd6dd60b5c71ca6c6502" -dependencies = [ - "async-trait", - "cfg-if", - "data-encoding", - "enum-as-inner", - "futures-channel", - "futures-io", - "futures-util", - "idna", - "ipnet", - "once_cell", - "rand 0.9.1", - "ring", - "thiserror 2.0.12", - "tinyvec", - "tokio", - "tracing", - "url", -] - -[[package]] -name = "hickory-resolver" -version = "0.25.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc62a9a99b0bfb44d2ab95a7208ac952d31060efc16241c87eaf36406fecf87a" -dependencies = [ - "cfg-if", - "futures-util", - "hickory-proto", - "ipconfig", - "moka", - "once_cell", - "parking_lot", - "rand 0.9.1", - "resolv-conf", - "smallvec", - "thiserror 2.0.12", - "tokio", - "tracing", -] - -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest", -] - -[[package]] -name = "home" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" -dependencies = [ - "windows-sys 0.59.0", -] - -[[package]] -name = "hostname" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" -dependencies = [ - "libc", - "match_cfg", - "winapi", -] - -[[package]] -name = "hostname" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56f203cd1c76362b69e3863fd987520ac36cf70a8c92627449b2f64a8cf7d65" -dependencies = [ - "cfg-if", - "libc", - "windows-link", -] - -[[package]] -name = "http" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - -[[package]] -name = "http-body" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" -dependencies = [ - "bytes", - "http", -] - -[[package]] -name = "http-body-util" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" -dependencies = [ - "bytes", - "futures-core", - "http", - "http-body", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" - -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - -[[package]] -name = "humansize" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7" -dependencies = [ - "libm", -] - -[[package]] -name = "hyper" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" -dependencies = [ - "bytes", - "futures-channel", - "futures-util", - "h2", - "http", - "http-body", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "smallvec", - "tokio", - "want", -] - -[[package]] -name = "hyper-rustls" -version = "0.27.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" -dependencies = [ - "http", - "hyper", - "hyper-util", - "rustls 0.23.31", - "rustls-native-certs", - "rustls-pki-types", - "tokio", - "tokio-rustls 0.26.2", - "tower-service", -] - -[[package]] -name = "hyper-tls" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" -dependencies = [ - "bytes", - "http-body-util", - "hyper", - "hyper-util", - "native-tls", - "tokio", - "tokio-native-tls", - "tower-service", -] - -[[package]] -name = "hyper-util" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e" -dependencies = [ - "base64 0.22.1", - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "http", - "http-body", - "hyper", - "ipnet", - "libc", - "percent-encoding", - "pin-project-lite", - "socket2 0.6.0", - "system-configuration", - "tokio", - "tower-service", - "tracing", - "windows-registry", -] - -[[package]] -name = "iana-time-zone" -version = "0.1.63" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "log", - "wasm-bindgen", - "windows-core 0.61.0", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - -[[package]] -name = "icu_collections" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_locid" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" -dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", -] - -[[package]] -name = "icu_locid_transform" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" -dependencies = [ - "displaydoc", - "icu_locid", - "icu_locid_transform_data", - "icu_provider", - "tinystr", - "zerovec", -] - -[[package]] -name = "icu_locid_transform_data" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7515e6d781098bf9f7205ab3fc7e9709d34554ae0b21ddbcb5febfa4bc7df11d" - -[[package]] -name = "icu_normalizer" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_normalizer_data", - "icu_properties", - "icu_provider", - "smallvec", - "utf16_iter", - "utf8_iter", - "write16", - "zerovec", -] - -[[package]] -name = "icu_normalizer_data" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5e8338228bdc8ab83303f16b797e177953730f601a96c25d10cb3ab0daa0cb7" - -[[package]] -name = "icu_properties" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_locid_transform", - "icu_properties_data", - "icu_provider", - "tinystr", - "zerovec", -] - -[[package]] -name = "icu_properties_data" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85fb8799753b75aee8d2a21d7c14d9f38921b54b3dbda10f5a3c7a7b82dba5e2" - -[[package]] -name = "icu_provider" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" -dependencies = [ - "displaydoc", - "icu_locid", - "icu_provider_macros", - "stable_deref_trait", - "tinystr", - "writeable", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_provider_macros" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "ident_case" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" - -[[package]] -name = "idna" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" -dependencies = [ - "idna_adapter", - "smallvec", - "utf8_iter", -] - -[[package]] -name = "idna_adapter" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" -dependencies = [ - "icu_normalizer", - "icu_properties", -] - -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", - "serde", -] - -[[package]] -name = "indexmap" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" -dependencies = [ - "equivalent", - "hashbrown 0.15.2", - "serde", -] - -[[package]] -name = "interval-heap" -version = "0.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11274e5e8e89b8607cfedc2910b6626e998779b48a019151c7604d0adcb86ac6" -dependencies = [ - "compare", -] - -[[package]] -name = "io-uring" -version = "0.7.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4" -dependencies = [ - "bitflags", - "cfg-if", - "libc", -] - -[[package]] -name = "ipconfig" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" -dependencies = [ - "socket2 0.5.9", - "widestring", - "windows-sys 0.48.0", - "winreg", -] - -[[package]] -name = "ipld-core" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "104718b1cc124d92a6d01ca9c9258a7df311405debb3408c445a36452f9bf8db" -dependencies = [ - "cid", - "serde", - "serde_bytes", -] - -[[package]] -name = "ipnet" -version = "2.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" - -[[package]] -name = "iri-string" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" -dependencies = [ - "memchr", - "serde", -] - -[[package]] -name = "is-terminal" -version = "0.4.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys 0.59.0", -] - -[[package]] -name = "is_terminal_polyfill" -version = "1.70.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" - -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - -[[package]] -name = "itertools" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" -dependencies = [ - "either", -] - -[[package]] -name = "itertools" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" - -[[package]] -name = "jetstream" -version = "0.1.1" -dependencies = [ - "anyhow", - "async-trait", - "atrium-api", - "chrono", - "clap", - "futures-util", - "log", - "metrics", - "serde", - "serde_json", - "thiserror 2.0.12", - "tokio", - "tokio-tungstenite 0.26.2", - "url", - "zstd", -] - -[[package]] -name = "jiff" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f33145a5cbea837164362c7bd596106eb7c5198f97d1ba6f6ebb3223952e488" -dependencies = [ - "jiff-static", - "log", - "portable-atomic", - "portable-atomic-util", - "serde", -] - -[[package]] -name = "jiff-static" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43ce13c40ec6956157a3635d97a1ee2df323b263f09ea14165131289cb0f5c19" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "jobserver" -version = "0.1.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" -dependencies = [ - "getrandom 0.3.3", - "libc", -] - -[[package]] -name = "jose-b64" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec69375368709666b21c76965ce67549f2d2db7605f1f8707d17c9656801b56" -dependencies = [ - "base64ct", - "serde", - "subtle", - "zeroize", -] - -[[package]] -name = "jose-jwa" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ab78e053fe886a351d67cf0d194c000f9d0dcb92906eb34d853d7e758a4b3a7" -dependencies = [ - "serde", -] - -[[package]] -name = "jose-jwk" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "280fa263807fe0782ecb6f2baadc28dffc04e00558a58e33bfdb801d11fd58e7" -dependencies = [ - "jose-b64", - "jose-jwa", - "p256", - "p384", - "rsa", - "serde", - "zeroize", -] - -[[package]] -name = "js-sys" -version = "0.3.77" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" -dependencies = [ - "once_cell", - "wasm-bindgen", -] - -[[package]] -name = "jsonwebtoken" -version = "9.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" -dependencies = [ - "base64 0.22.1", - "js-sys", - "pem", - "ring", - "serde", - "serde_json", - "simple_asn1", -] - -[[package]] -name = "langtag" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed60c85f254d6ae8450cec15eedd921efbc4d1bdf6fcf6202b9a58b403f6f805" -dependencies = [ - "serde", -] - -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -dependencies = [ - "spin", -] - -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - -[[package]] -name = "libc" -version = "0.2.174" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" - -[[package]] -name = "libfuzzer-sys" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf78f52d400cf2d84a3a973a78a592b4adc535739e0a5597a0da6f0c357adc75" -dependencies = [ - "arbitrary", - "cc", -] - -[[package]] -name = "libloading" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" -dependencies = [ - "cfg-if", - "windows-targets 0.52.6", -] - -[[package]] -name = "libm" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" - -[[package]] -name = "libproc" -version = "0.14.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e78a09b56be5adbcad5aa1197371688dc6bb249a26da3bca2011ee2fb987ebfb" -dependencies = [ - "bindgen 0.70.1", - "errno", - "libc", -] - -[[package]] -name = "libredox" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" -dependencies = [ - "bitflags", - "libc", -] - -[[package]] -name = "librocksdb-sys" -version = "0.17.1+9.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b7869a512ae9982f4d46ba482c2a304f1efd80c6412a3d4bf57bb79a619679f" -dependencies = [ - "bindgen 0.69.5", - "bzip2-sys", - "cc", - "libc", - "libz-sys", - "lz4-sys", - "zstd-sys", -] - -[[package]] -name = "libz-sys" -version = "1.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" -dependencies = [ - "cc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "links" -version = "0.1.0" -dependencies = [ - "anyhow", - "fluent-uri", - "nom", - "thiserror 2.0.12", - "tinyjson", -] - -[[package]] -name = "linux-raw-sys" -version = "0.4.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" - -[[package]] -name = "linux-raw-sys" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" - -[[package]] -name = "litemap" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" - -[[package]] -name = "lock_api" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" -dependencies = [ - "autocfg", - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" - -[[package]] -name = "loom" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" -dependencies = [ - "cfg-if", - "generator", - "scoped-tls", - "tracing", - "tracing-subscriber", -] - -[[package]] -name = "lru" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" -dependencies = [ - "hashbrown 0.15.2", -] - -[[package]] -name = "lru-slab" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" - -[[package]] -name = "lsm-tree" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a63a5e98a38b51765274137d8aedfbd848da5f4d016867e186b673fcc06a8c" -dependencies = [ - "byteorder", - "crossbeam-skiplist", - "double-ended-peekable", - "enum_dispatch", - "guardian", - "interval-heap", - "log", - "lz4_flex", - "path-absolutize", - "quick_cache", - "rustc-hash 2.1.1", - "self_cell", - "tempfile", - "value-log", - "varint-rs", - "xxhash-rust", -] - -[[package]] -name = "lz4" -version = "1.28.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" -dependencies = [ - "lz4-sys", -] - -[[package]] -name = "lz4-sys" -version = "1.11.1+lz4-1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "lz4_flex" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" - -[[package]] -name = "mach2" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" -dependencies = [ - "libc", -] - -[[package]] -name = "madsim" -version = "0.2.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db6694555643da293dfb89e33c2880a13b62711d64b6588bc7df6ce4110b27f1" -dependencies = [ - "ahash", - "async-channel", - "async-stream", - "async-task", - "bincode 1.3.3", - "bytes", - "downcast-rs", - "futures-util", - "lazy_static", - "libc", - "madsim-macros", - "naive-timer", - "panic-message", - "rand 0.8.5", - "rand_xoshiro 0.6.0", - "rustversion", - "serde", - "spin", - "tokio", - "tokio-util", - "toml", - "tracing", - "tracing-subscriber", -] - -[[package]] -name = "madsim-macros" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3d248e97b1a48826a12c3828d921e8548e714394bf17274dd0a93910dc946e1" -dependencies = [ - "darling 0.14.4", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "madsim-tokio" -version = "0.2.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d3eb2acc57c82d21d699119b859e2df70a91dbdb84734885a1e72be83bdecb5" -dependencies = [ - "madsim", - "spin", - "tokio", -] - -[[package]] -name = "match_cfg" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" - -[[package]] -name = "matchers" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" -dependencies = [ - "regex-automata 0.1.10", -] - -[[package]] -name = "matchit" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" - -[[package]] -name = "mediatype" -version = "0.19.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33746aadcb41349ec291e7f2f0a3aa6834d1d7c58066fb4b01f68efc4c4b7631" - -[[package]] -name = "memchr" -version = "2.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" - -[[package]] -name = "memoffset" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" -dependencies = [ - "autocfg", -] - -[[package]] -name = "metrics" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25dea7ac8057892855ec285c440160265225438c3c45072613c25a4b26e98ef5" -dependencies = [ - "ahash", - "portable-atomic", -] - -[[package]] -name = "metrics-exporter-prometheus" -version = "0.16.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd7399781913e5393588a8d8c6a2867bf85fb38eaf2502fdce465aad2dc6f034" -dependencies = [ - "base64 0.22.1", - "http-body-util", - "hyper", - "hyper-util", - "indexmap 2.9.0", - "ipnet", - "metrics", - "metrics-util 0.19.0", - "quanta", - "thiserror 1.0.69", - "tokio", - "tracing", -] - -[[package]] -name = "metrics-exporter-prometheus" -version = "0.17.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b166dea96003ee2531cf14833efedced545751d800f03535801d833313f8c15" -dependencies = [ - "base64 0.22.1", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-util", - "indexmap 2.9.0", - "ipnet", - "metrics", - "metrics-util 0.20.0", - "quanta", - "thiserror 2.0.12", - "tokio", - "tracing", -] - -[[package]] -name = "metrics-process" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a82c8add4382f29a122fa64fff1891453ed0f6b2867d971e7d60cb8dfa322ff" -dependencies = [ - "libc", - "libproc", - "mach2", - "metrics", - "once_cell", - "procfs", - "rlimit", - "windows", -] - -[[package]] -name = "metrics-util" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbd4884b1dd24f7d6628274a2f5ae22465c337c5ba065ec9b6edccddf8acc673" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", - "hashbrown 0.15.2", - "metrics", - "quanta", - "rand 0.8.5", - "rand_xoshiro 0.6.0", - "sketches-ddsketch", -] - -[[package]] -name = "metrics-util" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe8db7a05415d0f919ffb905afa37784f71901c9a773188876984b4f769ab986" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", - "hashbrown 0.15.2", - "metrics", - "quanta", - "rand 0.9.1", - "rand_xoshiro 0.7.0", - "sketches-ddsketch", -] - -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - -[[package]] -name = "mime_guess" -version = "2.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" -dependencies = [ - "mime", - "unicase", -] - -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - -[[package]] -name = "miniz_oxide" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" -dependencies = [ - "adler2", -] - -[[package]] -name = "mio" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" -dependencies = [ - "libc", - "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.52.0", -] - -[[package]] -name = "mixtrics" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adbcddf5a90b959eea97ae505e0391f5c6dd411fbf546d43b9c59ad1c3bd4391" -dependencies = [ - "itertools 0.14.0", - "parking_lot", -] - -[[package]] -name = "moka" -version = "0.12.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9321642ca94a4282428e6ea4af8cc2ca4eac48ac7a6a4ea8f33f76d0ce70926" -dependencies = [ - "async-lock", - "crossbeam-channel", - "crossbeam-epoch", - "crossbeam-utils", - "event-listener", - "futures-util", - "loom", - "parking_lot", - "portable-atomic", - "rustc_version", - "smallvec", - "tagptr", - "thiserror 1.0.69", - "uuid", -] - -[[package]] -name = "multer" -version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" -dependencies = [ - "bytes", - "encoding_rs", - "futures-util", - "http", - "httparse", - "memchr", - "mime", - "spin", - "tokio", - "version_check", -] - -[[package]] -name = "multibase" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b3539ec3c1f04ac9748a260728e855f261b4977f5c3406612c884564f329404" -dependencies = [ - "base-x", - "data-encoding", - "data-encoding-macro", -] - -[[package]] -name = "multihash" -version = "0.19.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b430e7953c29dd6a09afc29ff0bb69c6e306329ee6794700aee27b76a1aea8d" -dependencies = [ - "core2", - "serde", - "unsigned-varint", -] - -[[package]] -name = "naive-timer" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "034a0ad7deebf0c2abcf2435950a6666c3c15ea9d8fad0c0f48efa8a7f843fed" - -[[package]] -name = "nanorand" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" -dependencies = [ - "getrandom 0.2.15", -] - -[[package]] -name = "native-tls" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" -dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework 2.11.1", - "security-framework-sys", - "tempfile", -] - -[[package]] -name = "nix" -version = "0.30.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" -dependencies = [ - "bitflags", - "cfg-if", - "cfg_aliases", - "libc", -] - -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - -[[package]] -name = "nu-ansi-term" -version = "0.46.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" -dependencies = [ - "overload", - "winapi", -] - -[[package]] -name = "num-bigint" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" -dependencies = [ - "num-integer", - "num-traits", -] - -[[package]] -name = "num-bigint-dig" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" -dependencies = [ - "byteorder", - "lazy_static", - "libm", - "num-integer", - "num-iter", - "num-traits", - "rand 0.8.5", - "smallvec", - "zeroize", -] - -[[package]] -name = "num-conv" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" - -[[package]] -name = "num-format" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" -dependencies = [ - "arrayvec", - "itoa", -] - -[[package]] -name = "num-integer" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-iter" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-modular" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17bb261bf36fa7d83f4c294f834e91256769097b3cb505d44831e0a179ac647f" - -[[package]] -name = "num-order" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "537b596b97c40fcf8056d153049eb22f481c17ebce72a513ec9286e4986d1bb6" -dependencies = [ - "num-modular", -] - -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", - "libm", -] - -[[package]] -name = "num_threads" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" -dependencies = [ - "libc", -] - -[[package]] -name = "object" -version = "0.36.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" -dependencies = [ - "memchr", -] - -[[package]] -name = "oid-registry" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" -dependencies = [ - "asn1-rs", -] - -[[package]] -name = "once_cell" -version = "1.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" -dependencies = [ - "critical-section", - "portable-atomic", -] - -[[package]] -name = "openapiv3" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc02deea53ffe807708244e5914f6b099ad7015a207ee24317c22112e17d9c5c" -dependencies = [ - "indexmap 2.9.0", - "serde", - "serde_json", -] - -[[package]] -name = "openssl" -version = "0.10.72" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fedfea7d58a1f73118430a55da6a286e7b044961736ce96a16a17068ea25e5da" -dependencies = [ - "bitflags", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "openssl-probe" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" - -[[package]] -name = "openssl-src" -version = "300.5.0+3.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8ce546f549326b0e6052b649198487d91320875da901e7bd11a06d1ee3f9c2f" -dependencies = [ - "cc", -] - -[[package]] -name = "openssl-sys" -version = "0.9.107" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8288979acd84749c744a9014b4382d42b8f7b2592847b5afb2ed29e5d16ede07" -dependencies = [ - "cc", - "libc", - "openssl-src", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "ordered_hash_map" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab0e5f22bf6dd04abd854a8874247813a8fa2c8c1260eba6fbb150270ce7c176" -dependencies = [ - "hashbrown 0.13.2", -] - -[[package]] -name = "overload" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" - -[[package]] -name = "p256" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" -dependencies = [ - "ecdsa", - "elliptic-curve", - "primeorder", - "sha2", -] - -[[package]] -name = "p384" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" -dependencies = [ - "elliptic-curve", - "primeorder", -] - -[[package]] -name = "panic-message" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384e52fd8fbd4cbe3c317e8216260c21a0f9134de108cea8a4dd4e7e152c472d" - -[[package]] -name = "parking" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" - -[[package]] -name = "parking_lot" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-targets 0.52.6", -] - -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - -[[package]] -name = "path-absolutize" -version = "3.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4af381fe79fa195b4909485d99f73a80792331df0625188e707854f0b3383f5" -dependencies = [ - "path-dedot", -] - -[[package]] -name = "path-dedot" -version = "3.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07ba0ad7e047712414213ff67533e6dd477af0a4e1d14fb52343e53d30ea9397" -dependencies = [ - "once_cell", -] - -[[package]] -name = "pem" -version = "3.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38af38e8470ac9dee3ce1bae1af9c1671fffc44ddfd8bd1d0a3445bf349a8ef3" -dependencies = [ - "base64 0.22.1", - "serde", -] - -[[package]] -name = "pem-rfc7468" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" -dependencies = [ - "base64ct", -] - -[[package]] -name = "percent-encoding" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" - -[[package]] -name = "pest" -version = "2.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1db05f56d34358a8b1066f67cbb203ee3e7ed2ba674a6263a1d5ec6db2204323" -dependencies = [ - "memchr", - "thiserror 2.0.12", - "ucd-trie", -] - -[[package]] -name = "pest_derive" -version = "2.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb056d9e8ea77922845ec74a1c4e8fb17e7c218cc4fc11a15c5d25e189aa40bc" -dependencies = [ - "pest", - "pest_generator", -] - -[[package]] -name = "pest_generator" -version = "2.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87e404e638f781eb3202dc82db6760c8ae8a1eeef7fb3fa8264b2ef280504966" -dependencies = [ - "pest", - "pest_meta", - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "pest_meta" -version = "2.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edd1101f170f5903fde0914f899bb503d9ff5271d7ba76bbb70bea63690cc0d5" -dependencies = [ - "pest", - "sha2", -] - -[[package]] -name = "pin-project" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "pkcs1" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" -dependencies = [ - "der", - "pkcs8", - "spki", -] - -[[package]] -name = "pkcs8" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" -dependencies = [ - "der", - "spki", -] - -[[package]] -name = "pkg-config" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" - -[[package]] -name = "poem" -version = "3.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f977080932c87287147dca052951c3e2696f8759863f6b4e4c0c9ffe7a4cc8b" -dependencies = [ - "base64 0.22.1", - "bytes", - "chrono", - "futures-util", - "headers", - "http", - "http-body-util", - "httpdate", - "hyper", - "hyper-util", - "mime", - "mime_guess", - "multer", - "nix", - "parking_lot", - "percent-encoding", - "pin-project-lite", - "poem-derive", - "quick-xml", - "rcgen", - "regex", - "reqwest", - "rfc7239", - "ring", - "rustls-pemfile", - "serde", - "serde_json", - "serde_urlencoded", - "serde_yaml", - "smallvec", - "sync_wrapper", - "tempfile", - "thiserror 2.0.12", - "tokio", - "tokio-rustls 0.26.2", - "tokio-stream", - "tokio-util", - "tracing", - "wildmatch", - "x509-parser", -] - -[[package]] -name = "poem-derive" -version = "3.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "056e2fea6de1cb240ffe23cfc4fc370b629f8be83b5f27e16b7acd5231a72de4" -dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "poem-openapi" -version = "5.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ccbcc395bf4dd03df1da32da351b6b6732e4074ce27ddec315650e52a2be44c" -dependencies = [ - "base64 0.22.1", - "bytes", - "derive_more", - "futures-util", - "indexmap 2.9.0", - "itertools 0.14.0", - "mime", - "num-traits", - "poem", - "poem-openapi-derive", - "quick-xml", - "regex", - "serde", - "serde_json", - "serde_urlencoded", - "serde_yaml", - "thiserror 2.0.12", - "tokio", -] - -[[package]] -name = "poem-openapi-derive" -version = "5.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41273b691a3d467a8c44d05506afba9f7b6bd56c9cdf80123de13fe52d7ec587" -dependencies = [ - "darling 0.20.11", - "http", - "indexmap 2.9.0", - "mime", - "proc-macro-crate", - "proc-macro2", - "quote", - "regex", - "syn 2.0.103", - "thiserror 2.0.12", -] - -[[package]] -name = "portable-atomic" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" - -[[package]] -name = "portable-atomic-util" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" -dependencies = [ - "portable-atomic", -] - -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" - -[[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy 0.8.24", -] - -[[package]] -name = "prettyplease" -version = "0.2.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6837b9e10d61f45f987d50808f83d1ee3d206c66acf650c3e4ae2e1f6ddedf55" -dependencies = [ - "proc-macro2", - "syn 2.0.103", -] - -[[package]] -name = "primeorder" -version = "0.13.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" -dependencies = [ - "elliptic-curve", -] - -[[package]] -name = "proc-macro-crate" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" -dependencies = [ - "toml_edit", -] - -[[package]] -name = "proc-macro2" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "procfs" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" -dependencies = [ - "bitflags", - "hex", - "procfs-core", - "rustix 0.38.44", -] - -[[package]] -name = "procfs-core" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec" -dependencies = [ - "bitflags", - "hex", -] - -[[package]] -name = "quanta" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bd1fe6824cea6538803de3ff1bc0cf3949024db3d43c9643024bfb33a807c0e" -dependencies = [ - "crossbeam-utils", - "libc", - "once_cell", - "raw-cpuid", - "wasi 0.11.0+wasi-snapshot-preview1", - "web-sys", - "winapi", -] - -[[package]] -name = "quick-xml" -version = "0.36.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" -dependencies = [ - "memchr", - "serde", -] - -[[package]] -name = "quick_cache" -version = "0.6.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f8ed0655cbaf18a26966142ad23b95d8ab47221c50c4f73a1db7d0d2d6e3da8" -dependencies = [ - "equivalent", - "hashbrown 0.15.2", -] - -[[package]] -name = "quinn" -version = "0.11.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626214629cda6781b6dc1d316ba307189c85ba657213ce642d9c77670f8202c8" -dependencies = [ - "bytes", - "cfg_aliases", - "pin-project-lite", - "quinn-proto", - "quinn-udp", - "rustc-hash 2.1.1", - "rustls 0.23.31", - "socket2 0.5.9", - "thiserror 2.0.12", - "tokio", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-proto" -version = "0.11.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49df843a9161c85bb8aae55f101bc0bac8bcafd637a620d9122fd7e0b2f7422e" -dependencies = [ - "bytes", - "getrandom 0.3.3", - "lru-slab", - "rand 0.9.1", - "ring", - "rustc-hash 2.1.1", - "rustls 0.23.31", - "rustls-pki-types", - "slab", - "thiserror 2.0.12", - "tinyvec", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-udp" -version = "0.5.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcebb1209ee276352ef14ff8732e24cc2b02bbac986cd74a4c81bcb2f9881970" -dependencies = [ - "cfg_aliases", - "libc", - "once_cell", - "socket2 0.5.9", - "tracing", - "windows-sys 0.59.0", -] - -[[package]] -name = "quote" -version = "1.0.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "5.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" - -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - -[[package]] -name = "rand" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" -dependencies = [ - "rand_chacha 0.9.0", - "rand_core 0.9.3", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core 0.9.3", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.15", -] - -[[package]] -name = "rand_core" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" -dependencies = [ - "getrandom 0.3.3", -] - -[[package]] -name = "rand_xoshiro" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" -dependencies = [ - "rand_core 0.6.4", -] - -[[package]] -name = "rand_xoshiro" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" -dependencies = [ - "rand_core 0.9.3", -] - -[[package]] -name = "ratelimit" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36ea961700fd7260e7fa3701c8287d901b2172c51f9c1421fa0f21d7f7e184b7" -dependencies = [ - "clocksource", - "parking_lot", - "thiserror 1.0.69", -] - -[[package]] -name = "raw-cpuid" -version = "11.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6df7ab838ed27997ba19a4664507e6f82b41fe6e20be42929332156e5e85146" -dependencies = [ - "bitflags", -] - -[[package]] -name = "rcgen" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48406db8ac1f3cbc7dcdb56ec355343817958a356ff430259bb07baf7607e1e1" -dependencies = [ - "pem", - "ring", - "time", - "yasna", -] - -[[package]] -name = "redox_syscall" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2f103c6d277498fbceb16e84d317e2a400f160f46904d5f5410848c829511a3" -dependencies = [ - "bitflags", -] - -[[package]] -name = "redox_users" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" -dependencies = [ - "getrandom 0.2.15", - "libredox", - "thiserror 1.0.69", -] - -[[package]] -name = "ref-cast" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a0ae411dbe946a674d89546582cea4ba2bb8defac896622d6496f14c23ba5cf" -dependencies = [ - "ref-cast-impl", -] - -[[package]] -name = "ref-cast-impl" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "regex" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata 0.4.9", - "regex-syntax 0.8.5", -] - -[[package]] -name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" -dependencies = [ - "regex-syntax 0.6.29", -] - -[[package]] -name = "regex-automata" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax 0.8.5", -] - -[[package]] -name = "regex-syntax" -version = "0.6.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" - -[[package]] -name = "regex-syntax" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" - -[[package]] -name = "reqwest" -version = "0.12.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc931937e6ca3a06e3b6c0aa7841849b160a90351d6ab467a8b9b9959767531" -dependencies = [ - "async-compression", - "base64 0.22.1", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-tls", - "hyper-util", - "js-sys", - "log", - "mime", - "native-tls", - "percent-encoding", - "pin-project-lite", - "quinn", - "rustls 0.23.31", - "rustls-native-certs", - "rustls-pki-types", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tokio-native-tls", - "tokio-rustls 0.26.2", - "tokio-util", - "tower", - "tower-http", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - -[[package]] -name = "resolv-conf" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95325155c684b1c89f7765e30bc1c42e4a6da51ca513615660cb8a62ef9a88e3" - -[[package]] -name = "rfc6979" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" -dependencies = [ - "hmac", - "subtle", -] - -[[package]] -name = "rfc7239" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a82f1d1e38e9a85bb58ffcfadf22ed6f2c94e8cd8581ec2b0f80a2a6858350f" -dependencies = [ - "uncased", -] - -[[package]] -name = "ring" -version = "0.17.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" -dependencies = [ - "cc", - "cfg-if", - "getrandom 0.2.15", - "libc", - "untrusted", - "windows-sys 0.52.0", -] - -[[package]] -name = "rlimit" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7043b63bd0cd1aaa628e476b80e6d4023a3b50eb32789f2728908107bd0c793a" -dependencies = [ - "libc", -] - -[[package]] -name = "rocksdb" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26ec73b20525cb235bad420f911473b69f9fe27cc856c5461bccd7e4af037f43" -dependencies = [ - "libc", - "librocksdb-sys", -] - -[[package]] -name = "rsa" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78928ac1ed176a5ca1d17e578a1825f3d81ca54cf41053a592584b020cfd691b" -dependencies = [ - "const-oid", - "digest", - "num-bigint-dig", - "num-integer", - "num-traits", - "pkcs1", - "pkcs8", - "rand_core 0.6.4", - "signature", - "spki", - "subtle", - "zeroize", -] - -[[package]] -name = "rustc-demangle" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" - -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - -[[package]] -name = "rustc-hash" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" - -[[package]] -name = "rustc_version" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" -dependencies = [ - "semver", -] - -[[package]] -name = "rusticata-macros" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" -dependencies = [ - "nom", -] - -[[package]] -name = "rustix" -version = "0.38.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", -] - -[[package]] -name = "rustix" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys 0.9.4", - "windows-sys 0.59.0", -] - -[[package]] -name = "rustls" -version = "0.22.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" -dependencies = [ - "log", - "ring", - "rustls-pki-types", - "rustls-webpki 0.102.8", - "subtle", - "zeroize", -] - -[[package]] -name = "rustls" -version = "0.23.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0ebcbd2f03de0fc1122ad9bb24b127a5a6cd51d72604a3f3c50ac459762b6cc" -dependencies = [ - "aws-lc-rs", - "log", - "once_cell", - "ring", - "rustls-pki-types", - "rustls-webpki 0.103.4", - "subtle", - "zeroize", -] - -[[package]] -name = "rustls-native-certs" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" -dependencies = [ - "openssl-probe", - "rustls-pki-types", - "schannel", - "security-framework 3.2.0", -] - -[[package]] -name = "rustls-pemfile" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" -dependencies = [ - "rustls-pki-types", -] - -[[package]] -name = "rustls-pki-types" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" -dependencies = [ - "web-time", - "zeroize", -] - -[[package]] -name = "rustls-webpki" -version = "0.102.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" -dependencies = [ - "ring", - "rustls-pki-types", - "untrusted", -] - -[[package]] -name = "rustls-webpki" -version = "0.103.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a17884ae0c1b773f1ccd2bd4a8c72f16da897310a98b0e84bf349ad5ead92fc" -dependencies = [ - "aws-lc-rs", - "ring", - "rustls-pki-types", - "untrusted", -] - -[[package]] -name = "rustversion" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" - -[[package]] -name = "ryu" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" - -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - -[[package]] -name = "schannel" -version = "0.1.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" -dependencies = [ - "windows-sys 0.59.0", -] - -[[package]] -name = "schemars" -version = "0.8.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615" -dependencies = [ - "chrono", - "dyn-clone", - "schemars_derive", - "serde", - "serde_json", - "uuid", -] - -[[package]] -name = "schemars_derive" -version = "0.8.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d" -dependencies = [ - "proc-macro2", - "quote", - "serde_derive_internals", - "syn 2.0.103", -] - -[[package]] -name = "scoped-tls" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" - -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "sec1" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" -dependencies = [ - "base16ct", - "der", - "generic-array", - "pkcs8", - "subtle", - "zeroize", -] - -[[package]] -name = "security-framework" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" -dependencies = [ - "bitflags", - "core-foundation 0.9.4", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" -dependencies = [ - "bitflags", - "core-foundation 0.10.1", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "self_cell" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2fdfc24bc566f839a2da4c4295b82db7d25a24253867d5c64355abb5799bdbe" - -[[package]] -name = "semver" -version = "1.0.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" - -[[package]] -name = "serde" -version = "1.0.219" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_bytes" -version = "0.11.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8437fd221bde2d4ca316d61b90e337e9e702b3820b87d63caa9ba6c02bd06d96" -dependencies = [ - "serde", -] - -[[package]] -name = "serde_derive" -version = "1.0.219" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "serde_derive_internals" -version = "0.29.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "serde_html_form" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d2de91cf02bbc07cde38891769ccd5d4f073d22a40683aa4bc7a95781aaa2c4" -dependencies = [ - "form_urlencoded", - "indexmap 2.9.0", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "serde_json" -version = "1.0.141" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b9eff21ebe718216c6ec64e1d9ac57087aad11efc64e32002bce4a0d4c03d3" -dependencies = [ - "itoa", - "memchr", - "ryu", - "serde", -] - -[[package]] -name = "serde_path_to_error" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59fab13f937fa393d08645bf3a84bdfe86e296747b506ada67bb15f10f218b2a" -dependencies = [ - "itoa", - "serde", -] - -[[package]] -name = "serde_qs" -version = "1.0.0-rc.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cb0b9062a400c31442e67d1f2b1e7746bebd691110ebee1b7d0c7293b04fab1" -dependencies = [ - "itoa", - "percent-encoding", - "ryu", - "serde", - "thiserror 2.0.12", -] - -[[package]] -name = "serde_spanned" -version = "0.6.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" -dependencies = [ - "serde", -] - -[[package]] -name = "serde_tokenstream" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64060d864397305347a78851c51588fd283767e7e7589829e8121d65512340f1" -dependencies = [ - "proc-macro2", - "quote", - "serde", - "syn 2.0.103", -] - -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "serde_with" -version = "3.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa" -dependencies = [ - "base64 0.22.1", - "chrono", - "hex", - "indexmap 1.9.3", - "indexmap 2.9.0", - "serde", - "serde_derive", - "serde_json", - "serde_with_macros", - "time", -] - -[[package]] -name = "serde_with_macros" -version = "3.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e" -dependencies = [ - "darling 0.20.11", - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "serde_yaml" -version = "0.9.34+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" -dependencies = [ - "indexmap 2.9.0", - "itoa", - "ryu", - "serde", - "unsafe-libyaml", -] - -[[package]] -name = "sha1" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "sharded-slab" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" -dependencies = [ - "lazy_static", -] - -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "signal-hook-registry" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" -dependencies = [ - "libc", -] - -[[package]] -name = "signature" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" -dependencies = [ - "digest", - "rand_core 0.6.4", -] - -[[package]] -name = "simple_asn1" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb" -dependencies = [ - "num-bigint", - "num-traits", - "thiserror 2.0.12", - "time", -] - -[[package]] -name = "sketches-ddsketch" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1e9a774a6c28142ac54bb25d25562e6bcf957493a184f15ad4eebccb23e410a" - -[[package]] -name = "slab" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] - -[[package]] -name = "slingshot" -version = "0.1.0" -dependencies = [ - "atrium-api", - "atrium-common", - "atrium-identity", - "atrium-oauth", - "clap", - "ctrlc", - "foyer", - "hickory-resolver", - "jetstream", - "links", - "log", - "metrics", - "metrics-exporter-prometheus 0.17.2", - "poem", - "poem-openapi", - "reqwest", - "rustls 0.23.31", - "serde", - "serde_json", - "thiserror 2.0.12", - "time", - "tokio", - "tokio-util", - "tracing-subscriber", - "url", -] - -[[package]] -name = "slog" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" - -[[package]] -name = "slog-async" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72c8038f898a2c79507940990f05386455b3a317d8f18d4caea7cbc3d5096b84" -dependencies = [ - "crossbeam-channel", - "slog", - "take_mut", - "thread_local", -] - -[[package]] -name = "slog-bunyan" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcaaf6e68789d3f0411f1e72bc443214ef252a1038b6e344836e50442541f190" -dependencies = [ - "hostname 0.3.1", - "slog", - "slog-json", - "time", -] - -[[package]] -name = "slog-json" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e1e53f61af1e3c8b852eef0a9dee29008f55d6dd63794f3f12cef786cf0f219" -dependencies = [ - "serde", - "serde_json", - "slog", - "time", -] - -[[package]] -name = "slog-term" -version = "2.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6e022d0b998abfe5c3782c1f03551a596269450ccd677ea51c56f8b214610e8" -dependencies = [ - "is-terminal", - "slog", - "term", - "thread_local", - "time", -] - -[[package]] -name = "smallvec" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" - -[[package]] -name = "socket2" -version = "0.5.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - -[[package]] -name = "socket2" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" -dependencies = [ - "libc", - "windows-sys 0.59.0", -] - -[[package]] -name = "spacedust" -version = "0.1.0" -dependencies = [ - "async-trait", - "clap", - "ctrlc", - "dropshot", - "env_logger", - "futures", - "http", - "jetstream", - "links", - "log", - "metrics", - "metrics-exporter-prometheus 0.17.2", - "rand 0.9.1", - "schemars", - "semver", - "serde", - "serde_json", - "serde_qs", - "thiserror 2.0.12", - "tinyjson", - "tokio", - "tokio-tungstenite 0.27.0", - "tokio-util", -] - -[[package]] -name = "spin" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" -dependencies = [ - "lock_api", -] - -[[package]] -name = "spki" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" -dependencies = [ - "base64ct", - "der", -] - -[[package]] -name = "stable_deref_trait" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" - -[[package]] -name = "std-semaphore" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ae9eec00137a8eed469fb4148acd9fc6ac8c3f9b110f52cd34698c8b5bfa0e" - -[[package]] -name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - -[[package]] -name = "subtle" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" - -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "2.0.103" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4307e30089d6fd6aff212f2da3a1f9e32f3223b1f010fb09b7c95f90f3ca1e8" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "sync_wrapper" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" -dependencies = [ - "futures-core", -] - -[[package]] -name = "synstructure" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "system-configuration" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" -dependencies = [ - "bitflags", - "core-foundation 0.9.4", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "tagptr" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" - -[[package]] -name = "take_mut" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" - -[[package]] -name = "tempfile" -version = "3.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf" -dependencies = [ - "fastrand", - "getrandom 0.3.3", - "once_cell", - "rustix 1.0.5", - "windows-sys 0.59.0", -] - -[[package]] -name = "term" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" -dependencies = [ - "dirs-next", - "rustversion", - "winapi", -] - -[[package]] -name = "thiserror" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" -dependencies = [ - "thiserror-impl 1.0.69", -] - -[[package]] -name = "thiserror" -version = "2.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" -dependencies = [ - "thiserror-impl 2.0.12", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "thread_local" -version = "1.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" -dependencies = [ - "cfg-if", - "once_cell", -] - -[[package]] -name = "tikv-jemalloc-sys" -version = "0.6.0+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd3c60906412afa9c2b5b5a48ca6a5abe5736aec9eb48ad05037a677e52e4e2d" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "tikv-jemallocator" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cec5ff18518d81584f477e9bfdf957f5bb0979b0bac3af4ca30b5b3ae2d2865" -dependencies = [ - "libc", - "tikv-jemalloc-sys", -] - -[[package]] -name = "time" -version = "0.3.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" -dependencies = [ - "deranged", - "itoa", - "libc", - "num-conv", - "num_threads", - "powerfmt", - "serde", - "time-core", - "time-macros", -] - -[[package]] -name = "time-core" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" - -[[package]] -name = "time-macros" -version = "0.2.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" -dependencies = [ - "num-conv", - "time-core", -] - -[[package]] -name = "tinyjson" -version = "2.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ab95735ea2c8fd51154d01e39cf13912a78071c2d89abc49a7ef102a7dd725a" - -[[package]] -name = "tinystr" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" -dependencies = [ - "displaydoc", - "zerovec", -] - -[[package]] -name = "tinyvec" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "tokio" -version = "1.47.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43864ed400b6043a4757a25c7a64a8efde741aed79a056a2fb348a406701bb35" -dependencies = [ - "backtrace", - "bytes", - "io-uring", - "libc", - "mio", - "parking_lot", - "pin-project-lite", - "signal-hook-registry", - "slab", - "socket2 0.6.0", - "tokio-macros", - "windows-sys 0.59.0", -] - -[[package]] -name = "tokio-macros" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - -[[package]] -name = "tokio-rustls" -version = "0.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" -dependencies = [ - "rustls 0.22.4", - "rustls-pki-types", - "tokio", -] - -[[package]] -name = "tokio-rustls" -version = "0.26.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" -dependencies = [ - "rustls 0.23.31", - "tokio", -] - -[[package]] -name = "tokio-stream" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-tungstenite" -version = "0.26.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a9daff607c6d2bf6c16fd681ccb7eecc83e4e2cdc1ca067ffaadfca5de7f084" -dependencies = [ - "futures-util", - "log", - "native-tls", - "tokio", - "tokio-native-tls", - "tungstenite 0.26.2", -] - -[[package]] -name = "tokio-tungstenite" -version = "0.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "489a59b6730eda1b0171fcfda8b121f4bee2b35cba8645ca35c5f7ba3eb736c1" -dependencies = [ - "futures-util", - "log", - "tokio", - "tungstenite 0.27.0", -] - -[[package]] -name = "tokio-util" -version = "0.7.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "toml" -version = "0.8.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit", -] - -[[package]] -name = "toml_datetime" -version = "0.6.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" -dependencies = [ - "serde", -] - -[[package]] -name = "toml_edit" -version = "0.22.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" -dependencies = [ - "indexmap 2.9.0", - "serde", - "serde_spanned", - "toml_datetime", - "toml_write", - "winnow", -] - -[[package]] -name = "toml_write" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" - -[[package]] -name = "tower" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" -dependencies = [ - "futures-core", - "futures-util", - "pin-project-lite", - "sync_wrapper", - "tokio", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-http" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" -dependencies = [ - "bitflags", - "bytes", - "futures-util", - "http", - "http-body", - "iri-string", - "pin-project-lite", - "tower", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - -[[package]] -name = "tower-service" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" - -[[package]] -name = "tracing" -version = "0.1.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" -dependencies = [ - "log", - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "tracing-core" -version = "0.1.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" -dependencies = [ - "once_cell", - "valuable", -] - -[[package]] -name = "tracing-log" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" -dependencies = [ - "log", - "once_cell", - "tracing-core", -] - -[[package]] -name = "tracing-subscriber" -version = "0.3.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" -dependencies = [ - "matchers", - "nu-ansi-term", - "once_cell", - "regex", - "sharded-slab", - "smallvec", - "thread_local", - "tracing", - "tracing-core", - "tracing-log", -] - -[[package]] -name = "trait-variant" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - -[[package]] -name = "tungstenite" -version = "0.26.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13" -dependencies = [ - "bytes", - "data-encoding", - "http", - "httparse", - "log", - "native-tls", - "rand 0.9.1", - "sha1", - "thiserror 2.0.12", - "url", - "utf-8", -] - -[[package]] -name = "tungstenite" -version = "0.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eadc29d668c91fcc564941132e17b28a7ceb2f3ebf0b9dae3e03fd7a6748eb0d" -dependencies = [ - "bytes", - "data-encoding", - "http", - "httparse", - "log", - "rand 0.9.1", - "sha1", - "thiserror 2.0.12", - "utf-8", -] - -[[package]] -name = "twox-hash" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b907da542cbced5261bd3256de1b3a1bf340a3d37f93425a07362a1d687de56" -dependencies = [ - "rand 0.9.1", -] - -[[package]] -name = "typenum" -version = "1.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" - -[[package]] -name = "ucd-trie" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" - -[[package]] -name = "ufos" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "base64 0.22.1", - "bincode 2.0.1", - "cardinality-estimator-safe", - "chrono", - "clap", - "dropshot", - "env_logger", - "fjall", - "getrandom 0.3.3", - "http", - "jetstream", - "log", - "lsm-tree", - "metrics", - "metrics-exporter-prometheus 0.17.2", - "schemars", - "semver", - "serde", - "serde_json", - "serde_qs", - "sha2", - "tempfile", - "thiserror 2.0.12", - "tikv-jemallocator", - "tokio", - "tokio-util", -] - -[[package]] -name = "ufos-fuzz" -version = "0.0.0" -dependencies = [ - "bincode 2.0.1", - "cardinality-estimator", - "jetstream", - "libfuzzer-sys", - "tikv-jemallocator", - "ufos", -] - -[[package]] -name = "uncased" -version = "0.9.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" -dependencies = [ - "version_check", -] - -[[package]] -name = "unicase" -version = "2.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" - -[[package]] -name = "unicode-ident" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" - -[[package]] -name = "unicode-xid" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" - -[[package]] -name = "unsafe-libyaml" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" - -[[package]] -name = "unsigned-varint" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" - -[[package]] -name = "untrusted" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" - -[[package]] -name = "unty" -version = "0.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" - -[[package]] -name = "url" -version = "2.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", -] - -[[package]] -name = "utf-8" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" - -[[package]] -name = "utf16_iter" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" - -[[package]] -name = "utf8_iter" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" - -[[package]] -name = "utf8parse" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" - -[[package]] -name = "uuid" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" -dependencies = [ - "getrandom 0.3.3", - "serde", -] - -[[package]] -name = "valuable" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" - -[[package]] -name = "value-log" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd29b17c041f94e0885179637289815cd038f0c9fc19c4549d5a97017404fb7d" -dependencies = [ - "byteorder", - "byteview", - "interval-heap", - "log", - "path-absolutize", - "rustc-hash 2.1.1", - "tempfile", - "varint-rs", - "xxhash-rust", -] - -[[package]] -name = "varint-rs" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23" - -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "virtue" -version = "0.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" - -[[package]] -name = "waitgroup" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1f50000a783467e6c0200f9d10642f4bc424e39efc1b770203e88b488f79292" -dependencies = [ - "atomic-waker", -] - -[[package]] -name = "walkdir" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" -dependencies = [ - "same-file", - "winapi-util", -] - -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - -[[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" - -[[package]] -name = "wasi" -version = "0.14.2+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" -dependencies = [ - "wit-bindgen-rt", -] - -[[package]] -name = "wasm-bindgen" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" -dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" -dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn 2.0.103", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.50" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" -dependencies = [ - "cfg-if", - "js-sys", - "once_cell", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "web-sys" -version = "0.3.77" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "web-time" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "which" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" -dependencies = [ - "either", - "home", - "once_cell", - "rustix 0.38.44", -] - -[[package]] -name = "who-am-i" -version = "0.1.0" -dependencies = [ - "atrium-api", - "atrium-common", - "atrium-identity", - "atrium-oauth", - "axum", - "axum-extra", - "axum-template", - "clap", - "ctrlc", - "dashmap", - "elliptic-curve", - "handlebars", - "hickory-resolver", - "jose-jwk", - "jsonwebtoken", - "metrics", - "metrics-exporter-prometheus 0.17.2", - "p256", - "pkcs8", - "rand 0.9.1", - "reqwest", - "serde", - "serde_json", - "thiserror 2.0.12", - "tokio", - "tokio-util", - "url", -] - -[[package]] -name = "widestring" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd7cf3379ca1aac9eea11fba24fd7e315d621f8dfe35c8d7d2be8b793726e07d" - -[[package]] -name = "wildmatch" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ce1ab1f8c62655ebe1350f589c61e505cf94d385bc6a12899442d9081e71fd" - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-util" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" -dependencies = [ - "windows-sys 0.59.0", -] - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows" -version = "0.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6" -dependencies = [ - "windows-core 0.58.0", - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-core" -version = "0.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99" -dependencies = [ - "windows-implement 0.58.0", - "windows-interface 0.58.0", - "windows-result 0.2.0", - "windows-strings 0.1.0", - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-core" -version = "0.61.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980" -dependencies = [ - "windows-implement 0.60.0", - "windows-interface 0.59.1", - "windows-link", - "windows-result 0.3.4", - "windows-strings 0.4.2", -] - -[[package]] -name = "windows-implement" -version = "0.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "windows-implement" -version = "0.60.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "windows-interface" -version = "0.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "windows-interface" -version = "0.59.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "windows-link" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" - -[[package]] -name = "windows-registry" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3bab093bdd303a1240bb99b8aba8ea8a69ee19d34c9e2ef9594e708a4878820" -dependencies = [ - "windows-link", - "windows-result 0.3.4", - "windows-strings 0.4.2", -] - -[[package]] -name = "windows-result" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-result" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-strings" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" -dependencies = [ - "windows-result 0.2.0", - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-strings" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.5", -] - -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", -] - -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - -[[package]] -name = "winnow" -version = "0.7.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74c7b26e3480b707944fc872477815d29a8e429d2f93a1ce000f5fa84a15cbcd" -dependencies = [ - "memchr", -] - -[[package]] -name = "winreg" -version = "0.50.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - -[[package]] -name = "wit-bindgen-rt" -version = "0.39.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" -dependencies = [ - "bitflags", -] - -[[package]] -name = "write16" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" - -[[package]] -name = "writeable" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" - -[[package]] -name = "wyhash" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf6e163c25e3fac820b4b453185ea2dea3b6a3e0a721d4d23d75bd33734c295" -dependencies = [ - "rand_core 0.6.4", -] - -[[package]] -name = "x509-parser" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4569f339c0c402346d4a75a9e39cf8dad310e287eef1ff56d4c68e5067f53460" -dependencies = [ - "asn1-rs", - "data-encoding", - "der-parser", - "lazy_static", - "nom", - "oid-registry", - "rusticata-macros", - "thiserror 2.0.12", - "time", -] - -[[package]] -name = "xxhash-rust" -version = "0.8.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" - -[[package]] -name = "yasna" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" -dependencies = [ - "time", -] - -[[package]] -name = "yoke" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" -dependencies = [ - "serde", - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", - "synstructure", -] - -[[package]] -name = "zerocopy" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" -dependencies = [ - "zerocopy-derive 0.7.35", -] - -[[package]] -name = "zerocopy" -version = "0.8.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" -dependencies = [ - "zerocopy-derive 0.8.24", -] - -[[package]] -name = "zerocopy-derive" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "zerofrom" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", - "synstructure", -] - -[[package]] -name = "zeroize" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" -dependencies = [ - "serde", -] - -[[package]] -name = "zerovec" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" -dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", -] - -[[package]] -name = "zstd" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "7.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" -dependencies = [ - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.15+zstd.1.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" -dependencies = [ - "bindgen 0.71.1", - "cc", - "pkg-config", -] diff --git a/Cargo.toml b/Cargo.toml deleted file mode 100644 index 6604a84..0000000 --- a/Cargo.toml +++ /dev/null @@ -1,12 +0,0 @@ -[workspace] -resolver = "2" -members = [ - "links", - "constellation", - "jetstream", - "ufos", - "ufos/fuzz", - "spacedust", - "who-am-i", - "slingshot", -] diff --git a/Makefile b/Makefile deleted file mode 100644 index eb29af2..0000000 --- a/Makefile +++ /dev/null @@ -1,14 +0,0 @@ -.PHONY: check test fmt clippy -all: check - -test: - cargo test --all-features - -fmt: - cargo fmt --package links --package constellation --package ufos --package spacedust --package who-am-i --package slingshot - cargo +nightly fmt --package jetstream - -clippy: - cargo clippy --all-targets --all-features -- -D warnings - -check: test fmt clippy diff --git a/constellation/Cargo.toml b/constellation/Cargo.toml deleted file mode 100644 index 82b82b9..0000000 --- a/constellation/Cargo.toml +++ /dev/null @@ -1,48 +0,0 @@ -[package] -name = "constellation" -version = "0.1.0" -edition = "2021" -default-run = "main" - -[dependencies] -anyhow = "1.0.95" -askama = { version = "0.12.1", features = ["serde-json"] } -axum = "0.8.1" -axum-extra = { version = "0.10.0", features = ["typed-header"] } -axum-metrics = "0.2" -bincode = "1.3.3" -clap = { version = "4.5.26", features = ["derive"] } -ctrlc = "3.4.5" -flume = { version = "0.11.1", default-features = false } -fs4 = { version = "0.12.0", features = ["sync"] } -headers-accept = "0.1.4" -links = { path = "../links" } -mediatype = "0.19.18" -metrics = "0.24.1" -metrics-exporter-prometheus = { version = "0.16.1", default-features = false, features = ["http-listener"] } -metrics-process = "2.4.0" -num-format = "0.4.4" -ratelimit = "0.10.0" -rocksdb = { version = "0.23.0", optional = true } -serde = { version = "1.0.215", features = ["derive"] } -serde_json = "1.0.139" -serde_with = { version = "3.12.0", features = ["hex"] } -tinyjson = "2.5.1" -tokio-util = "0.7.13" -tower-http = { version = "0.6.2", features = ["cors"] } -zstd = "0.13.2" - -[dependencies.tokio] -version = "1.44" -features = ["full"] - -[dependencies.tungstenite] -version = "0.26.1" -features = ["native-tls"] - -[dev-dependencies] -tempfile = "3.15.0" - -[features] -default = ["rocks"] -rocks = ["dep:rocksdb"] diff --git a/constellation/readme.md b/constellation/readme.md deleted file mode 100644 index 6bb292d..0000000 --- a/constellation/readme.md +++ /dev/null @@ -1,181 +0,0 @@ -# constellation 🌌 - -A global atproto backlink index ✨ - -- Self hostable: handles the full write throughput of the global atproto firehose on a raspberry pi 4b + single SSD -- Storage efficient: less than 2GB/day disk consumption indexing all references in all lexicons and all non-atproto URLs -- Handles record deletion, account de/re-activation, and account deletion, ensuring accurate link counts and respecting users data choices -- Simple JSON API - -All social interactions in atproto tend to be represented by links (or references) between PDS records. This index can answer questions like "how many likes does a bsky post have", "who follows an account", "what are all the comments on a [frontpage](https://frontpage.fyi/) post", and more. - -- **status**: works! api is unstable and likely to change, and no known instances have a full network backfill yet. -- source: [./constellation/](./constellation/) -- public instance: [constellation.microcosm.blue](https://constellation.microcosm.blue/) - -_note: the public instance currently runs on a little raspberry pi in my house, feel free to use it! it comes with only with best-effort uptime, no commitment to not breaking the api for now, and possible rate-limiting. if you want to be nice you can put your project name and bsky username (or email) in your user-agent header for api requests._ - - -## API endpoints - -currently this is a bit out of date -- refer to the [api docs hosted by the app itself](https://constellation.microcosm.blue/) for now. they also let you try out live requests. - -terms as used here: - -- "URI": a URI, AT-URI, or DID. -- "JSON path": a dot-separated (and dot-prefixed, for now) path to a field in an atproto record. Arrays are noted by `[]` and cannot contain a specific index. - -### `GET /links/count` - -The number of backlinks to a URI from a specified collection + json path. - -#### Required URL parameters - -- `target` (required): the URI. must be URL-encoded. - - example: `at%3A%2F%2Fdid%3Aplc%3A57vlzz2egy6eqr4nksacmbht%2Fapp.bsky.feed.post%2F3lg2pgq3gq22b` -- `collection` (required): the source NSID of referring documents to consider. - - example: `app.bsky.feed.post` -- `path` (required): the JSON path in referring documents to consider. - - example: `.subject.uri` - -#### Response - -A number (u64) in plain text format - -#### cURL example: Get a count of all bluesky likes for a post - -```bash -curl '/links/count?target=at%3A%2F%2Fdid%3Aplc%3A57vlzz2egy6eqr4nksacmbht%2Fapp.bsky.feed.post%2F3lg2pgq3gq22b&collection=app.bsky.feed.like&path=.subject.uri' - -40 -``` - -### `GET /links/all/count` - -The number of backlinks to a URI from any source collection or json path - -#### Required URL parameters - -- `target` (required): the URI. must be URL-encoded. - - example: `did:plc:vc7f4oafdgxsihk4cry2xpze` - -#### Response - -A JSON object `{[NSID]: {[JSON path]: [N]}}` - -#### cURL example: Get reference counts to a DID from any collection at any path - -```bash -curl '/links/all/count?target=did:plc:vc7f4oafdgxsihk4cry2xpze' - -curl '/links/all/count?target=did:plc:vc7f4oafdgxsihk4cry2xpze' -{ - "app.bsky.graph.block": { ".subject": 13 }, - "app.bsky.graph.follow": { ".subject": 159 }, - "app.bsky.feed.post": { ".facets[].features[].did": 16 }, - "app.bsky.graph.listitem": { ".subject": 6 }, - "app.bsky.graph.starterpack": - { - ".feeds[].creator.did": 1, - ".feeds[].creator.labels[].src": 1 - } -} -``` - - -some todos - -- [x] find links and write them to rocksdb -- [x] handle account active status -- [x] handle account deletion -- [ ] handle account privacy setting? (is this a bsky-nsid-specific config and should that matter?) - - instead of looking this up, should be able to listen for it to be published on the firehose. - - this should _work_, but without backfill it won't be accurate. targeted backfill might be an option. -- [x] move ownership of canonical seq to an owned non-atomic -- [x] custom path for db storage -- [x] api server to look up backlink count -- [~] other useful endpoints for the api server - - [x] show all nisd/path links to target - - [x] get backlinking dids - - [x] paging for all backlinking dids - - [x] get count + most recent dids - - [ ] get count with any dids from provided set -- [~] write this readme -- [?] fix it sometimes getting stuck - - seems to unstick in my possibly-different repro (letting laptop fall asleep) after a bit. - - [ ] add a detection for no new links coming in after some period - - [x] add tcp connect, read, and write timeouts 🤞 -- [x] handle jetstream restart: don't miss events (currently sketch: rewinds cursor by 1us so we will always double-count at least one event) - - [x] especially: figure out what the risk is to rotating to another jetstream server in terms of gap/overlap from a different jetstream instance's cursor (follow up separately) - - [x] jetstream: don't rotate servers, explicitly pass via cli -- [x] metrics! - - [x] event ts lag -- [x] machine resource metrics - - [x] disk consumption - - [x] cpu usage - - [x] mem usage - - [x] network? -- [x] make all rocks apis return Result instead of unwrapping -- [~] handle all the unwraps -- [ ] deadletter queue of some kind for failed db writes - - [ ] also for valid json that was rejected? -- [x] get it running on raspi -- [x] get an estimate of disk usage per day after a few days of running - - very close to 1GB with data model before adding rkeys to linkers + fixing paths -- [x] make the did_init check only happen on test config (or remove it) (removed) -- [ ] actual error types (thiserror?) for lib-ish code -- [~] clean up the main readme -- [x] web server metrics - - [x] origin and ua labels -- [ ] tokio metrics? -- [x] handle shutdown cleanly -- be nice to rocksdb -- [x] add user-agent to jetstream request -- [ ] wow the shutdown stuff i wrote is really bad and doesn't work a lot -- [x] serve html for browser requests -- [ ] add a health check endpoint -- [x] add seq numbers to metrics -- [ ] persist the jetstream server url, error if started with a different one (maybe with --switch-streams or something) -- [ ] put delete-account tasks into a separate (persisted?) task queue for the writer so it can work on them incrementally. -- [ ] jetstream: connect retry: only reset counter after some *time* has passed. -- [x] either count or estimate the total number of links added (distinct from link targets) -- [x] jetstream: don't crash on connection refused (retry * backoff) -- [x] allow cors requests (ie. atproto-browser. (but it's really meant for backends)) -- [x] api: get distinct linking dids (https://bsky.app/profile/bnewbold.net/post/3lhhzejv7zc2h) - - [x] endpoint for count - - [x] endpoint for listing them - - [x] add to exploratory /all endpoint -- [ ] nginx: support http2 -- [x] nginx metrics -- [ ] add TimeoutLayer for axum -- [~] rocksdb metrics - - [x] write ops (count? per actionable?) - - [x] write time hist - - [ ] read ops (api) - - [ ] expose internal stats? -- [ ] figure out what's the right thing to do if merge op fails. happened on startup after an unclean reboot. -- [x] backups! - - [x] manual backup on startup - - [x] background task to create backups on an interval -- [ ] add a low-ulimit check on startup? - -cache -- [ ] set api response headers - - [ ] put "stale-while-revalidate" in Cache-Control w/ num seconds - - [ ] put "stale-if-error" in Cache-Control w/ num seconds - - [ ] set Expires or Cache-Control expires - - [ ] add Accept to vary response -- [ ] cache vary: might need to take bsky account privacy setting into account (unless this ends up being in query) - -data fixes -- [x] add rkey to linkers 🤦‍♀️ -- [x] don't remove deleted links from the reverse records -- null them out. this will keep things stable for paging. -- [x] don't show deactivated accounts in link responses -- [ ] canonicalize handles to dids! -- [ ] links: - - [~] pull `$type`/`type` from object children of arrays (distinguish replies, quotes, etc) - - just $type to start - - [ ] rewrite the entire "path" stuff - - [ ] actually define the format (deal with in-band dots etc) - - [x] ~_could_ throw cid neighbour into the target. probably should? but it's a lot of high volume uncompressible bytes~ - - and it could be looked up from the linker's doc - - ^^ for now, look up from source doc to get cid. might revisit this later. diff --git a/constellation/src/bin/main.rs b/constellation/src/bin/main.rs deleted file mode 100644 index b61f3c3..0000000 --- a/constellation/src/bin/main.rs +++ /dev/null @@ -1,272 +0,0 @@ -use anyhow::{bail, Result}; -use clap::{Parser, ValueEnum}; -use metrics_exporter_prometheus::PrometheusBuilder; -use std::num::NonZero; -use std::path::PathBuf; -use std::sync::{atomic::AtomicU32, Arc}; -use std::thread; -use std::time; -use tokio::runtime; -use tokio_util::sync::CancellationToken; - -use constellation::consumer::consume; -use constellation::server::serve; -#[cfg(feature = "rocks")] -use constellation::storage::RocksStorage; -use constellation::storage::{LinkReader, LinkStorage, MemStorage, StorageStats}; - -const MONITOR_INTERVAL: time::Duration = time::Duration::from_secs(15); - -/// Aggregate links in the at-mosphere -#[derive(Parser, Debug)] -#[command(version, about, long_about = None)] -struct Args { - #[arg(short, long)] - /// Jetstream server to connect to (exclusive with --fixture). Provide either a wss:// URL, or a shorhand value: - /// 'us-east-1', 'us-east-2', 'us-west-1', or 'us-west-2' - #[arg(short, long)] - jetstream: String, - // TODO: make this part of rocks' own sub-config? - /// Where to store data on disk, for backends that use disk storage - #[arg(short, long)] - data: Option, - /// Storage backend to use - #[arg(short, long)] - #[clap(value_enum, default_value_t = StorageBackend::Memory)] - backend: StorageBackend, - /// Initiate a database backup into this dir, if supported by the storage - #[arg(long)] - backup: Option, - /// Start a background task to take backups every N hours - #[arg(long)] - backup_interval: Option, - /// Keep at most this many backups purging oldest first, requires --backup-interval - #[arg(long)] - max_old_backups: Option, - /// Saved jsonl from jetstream to use instead of a live subscription - #[arg(short, long)] - fixture: Option, -} - -#[derive(Debug, Clone, ValueEnum)] -enum StorageBackend { - Memory, - #[cfg(feature = "rocks")] - Rocks, -} - -fn jetstream_url(provided: &str) -> String { - match provided { - "us-east-1" => "wss://jetstream1.us-east.bsky.network/subscribe".into(), - "us-east-2" => "wss://jetstream2.us-east.bsky.network/subscribe".into(), - "us-west-1" => "wss://jetstream1.us-west.bsky.network/subscribe".into(), - "us-west-2" => "wss://jetstream2.us-west.bsky.network/subscribe".into(), - custom => custom.into(), - } -} - -fn main() -> Result<()> { - let args = Args::parse(); - - println!("starting with storage backend: {:?}...", args.backend); - - let fixture = args.fixture; - if let Some(ref p) = fixture { - println!("using fixture at {p:?}..."); - } - - let stream = jetstream_url(&args.jetstream); - println!("using jetstream server {stream:?}...",); - - let stay_alive = CancellationToken::new(); - - match args.backend { - StorageBackend::Memory => run(MemStorage::new(), fixture, None, stream, stay_alive), - #[cfg(feature = "rocks")] - StorageBackend::Rocks => { - let storage_dir = args.data.clone().unwrap_or("rocks.test".into()); - println!("starting rocksdb..."); - let mut rocks = RocksStorage::new(storage_dir)?; - if let Some(backup_dir) = args.backup { - let auto_backup = match (args.backup_interval, args.max_old_backups) { - (Some(interval_hrs), copies) => Some((interval_hrs, copies)), - (None, None) => None, - (None, Some(_)) => bail!("invalid backup config: --max-old-backups requires --backup-interval to be configured"), - }; - rocks.start_backup(backup_dir, auto_backup, stay_alive.clone())?; - } - println!("rocks ready."); - run(rocks, fixture, args.data, stream, stay_alive) - } - } -} - -fn run( - mut storage: impl LinkStorage, - fixture: Option, - data_dir: Option, - stream: String, - stay_alive: CancellationToken, -) -> Result<()> { - ctrlc::set_handler({ - let mut desperation: u8 = 0; - let stay_alive = stay_alive.clone(); - move || match desperation { - 0 => { - println!("ok, shutting down..."); - stay_alive.cancel(); - desperation += 1; - } - 1.. => panic!("fine, panicking!"), - } - })?; - - let qsize = Arc::new(AtomicU32::new(0)); - - thread::scope(|s| { - let readable = storage.to_readable(); - - s.spawn({ - let qsize = qsize.clone(); - let stay_alive = stay_alive.clone(); - let staying_alive = stay_alive.clone(); - move || { - if let Err(e) = consume(storage, qsize, fixture, stream, staying_alive) { - eprintln!("jetstream finished with error: {e}"); - } - stay_alive.drop_guard(); - } - }); - - s.spawn({ - let readable = readable.clone(); - let stay_alive = stay_alive.clone(); - let staying_alive = stay_alive.clone(); - || { - runtime::Builder::new_multi_thread() - .worker_threads(1) - .max_blocking_threads(2) - .enable_all() - .build() - .expect("axum startup") - .block_on(async { - install_metrics_server()?; - serve(readable, "0.0.0.0:6789", staying_alive).await - }) - .unwrap(); - stay_alive.drop_guard(); - } - }); - - s.spawn(move || { // monitor thread - let stay_alive = stay_alive.clone(); - let check_alive = stay_alive.clone(); - - let process_collector = metrics_process::Collector::default(); - process_collector.describe(); - metrics::describe_gauge!( - "storage_available", - metrics::Unit::Bytes, - "available to be allocated" - ); - metrics::describe_gauge!( - "storage_free", - metrics::Unit::Bytes, - "unused bytes in filesystem" - ); - if let Some(ref p) = data_dir { - if let Err(e) = fs4::available_space(p) { - eprintln!("fs4 failed to get available space. may not be supported here? space metrics may be absent. e: {e:?}"); - } else { - println!("disk space monitoring should work, watching at {p:?}"); - } - } - - 'monitor: loop { - match readable.get_stats() { - Ok(StorageStats { dids, targetables, linking_records }) => { - metrics::gauge!("storage.stats.dids").set(dids as f64); - metrics::gauge!("storage.stats.targetables").set(targetables as f64); - metrics::gauge!("storage.stats.linking_records").set(linking_records as f64); - } - Err(e) => eprintln!("failed to get stats: {e:?}"), - } - - process_collector.collect(); - if let Some(ref p) = data_dir { - if let Ok(avail) = fs4::available_space(p) { - metrics::gauge!("storage.available").set(avail as f64); - } - if let Ok(free) = fs4::free_space(p) { - metrics::gauge!("storage.free").set(free as f64); - } - } - let wait = time::Instant::now(); - while wait.elapsed() < MONITOR_INTERVAL { - thread::sleep(time::Duration::from_millis(100)); - if check_alive.is_cancelled() { - break 'monitor - } - } - } - stay_alive.drop_guard(); - }); - }); - - println!("byeeee"); - - Ok(()) -} - -fn install_metrics_server() -> Result<()> { - println!("installing metrics server..."); - let host = [0, 0, 0, 0]; - let port = 8765; - PrometheusBuilder::new() - .set_quantiles(&[0.5, 0.9, 0.99, 1.0])? - .set_bucket_duration(time::Duration::from_secs(30))? - .set_bucket_count(NonZero::new(10).unwrap()) // count * duration = 5 mins. stuff doesn't happen that fast here. - .set_enable_unit_suffix(true) - .with_http_listener((host, port)) - .install()?; - println!( - "metrics server installed! listening on http://{}.{}.{}.{}:{port}", - host[0], host[1], host[2], host[3] - ); - Ok(()) -} - -#[cfg(test)] -mod tests { - use constellation::consumer::get_actionable; - use constellation::storage::{LinkReader, LinkStorage, MemStorage}; - - #[test] - fn test_create_like_integrated() { - let mut storage = MemStorage::new(); - - let rec = r#"{ - "did":"did:plc:icprmty6ticzracr5urz4uum", - "time_us":1736448492661668, - "kind":"commit", - "commit":{"rev":"3lfddpt5qa62c","operation":"create","collection":"app.bsky.feed.like","rkey":"3lfddpt5djw2c","record":{ - "$type":"app.bsky.feed.like", - "createdAt":"2025-01-09T18:48:10.412Z", - "subject":{"cid":"bafyreihazf62qvmusup55ojhkzwbmzee6rxtsug3e6eg33mnjrgthxvozu","uri":"at://did:plc:lphckw3dz4mnh3ogmfpdgt6z/app.bsky.feed.post/3lfdau5f7wk23"} - }, - "cid":"bafyreidgcs2id7nsbp6co42ind2wcig3riwcvypwan6xdywyfqklovhdjq"} - }"#.parse().unwrap(); - let (action, ts) = get_actionable(&rec).unwrap(); - storage.push(&action, ts).unwrap(); - assert_eq!( - storage - .get_count( - "at://did:plc:lphckw3dz4mnh3ogmfpdgt6z/app.bsky.feed.post/3lfdau5f7wk23", - "app.bsky.feed.like", - ".subject.uri" - ) - .unwrap(), - 1 - ); - } -} diff --git a/constellation/src/bin/rocks-link-stats.rs b/constellation/src/bin/rocks-link-stats.rs deleted file mode 100644 index 4a24a71..0000000 --- a/constellation/src/bin/rocks-link-stats.rs +++ /dev/null @@ -1,281 +0,0 @@ -use bincode::config::Options; -use clap::Parser; -use serde::Serialize; -use std::collections::HashMap; -use std::path::PathBuf; - -use tokio_util::sync::CancellationToken; - -use constellation::storage::rocks_store::{ - Collection, DidId, RKey, RPath, Target, TargetKey, TargetLinkers, _bincode_opts, -}; -use constellation::storage::RocksStorage; -use constellation::Did; - -use links::parse_any_link; -use rocksdb::IteratorMode; -use std::time; - -/// Aggregate links in the at-mosphere -#[derive(Parser, Debug)] -#[command(version, about, long_about = None)] -struct Args { - /// where is rocksdb's data - #[arg(short, long)] - data: PathBuf, - /// slow down so we don't kill the firehose consumer, if running concurrently - #[arg(short, long)] - limit: Option, -} - -type LinkType = String; - -#[derive(Debug, Eq, Hash, PartialEq, Serialize)] -struct SourceLink(Collection, RPath, LinkType, Option); // last is target collection, if it's an at-uri link with a collection - -#[derive(Debug, Serialize)] -struct SourceSample { - did: String, - rkey: String, -} - -#[derive(Debug, Default, Serialize)] -struct Bucket { - count: u64, - sum: u64, - sample: Option, -} - -#[derive(Debug, Default, Serialize)] -struct Buckets([Bucket; 23]); - -const BUCKETS: [u64; 23] = [ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 16, 32, 64, 128, 256, 512, 1024, 4096, 16_384, 65_535, - 262_144, 1_048_576, -]; - -// b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b12, b16, b32, b64, b128, b256, b512, b1024, b4096, b16384, b65535, b262144, bmax - -static DID_IDS_CF: &str = "did_ids"; -static TARGET_IDS_CF: &str = "target_ids"; -static TARGET_LINKERS_CF: &str = "target_links"; - -const REPORT_INTERVAL: usize = 50_000; - -type Stats = HashMap; - -#[derive(Debug, Serialize)] -struct Printable { - collection: String, - path: String, - link_type: String, - target_collection: Option, - buckets: Buckets, -} - -#[derive(Debug, Default)] -struct ErrStats { - failed_to_get_sample: usize, - failed_to_read_target_id: usize, - failed_to_deserialize_target_key: usize, - failed_to_parse_target_as_link: usize, - failed_to_get_links: usize, - failed_to_deserialize_linkers: usize, -} - -fn thousands(n: usize) -> String { - n.to_string() - .as_bytes() - .rchunks(3) - .rev() - .map(std::str::from_utf8) - .collect::, _>>() - .unwrap() - .join(",") -} - -fn main() { - let args = Args::parse(); - - let limit = args.limit.map(|amount| { - ratelimit::Ratelimiter::builder(amount, time::Duration::from_secs(1)) - .max_tokens(amount) - .initial_available(amount) - .build() - .unwrap() - }); - - eprintln!("starting rocksdb..."); - let rocks = RocksStorage::open_readonly(args.data).unwrap(); - eprintln!("rocks ready."); - - let RocksStorage { ref db, .. } = rocks; - - let stay_alive = CancellationToken::new(); - ctrlc::set_handler({ - let mut desperation: u8 = 0; - let stay_alive = stay_alive.clone(); - move || match desperation { - 0 => { - eprintln!("ok, shutting down..."); - stay_alive.cancel(); - desperation += 1; - } - 1.. => panic!("fine, panicking!"), - } - }) - .unwrap(); - - let mut stats = Stats::new(); - let mut err_stats: ErrStats = Default::default(); - - let did_ids_cf = db.cf_handle(DID_IDS_CF).unwrap(); - let target_id_cf = db.cf_handle(TARGET_IDS_CF).unwrap(); - let target_links_cf = db.cf_handle(TARGET_LINKERS_CF).unwrap(); - - let t0 = time::Instant::now(); - let mut t_prev = t0; - - let mut i = 0; - for item in db.iterator_cf(&target_id_cf, IteratorMode::Start) { - if stay_alive.is_cancelled() { - break; - } - - if let Some(ref limiter) = limit { - if let Err(dur) = limiter.try_wait() { - std::thread::sleep(dur) - } - } - - if i > 0 && i % REPORT_INTERVAL == 0 { - let now = time::Instant::now(); - let rate = (REPORT_INTERVAL as f32) / (now.duration_since(t_prev).as_secs_f32()); - eprintln!( - "{i}\t({}k)\t{:.2}\t{rate:.1}/s", - thousands(i / 1000), - t0.elapsed().as_secs_f32() - ); - t_prev = now; - } - i += 1; - - let Ok((target_key, target_id)) = item else { - err_stats.failed_to_read_target_id += 1; - continue; - }; - - let Ok(TargetKey(Target(target), collection, rpath)) = - _bincode_opts().deserialize(&target_key) - else { - err_stats.failed_to_deserialize_target_key += 1; - continue; - }; - - let source = { - let Some(parsed) = parse_any_link(&target) else { - err_stats.failed_to_parse_target_as_link += 1; - continue; - }; - SourceLink( - collection, - rpath, - parsed.name().into(), - parsed.at_uri_collection().map(Collection), - ) - }; - - let Ok(Some(links_raw)) = db.get_cf(&target_links_cf, &target_id) else { - err_stats.failed_to_get_links += 1; - continue; - }; - let Ok(linkers) = _bincode_opts().deserialize::(&links_raw) else { - err_stats.failed_to_deserialize_linkers += 1; - continue; - }; - let (n, _) = linkers.count(); - - if n == 0 { - continue; - } - - let mut bucket = 0; - for edge in BUCKETS { - if n <= edge || bucket == 22 { - break; - } - bucket += 1; - } - - let b = &mut stats.entry(source).or_default().0[bucket]; - b.count += 1; - b.sum += n; - if b.sample.is_none() { - let (DidId(did_id), RKey(k)) = &linkers.0[(n - 1) as usize]; - if let Ok(Some(did_bytes)) = db.get_cf(&did_ids_cf, did_id.to_be_bytes()) { - if let Ok(Did(did)) = _bincode_opts().deserialize(&did_bytes) { - b.sample = Some(SourceSample { - did, - rkey: k.clone(), - }); - } else { - err_stats.failed_to_get_sample += 1; - } - } else { - err_stats.failed_to_get_sample += 1; - } - } - - // if i >= 40_000 { - // break; - // } - } - - let dt = t0.elapsed(); - - eprintln!("gathering stats for output..."); - - let itemified = stats - .into_iter() - .map( - |( - SourceLink(Collection(collection), RPath(path), link_type, target_collection), - buckets, - )| Printable { - collection, - path, - link_type, - target_collection: target_collection.map(|Collection(c)| c), - buckets, - }, - ) - .collect::>(); - - match serde_json::to_string(&itemified) { - Ok(s) => println!("{s}"), - Err(e) => eprintln!("failed to serialize results: {e:?}"), - } - - eprintln!( - "{} summarizing {} link targets in {:.1}s", - if stay_alive.is_cancelled() { - "STOPPED" - } else { - "FINISHED" - }, - thousands(i), - dt.as_secs_f32() - ); - eprintln!("{err_stats:?}"); - eprintln!("bye."); -} - -// scan plan - -// buckets (backlink count) -// 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 16, 32, 64, 128, 256, 512, 1024, 4096, 16384, 65535, 262144, 1048576+ -// by -// - collection -// - json path -// - link type -// samples for each bucket for each variation diff --git a/constellation/src/bin/rocks-restore-from-backup.rs b/constellation/src/bin/rocks-restore-from-backup.rs deleted file mode 100644 index b058998..0000000 --- a/constellation/src/bin/rocks-restore-from-backup.rs +++ /dev/null @@ -1,54 +0,0 @@ -use crate::time::Instant; -use anyhow::Result; -use clap::Parser; -use std::path::PathBuf; - -use rocksdb::backup::{BackupEngine, BackupEngineOptions, RestoreOptions}; - -use std::time; - -/// Aggregate links in the at-mosphere -#[derive(Parser, Debug)] -#[command(version, about, long_about = None)] -struct Args { - /// the backup directory to restore *from* - #[arg(long)] - from_backup_dir: PathBuf, - /// the db dir to restore *to* - #[arg(long)] - to_data_dir: PathBuf, -} - -fn main() -> Result<()> { - let args = Args::parse(); - - eprintln!( - "restoring latest rocksdb backup from {:?} to {:?}...", - args.from_backup_dir, args.to_data_dir - ); - - let mut engine = BackupEngine::open( - &BackupEngineOptions::new(args.from_backup_dir)?, - &rocksdb::Env::new()?, - )?; - - let t0 = Instant::now(); - if let Err(e) = engine.restore_from_latest_backup( - &args.to_data_dir, - &args.to_data_dir, - &RestoreOptions::default(), - ) { - eprintln!( - "restoring from backup failed after {:?}: {e:?}", - t0.elapsed() - ); - } else { - eprintln!( - "success, restored latest from backup after {:?}", - t0.elapsed() - ); - } - - eprintln!("bye."); - Ok(()) -} diff --git a/constellation/src/consumer/jetstream.rs b/constellation/src/consumer/jetstream.rs deleted file mode 100644 index ea56864..0000000 --- a/constellation/src/consumer/jetstream.rs +++ /dev/null @@ -1,329 +0,0 @@ -use anyhow::{bail, Result}; -use metrics::{ - counter, describe_counter, describe_gauge, describe_histogram, gauge, histogram, Unit, -}; -use std::io::{Cursor, ErrorKind, Read}; -use std::net::ToSocketAddrs; -use std::thread; -use std::time; -use tinyjson::JsonValue; -use tokio_util::sync::CancellationToken; -use tungstenite::{client::IntoClientRequest, Error as TError, Message}; -use zstd::dict::DecoderDictionary; - -const JETSTREAM_ZSTD_DICTIONARY: &[u8] = include_bytes!("../../zstd/dictionary"); - -pub fn consume_jetstream( - sender: flume::Sender, - cursor: Option, - stream: String, - staying_alive: CancellationToken, -) -> Result<()> { - describe_counter!( - "jetstream_connnect", - Unit::Count, - "attempts to connect to a jetstream server" - ); - describe_counter!( - "jetstream_read", - Unit::Count, - "attempts to read an event from jetstream" - ); - describe_counter!( - "jetstream_read_fail", - Unit::Count, - "failures to read events from jetstream" - ); - describe_counter!( - "jetstream_read_bytes", - Unit::Bytes, - "total received message bytes from jetstream" - ); - describe_counter!( - "jetstream_read_bytes_decompressed", - Unit::Bytes, - "total decompressed message bytes from jetstream" - ); - describe_histogram!( - "jetstream_read_bytes_decompressed", - Unit::Bytes, - "decompressed size of jetstream messages" - ); - describe_counter!( - "jetstream_events", - Unit::Count, - "valid json messages received" - ); - describe_histogram!( - "jetstream_events_queued", - Unit::Count, - "event messages waiting in queue" - ); - describe_gauge!( - "jetstream_cursor_age", - Unit::Microseconds, - "microseconds between our clock and the jetstream event's time_us" - ); - - let dict = DecoderDictionary::copy(JETSTREAM_ZSTD_DICTIONARY); - let mut connect_retries = 0; - let mut latest_cursor = cursor; - 'outer: loop { - let stream_url = format!( - "{stream}?compress=true{}", - latest_cursor - .map(|c| { - println!("starting with cursor from {:?} ago...", ts_age(c)); - format!("&cursor={c}") - }) - .unwrap_or("".into()) - ); - let mut req = (&stream_url).into_client_request()?; - let ua = format!("microcosm/constellation v{}", env!("CARGO_PKG_VERSION")); - req.headers_mut().insert("user-agent", ua.parse()?); - - let host = req.uri().host().expect("jetstream request uri has a host"); - let port = req.uri().port().map(|p| p.as_u16()).unwrap_or(443); - let dest = format!("{host}:{port}"); - let addr = match dest.to_socket_addrs().map(|mut d| d.next()) { - Ok(Some(a)) => a, - Ok(None) => { - eprintln!( - "jetstream: could not resolve an address for {dest:?}. retrying after a bit?" - ); - thread::sleep(time::Duration::from_secs(15)); - continue; - } - Err(e) => { - eprintln!("jetstream failed to resolve address {dest:?}: {e:?} waiting and then retrying..."); - thread::sleep(time::Duration::from_secs(3)); - continue; - } - }; - let tcp_stream = match std::net::TcpStream::connect_timeout( - &addr, - time::Duration::from_secs(8), - ) { - Ok(s) => s, - Err(e) => { - eprintln!( - "jetstream failed to make tcp connection: {e:?}. (todo: clean up retry logic)" - ); - connect_retries += 1; - if connect_retries >= 7 { - eprintln!("jetstream: no more connect retries, breaking out."); - break; - } - let backoff = time::Duration::from_secs(connect_retries.try_into().unwrap()); - eprintln!("jetstream tcp failed to connect: {e:?}. backing off {backoff:?} before retrying..."); - thread::sleep(backoff); - continue; - } - }; - tcp_stream.set_read_timeout(Some(time::Duration::from_secs(4)))?; - tcp_stream.set_write_timeout(Some(time::Duration::from_secs(4)))?; - - counter!("jetstream_connect", "url" => stream.clone(), "is_retry" => (connect_retries > 0).to_string()).increment(1); - println!("jetstream connecting, attempt #{connect_retries}, {stream_url:?} with user-agent: {ua:?}"); - let mut socket = match tungstenite::client_tls(req, tcp_stream) { - Ok((socket, _)) => { - println!("jetstream connected."); - // connect_retries = 0; // only reset once we have received a message vvv - socket - } - Err(e) => { - connect_retries += 1; - if connect_retries >= 7 { - eprintln!("jetstream: no more connect retries, breaking out."); - break; - } - let backoff = time::Duration::from_secs(connect_retries.try_into().unwrap()); - eprintln!("jetstream failed to connect: {e:?}. backing off {backoff:?} before retrying..."); - thread::sleep(backoff); - continue; - } - }; - - loop { - if !socket.can_read() { - eprintln!("jetstream: socket says we cannot read -- flushing then breaking out."); - if let Err(e) = socket.flush() { - eprintln!("error while flushing socket: {e:?}"); - } - break; - } - - if staying_alive.is_cancelled() { - eprintln!("jetstream: cancelling"); - // TODO: cleanly close the connection? - break 'outer; - } - - counter!("jetstream_read").increment(1); - let b = match socket.read() { - Ok(Message::Binary(b)) => b, - Ok(Message::Text(_)) => { - counter!("jetstream_read_fail", "url" => stream.clone(), "reason" => "received text") - .increment(1); - eprintln!("jetstream: unexpected text message, should be binary for compressed (ignoring)"); - continue; - } - Ok(Message::Close(f)) => { - counter!("jetstream_read_fail", "url" => stream.clone(), "reason" => "server closed") - .increment(1); - println!("jetstream: closing the connection: {f:?}"); - continue; - } - Ok(Message::Ping(bytes)) => { - let _ = socket.send(Message::Pong(bytes)); - continue; - } - Ok(m) => { - counter!("jetstream_read_fail", "url" => stream.clone(), "reason" => "unexpected message", "message" => format!("{m:?}")).increment(1); - eprintln!("jetstream: unexpected from read (ignoring): {m:?}"); - continue; - } - Err(TError::ConnectionClosed) => { - // clean exit - counter!("jetstream_read_fail", "url" => stream.clone(), "reason" => "clean close") - .increment(1); - println!("jetstream closed the websocket cleanly."); - break; - } - Err(TError::AlreadyClosed) => { - // programming error - counter!("jetstream_read_fail", "url" => stream.clone(), "reason" => "already closed") - .increment(1); - eprintln!( - "jetstream: got AlreadyClosed trying to .read() websocket. probably a bug." - ); - break; - } - Err(TError::Capacity(e)) => { - counter!("jetstream_read_fail", "url" => stream.clone(), "reason" => "capacity error") - .increment(1); - eprintln!("jetstream: capacity error (ignoring): {e:?}"); - continue; - } - Err(TError::Utf8) => { - counter!("jetstream_read_fail", "url" => stream.clone(), "reason" => "utf8 error") - .increment(1); - eprintln!("jetstream: utf8 error (ignoring)"); - continue; - } - Err(e) => { - eprintln!("jetstream: could not read message from socket. closing: {e:?}"); - if let TError::Io(io_err) = e { - if matches!(io_err.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut) { - counter!("jetstream_read_fail", "url" => stream.clone(), "reason" => "timed out").increment(1); - println!("jetstream socket timed out. bailing to reconnect -- should we be trying to close first?"); - break; - } - } - match socket.close(None) { - Err(TError::ConnectionClosed) => { - counter!("jetstream_read_fail", "url" => stream.clone(), "reason" => "clean close").increment(1); - println!("jetstream closed the websocket cleanly."); - break; - } - r => eprintln!("jetstream: close result after error: {r:?}"), - } - counter!("jetstream_read_fail", "url" => stream.clone(), "reason" => "read error") - .increment(1); - // if we didn't immediately get ConnectionClosed, we should keep polling read - // until we get it. - continue; - } - }; - - counter!("jetstream_read_bytes", "url" => stream.clone()).increment(b.len() as u64); - let mut cursor = Cursor::new(b); - let mut decoder = match zstd::stream::Decoder::with_prepared_dictionary( - &mut cursor, - &dict, - ) { - Ok(d) => d, - Err(e) => { - counter!("jetstream_read_fail", "url" => stream.clone(), "reason" => "zstd decompress") - .increment(1); - eprintln!("jetstream: failed to decompress zstd message: {e:?}"); - continue; - } - }; - - let mut s = String::new(); - match decoder.read_to_string(&mut s) { - Ok(n) => { - counter!("jetstream_read_bytes_decompressed", "url" => stream.clone()) - .increment(n as u64); - histogram!("jetstream_read_bytes_decompressed", "url" => stream.clone()) - .record(n as f64); - } - Err(e) => { - counter!("jetstream_read_fail", "url" => stream.clone(), "reason" => "zstd string decode") - .increment(1); - eprintln!("jetstream: failed to decode zstd: {e:?}"); - continue; - } - } - - let v = match s.parse() { - Ok(v) => v, - Err(e) => { - counter!("jetstream_read_fail", "url" => stream.clone(), "reason" => "json parse") - .increment(1); - eprintln!("jetstream: failed to parse message as json: {e:?}"); - continue; - } - }; - - // bit of a hack to have this here for now... - let ts = match get_event_time(&v) { - Some(ts) => ts, - None => { - counter!("jetstream_read_fail", "url" => stream.clone(), "reason" => "invalid event") - .increment(1); - eprintln!("jetstream: encountered an event without a timestamp: ignoring it."); - continue; - } - }; - - if let Err(flume::SendError(_rejected)) = sender.send(v) { - counter!("jetstream_events", "url" => stream.clone()).increment(1); - if sender.is_disconnected() { - eprintln!("jetstream: send channel disconnected -- nothing to do, bye."); - bail!("jetstream: send channel disconnected"); - } - eprintln!( - "jetstream: failed to send on channel, dropping update! (FIXME / HANDLEME)" - ); - } - histogram!("jetstream_events_queued", "url" => stream.clone()) - .record(sender.len() as f64); - - // only actually update our cursor after we've managed to queue the event - latest_cursor = Some(ts); - gauge!("jetstream_cursor_age", "url" => stream.clone()) - .set(ts_age(ts).as_micros() as f64); - - // great news if we got this far -- might be safe to assume the connection is up. - connect_retries = 0; - } - } - Ok(()) -} - -fn get_event_time(v: &JsonValue) -> Option { - if let JsonValue::Object(root) = v { - if let JsonValue::Number(time_us) = root.get("time_us")? { - return Some(*time_us as u64); - }; - }; - None -} - -fn ts_age(ts: u64) -> time::Duration { - (time::UNIX_EPOCH + time::Duration::from_micros(ts)) - .elapsed() - .unwrap_or(time::Duration::from_secs(0)) // saturate zero if ts > our system time -} diff --git a/constellation/src/consumer/jsonl_file.rs b/constellation/src/consumer/jsonl_file.rs deleted file mode 100644 index 7a65e8e..0000000 --- a/constellation/src/consumer/jsonl_file.rs +++ /dev/null @@ -1,19 +0,0 @@ -use anyhow::{bail, Result}; -use std::fs::File; -use std::io::{self, BufRead}; -use std::path::PathBuf; -use tinyjson::JsonValue; - -pub fn consume_jsonl_file(f: PathBuf, sender: flume::Sender) -> Result<()> { - let file = File::open(f)?; - for line in io::BufReader::new(file).lines().map_while(Result::ok) { - if let Err(flume::SendError(_rejected)) = sender.send(line.parse()?) { - if sender.is_disconnected() { - bail!("fixture: send channel disconnected -- nothing to do, bye."); - } - eprintln!("fixture: failed to send on channel, dropping update! (FIXME / HANDLEME)"); - } - } - println!("reached end of jsonl file"); - Ok(()) -} diff --git a/constellation/src/consumer/mod.rs b/constellation/src/consumer/mod.rs deleted file mode 100644 index 4c52980..0000000 --- a/constellation/src/consumer/mod.rs +++ /dev/null @@ -1,355 +0,0 @@ -mod jetstream; -mod jsonl_file; - -use crate::storage::LinkStorage; -use crate::{ActionableEvent, RecordId}; -use anyhow::Result; -use jetstream::consume_jetstream; -use jsonl_file::consume_jsonl_file; -use links::collect_links; -use metrics::{counter, describe_counter, describe_histogram, histogram, Unit}; -use std::path::PathBuf; -use std::sync::atomic::{AtomicU32, Ordering}; -use std::sync::Arc; -use std::thread; -use tinyjson::JsonValue; -use tokio_util::sync::CancellationToken; - -pub fn consume( - mut store: impl LinkStorage, - qsize: Arc, - fixture: Option, - stream: String, - staying_alive: CancellationToken, -) -> Result<()> { - describe_counter!( - "consumer_events_non_actionable", - Unit::Count, - "count of non-actionable events" - ); - describe_counter!( - "consumer_events_actionable", - Unit::Count, - "count of action by type. *all* atproto record delete events are included" - ); - describe_counter!( - "consumer_events_actionable_links", - Unit::Count, - "total links encountered" - ); - describe_histogram!( - "consumer_events_actionable_links", - Unit::Count, - "number of links per message" - ); - - let (receiver, consumer_handle) = if let Some(f) = fixture { - let (sender, receiver) = flume::bounded(21); - ( - receiver, - thread::spawn(move || consume_jsonl_file(f, sender)), - ) - } else { - let (sender, receiver) = flume::bounded(32_768); // eek - let cursor = store.get_cursor().unwrap(); - ( - receiver, - thread::spawn(move || consume_jetstream(sender, cursor, stream, staying_alive)), - ) - }; - - for update in receiver.iter() { - if let Some((action, ts)) = get_actionable(&update) { - { - store.push(&action, ts).unwrap(); - qsize.store(receiver.len().try_into().unwrap(), Ordering::Relaxed); - } - } else { - counter!("consumer_events_non_actionable").increment(1); - } - } - - consumer_handle.join().unwrap() -} - -pub fn get_actionable(event: &JsonValue) -> Option<(ActionableEvent, u64)> { - let JsonValue::Object(root) = event else { - return None; - }; - let JsonValue::Number(time_us) = root.get("time_us")? else { - return None; - }; - let cursor = *time_us as u64; - // todo: clean up - match event { - JsonValue::Object(root) - if root.get("kind") == Some(&JsonValue::String("commit".to_string())) => - { - let JsonValue::String(did) = root.get("did")? else { - return None; - }; - let JsonValue::Object(commit) = root.get("commit")? else { - return None; - }; - let JsonValue::String(collection) = commit.get("collection")? else { - return None; - }; - let JsonValue::String(rkey) = commit.get("rkey")? else { - return None; - }; - match commit.get("operation")? { - JsonValue::String(op) if op == "create" => { - let links = collect_links(commit.get("record")?); - counter!("consumer_events_actionable", "action_type" => "create_links", "collection" => collection.clone()).increment(1); - histogram!("consumer_events_actionable_links", "action_type" => "create_links", "collection" => collection.clone()).record(links.len() as f64); - for link in &links { - counter!("consumer_events_actionable_links", - "action_type" => "create_links", - "collection" => collection.clone(), - "path" => link.path.clone(), - "link_type" => link.target.name(), - ) - .increment(links.len() as u64); - } - if links.is_empty() { - None - } else { - Some(( - ActionableEvent::CreateLinks { - record_id: RecordId { - did: did.into(), - collection: collection.clone(), - rkey: rkey.clone(), - }, - links, - }, - cursor, - )) - } - } - JsonValue::String(op) if op == "update" => { - let links = collect_links(commit.get("record")?); - counter!("consumer_events_actionable", "action_type" => "update_links", "collection" => collection.clone()).increment(1); - histogram!("consumer_events_actionable_links", "action_type" => "update_links", "collection" => collection.clone()).record(links.len() as f64); - for link in &links { - counter!("consumer_events_actionable_links", - "action_type" => "update_links", - "collection" => collection.clone(), - "path" => link.path.clone(), - "link_type" => link.target.name(), - ) - .increment(links.len() as u64); - } - Some(( - ActionableEvent::UpdateLinks { - record_id: RecordId { - did: did.into(), - collection: collection.clone(), - rkey: rkey.clone(), - }, - new_links: links, - }, - cursor, - )) - } - JsonValue::String(op) if op == "delete" => { - counter!("consumer_events_actionable", "action_type" => "delete_record", "collection" => collection.clone()).increment(1); - Some(( - ActionableEvent::DeleteRecord(RecordId { - did: did.into(), - collection: collection.clone(), - rkey: rkey.clone(), - }), - cursor, - )) - } - _ => None, - } - } - JsonValue::Object(root) - if root.get("kind") == Some(&JsonValue::String("account".to_string())) => - { - let JsonValue::Object(account) = root.get("account")? else { - return None; - }; - let did = account.get("did")?.get::()?.clone(); - match (account.get("active")?.get::()?, account.get("status")) { - (true, None) => { - counter!("consumer_events_actionable", "action_type" => "account", "action" => "activate").increment(1); - Some((ActionableEvent::ActivateAccount(did.into()), cursor)) - } - (false, Some(JsonValue::String(status))) => match status.as_ref() { - "deactivated" => { - counter!("consumer_events_actionable", "action_type" => "account", "action" => "deactivate").increment(1); - Some((ActionableEvent::DeactivateAccount(did.into()), cursor)) - } - "deleted" => { - counter!("consumer_events_actionable", "action_type" => "account", "action" => "delete").increment(1); - Some((ActionableEvent::DeleteAccount(did.into()), cursor)) - } - // TODO: are we missing handling for suspended and deactivated accounts? - _ => None, - }, - _ => None, - } - } - _ => None, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use links::{CollectedLink, Link}; - - #[test] - fn test_create_like() { - let rec = r#"{ - "did":"did:plc:icprmty6ticzracr5urz4uum", - "time_us":1736448492661668, - "kind":"commit", - "commit":{"rev":"3lfddpt5qa62c","operation":"create","collection":"app.bsky.feed.like","rkey":"3lfddpt5djw2c","record":{ - "$type":"app.bsky.feed.like", - "createdAt":"2025-01-09T18:48:10.412Z", - "subject":{"cid":"bafyreihazf62qvmusup55ojhkzwbmzee6rxtsug3e6eg33mnjrgthxvozu","uri":"at://did:plc:lphckw3dz4mnh3ogmfpdgt6z/app.bsky.feed.post/3lfdau5f7wk23"} - }, - "cid":"bafyreidgcs2id7nsbp6co42ind2wcig3riwcvypwan6xdywyfqklovhdjq"} - }"#.parse().unwrap(); - let action = get_actionable(&rec); - assert_eq!( - action, - Some(( - ActionableEvent::CreateLinks { - record_id: RecordId { - did: "did:plc:icprmty6ticzracr5urz4uum".into(), - collection: "app.bsky.feed.like".into(), - rkey: "3lfddpt5djw2c".into(), - }, - links: vec![CollectedLink { - path: ".subject.uri".into(), - target: Link::AtUri( - "at://did:plc:lphckw3dz4mnh3ogmfpdgt6z/app.bsky.feed.post/3lfdau5f7wk23" - .into() - ) - },], - }, - 1736448492661668 - )) - ) - } - - #[test] - fn test_update_profile() { - let rec = r#"{ - "did":"did:plc:tcmiubbjtkwhmnwmrvr2eqnx", - "time_us":1736453696817289,"kind":"commit", - "commit":{ - "rev":"3lfdikw7q772c", - "operation":"update", - "collection":"app.bsky.actor.profile", - "rkey":"self", - "record":{ - "$type":"app.bsky.actor.profile", - "avatar":{"$type":"blob","ref":{"$link":"bafkreidcg5jzz3hpdtlc7um7w5masiugdqicc5fltuajqped7fx66hje54"},"mimeType":"image/jpeg","size":295764}, - "banner":{"$type":"blob","ref":{"$link":"bafkreiahaswf2yex2zfn3ynpekhw6mfj7254ra7ly27zjk73czghnz2wni"},"mimeType":"image/jpeg","size":856461}, - "createdAt":"2024-08-30T21:33:06.945Z", - "description":"Professor, QUB | Belfast via Derry \\n\\nViews personal | Reposts are not an endorsement\\n\\nhttps://go.qub.ac.uk/charvey", - "displayName":"Colin Harvey", - "pinnedPost":{"cid":"bafyreifyrepqer22xsqqnqulpcxzpu7wcgeuzk6p5c23zxzctaiwmlro7y","uri":"at://did:plc:tcmiubbjtkwhmnwmrvr2eqnx/app.bsky.feed.post/3lf66ri63u22t"} - }, - "cid":"bafyreiem4j5p7duz67negvqarq3s5h7o45fvytevhrzkkn2p6eqdkcf74m" - } - }"#.parse().unwrap(); - let action = get_actionable(&rec); - assert_eq!( - action, - Some(( - ActionableEvent::UpdateLinks { - record_id: RecordId { - did: "did:plc:tcmiubbjtkwhmnwmrvr2eqnx".into(), - collection: "app.bsky.actor.profile".into(), - rkey: "self".into(), - }, - new_links: vec![CollectedLink { - path: ".pinnedPost.uri".into(), - target: Link::AtUri( - "at://did:plc:tcmiubbjtkwhmnwmrvr2eqnx/app.bsky.feed.post/3lf66ri63u22t" - .into() - ), - },], - }, - 1736453696817289 - )) - ) - } - - #[test] - fn test_delete_like() { - let rec = r#"{ - "did":"did:plc:3pa2ss4l2sqzhy6wud4btqsj", - "time_us":1736448492690783, - "kind":"commit", - "commit":{"rev":"3lfddpt7vnx24","operation":"delete","collection":"app.bsky.feed.like","rkey":"3lbiu72lczk2w"} - }"#.parse().unwrap(); - let action = get_actionable(&rec); - assert_eq!( - action, - Some(( - ActionableEvent::DeleteRecord(RecordId { - did: "did:plc:3pa2ss4l2sqzhy6wud4btqsj".into(), - collection: "app.bsky.feed.like".into(), - rkey: "3lbiu72lczk2w".into(), - }), - 1736448492690783 - )) - ) - } - - #[test] - fn test_delete_account() { - let rec = r#"{ - "did":"did:plc:zsgqovouzm2gyksjkqrdodsw", - "time_us":1736451739215876, - "kind":"account", - "account":{"active":false,"did":"did:plc:zsgqovouzm2gyksjkqrdodsw","seq":3040934738,"status":"deleted","time":"2025-01-09T19:42:18.972Z"} - }"#.parse().unwrap(); - let action = get_actionable(&rec); - assert_eq!( - action, - Some(( - ActionableEvent::DeleteAccount("did:plc:zsgqovouzm2gyksjkqrdodsw".into()), - 1736451739215876 - )) - ) - } - - #[test] - fn test_deactivate_account() { - let rec = r#"{ - "did":"did:plc:l4jb3hkq7lrblferbywxkiol","time_us":1736451745611273,"kind":"account","account":{"active":false,"did":"did:plc:l4jb3hkq7lrblferbywxkiol","seq":3040939563,"status":"deactivated","time":"2025-01-09T19:42:22.035Z"} - }"#.parse().unwrap(); - let action = get_actionable(&rec); - assert_eq!( - action, - Some(( - ActionableEvent::DeactivateAccount("did:plc:l4jb3hkq7lrblferbywxkiol".into()), - 1736451745611273 - )) - ) - } - - #[test] - fn test_activate_account() { - let rec = r#"{ - "did":"did:plc:nct6zfb2j4emoj4yjomxwml2","time_us":1736451747292706,"kind":"account","account":{"active":true,"did":"did:plc:nct6zfb2j4emoj4yjomxwml2","seq":3040940775,"time":"2025-01-09T19:42:26.924Z"} - }"#.parse().unwrap(); - let action = get_actionable(&rec); - assert_eq!( - action, - Some(( - ActionableEvent::ActivateAccount("did:plc:nct6zfb2j4emoj4yjomxwml2".into()), - 1736451747292706 - )) - ) - } -} diff --git a/constellation/src/lib.rs b/constellation/src/lib.rs deleted file mode 100644 index bcb4e9b..0000000 --- a/constellation/src/lib.rs +++ /dev/null @@ -1,58 +0,0 @@ -pub mod consumer; -pub mod server; -pub mod storage; - -use links::CollectedLink; -use serde::{Deserialize, Serialize}; -use std::convert::From; - -#[derive(Debug, PartialEq)] -pub enum ActionableEvent { - CreateLinks { - record_id: RecordId, - links: Vec, - }, - UpdateLinks { - record_id: RecordId, - new_links: Vec, - }, - DeleteRecord(RecordId), - ActivateAccount(Did), - DeactivateAccount(Did), - DeleteAccount(Did), -} - -#[derive(Debug, PartialEq, Eq, Hash, Clone, Serialize, Deserialize)] -pub struct Did(pub String); - -impl> From for Did { - fn from(s: T) -> Self { - Self(s.into()) - } -} - -#[derive(Debug, PartialEq, Serialize, Deserialize)] -pub struct RecordId { - pub did: Did, - pub collection: String, - pub rkey: String, -} - -impl RecordId { - pub fn did(&self) -> Did { - self.did.clone() - } - pub fn collection(&self) -> String { - self.collection.clone() - } - pub fn rkey(&self) -> String { - self.rkey.clone() - } -} - -/// maybe the worst type in this repo, and there are some bad types -#[derive(Debug, Serialize, PartialEq)] -pub struct CountsByCount { - pub records: u64, - pub distinct_dids: u64, -} diff --git a/constellation/src/server/acceptable.rs b/constellation/src/server/acceptable.rs deleted file mode 100644 index f00432d..0000000 --- a/constellation/src/server/acceptable.rs +++ /dev/null @@ -1,29 +0,0 @@ -use askama::Template; -use axum::http::StatusCode; -use axum::response::{Html, IntoResponse, Json, Response}; -use axum_extra::TypedHeader; -use headers_accept::Accept; -use mediatype::names::{APPLICATION, HTML, JSON, TEXT}; -use mediatype::MediaType; -use serde::Serialize; - -const TEXT_HTML: MediaType = MediaType::new(TEXT, HTML); -const APPLICATION_JSON: MediaType = MediaType::new(APPLICATION, JSON); -const AVAILABLE: &[MediaType] = &[APPLICATION_JSON, TEXT_HTML]; - -pub type ExtractAccept = Option>; - -pub fn acceptable(accept: ExtractAccept, thing: T) -> Response { - if let Some(accepting) = accept { - if accepting.negotiate(AVAILABLE) == Some(&TEXT_HTML) { - match thing.render() { - Ok(content) => return Html(content).into_response(), - Err(e) => { - eprintln!("template rendering failed: {e:?}"); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } - } - } - } - Json(thing).into_response() -} diff --git a/constellation/src/server/filters.rs b/constellation/src/server/filters.rs deleted file mode 100644 index 9339a50..0000000 --- a/constellation/src/server/filters.rs +++ /dev/null @@ -1,24 +0,0 @@ -use links::{parse_any_link, Link}; -use num_format::{Locale, ToFormattedString}; - -pub fn to_browseable(s: &str) -> askama::Result> { - Ok({ - if let Some(link) = parse_any_link(s) { - match link { - Link::AtUri(at_uri) => at_uri.strip_prefix("at://").map(|noproto| { - format!("https://atproto-browser-plus-links.vercel.app/at/{noproto}") - }), - Link::Did(did) => Some(format!( - "https://atproto-browser-plus-links.vercel.app/at/{did}" - )), - Link::Uri(uri) => Some(uri), - } - } else { - None - } - }) -} - -pub fn human_number(n: &u64) -> askama::Result { - Ok(n.to_formatted_string(&Locale::en)) -} diff --git a/constellation/src/server/mod.rs b/constellation/src/server/mod.rs deleted file mode 100644 index 99ad859..0000000 --- a/constellation/src/server/mod.rs +++ /dev/null @@ -1,435 +0,0 @@ -use askama::Template; -use axum::{ - extract::{Query, Request}, - http::{self, header}, - middleware::{self, Next}, - response::{IntoResponse, Response}, - routing::get, - Router, -}; -use axum_metrics::{ExtraMetricLabels, MetricLayer}; -use bincode::Options; -use serde::{Deserialize, Serialize}; -use serde_with::serde_as; -use std::collections::HashMap; -use std::time::{Duration, UNIX_EPOCH}; -use tokio::net::{TcpListener, ToSocketAddrs}; -use tokio::task::block_in_place; -use tokio_util::sync::CancellationToken; - -use crate::storage::{LinkReader, StorageStats}; -use crate::{CountsByCount, Did, RecordId}; - -mod acceptable; -mod filters; - -use acceptable::{acceptable, ExtractAccept}; - -const DEFAULT_CURSOR_LIMIT: u64 = 16; -const DEFAULT_CURSOR_LIMIT_MAX: u64 = 100; - -const INDEX_BEGAN_AT_TS: u64 = 1738083600; // TODO: not this - -pub async fn serve(store: S, addr: A, stay_alive: CancellationToken) -> anyhow::Result<()> -where - S: LinkReader, - A: ToSocketAddrs, -{ - let app = Router::new() - .route("/robots.txt", get(robots)) - .route( - "/", - get({ - let store = store.clone(); - move |accept| async { block_in_place(|| hello(accept, store)) } - }), - ) - .route( - "/links/count", - get({ - let store = store.clone(); - move |accept, query| async { block_in_place(|| count_links(accept, query, store)) } - }), - ) - .route( - "/links/count/distinct-dids", - get({ - let store = store.clone(); - move |accept, query| async { - block_in_place(|| count_distinct_dids(accept, query, store)) - } - }), - ) - .route( - "/links", - get({ - let store = store.clone(); - move |accept, query| async { block_in_place(|| get_links(accept, query, store)) } - }), - ) - .route( - "/links/distinct-dids", - get({ - let store = store.clone(); - move |accept, query| async { - block_in_place(|| get_distinct_dids(accept, query, store)) - } - }), - ) - .route( - // deprecated - "/links/all/count", - get({ - let store = store.clone(); - move |accept, query| async { - block_in_place(|| count_all_links(accept, query, store)) - } - }), - ) - .route( - "/links/all", - get({ - let store = store.clone(); - move |accept, query| async { - block_in_place(|| explore_links(accept, query, store)) - } - }), - ) - .layer(tower_http::cors::CorsLayer::permissive()) - .layer(middleware::from_fn(add_lables)) - .layer(MetricLayer::default()); - - let listener = TcpListener::bind(addr).await?; - println!("api: listening at http://{:?}", listener.local_addr()?); - axum::serve(listener, app) - .with_graceful_shutdown(async move { stay_alive.cancelled().await }) - .await?; - - Ok(()) -} - -async fn add_lables(request: Request, next: Next) -> Response { - let origin = request - .headers() - .get(header::ORIGIN) - .and_then(|o| o.to_str().map(|v| v.to_owned()).ok()); - let user_agent = request.headers().get(header::USER_AGENT).and_then(|ua| { - ua.to_str() - .map(|v| { - if v.starts_with("Mozilla/") { - "Mozilla/...".into() - } else { - v.to_owned() - } - }) - .ok() - }); - - let mut res = next.run(request).await; - - let mut labels = Vec::new(); - if let Some(o) = origin { - labels.push(metrics::Label::new("origin", o)); - } - if let Some(ua) = user_agent { - labels.push(metrics::Label::new("user_agent", ua)); - } - res.extensions_mut().insert(ExtraMetricLabels(labels)); - res -} - -async fn robots() -> &'static str { - "\ -User-agent: * -Disallow: /links -Disallow: /links/ - " -} - -#[derive(Template, Serialize, Deserialize)] -#[template(path = "hello.html.j2")] -struct HelloReponse { - help: &'static str, - days_indexed: u64, - stats: StorageStats, -} -fn hello( - accept: ExtractAccept, - store: impl LinkReader, -) -> Result { - let stats = store - .get_stats() - .map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)?; - let days_indexed = (UNIX_EPOCH + Duration::from_secs(INDEX_BEGAN_AT_TS)) - .elapsed() - .map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)? - .as_secs() - / 86400; - Ok(acceptable(accept, HelloReponse { - help: "open this URL in a web browser (or request with Accept: text/html) for information about this API.", - days_indexed, - stats, - })) -} - -#[derive(Clone, Deserialize)] -struct GetLinksCountQuery { - target: String, - collection: String, - path: String, -} -#[derive(Template, Serialize)] -#[template(path = "links-count.html.j2")] -struct GetLinksCountResponse { - total: u64, - #[serde(skip_serializing)] - query: GetLinksCountQuery, -} -fn count_links( - accept: ExtractAccept, - query: Query, - store: impl LinkReader, -) -> Result { - let total = store - .get_count(&query.target, &query.collection, &query.path) - .map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)?; - Ok(acceptable( - accept, - GetLinksCountResponse { - total, - query: (*query).clone(), - }, - )) -} - -#[derive(Clone, Deserialize)] -struct GetDidsCountQuery { - target: String, - collection: String, - path: String, -} -#[derive(Template, Serialize)] -#[template(path = "dids-count.html.j2")] -struct GetDidsCountResponse { - total: u64, - #[serde(skip_serializing)] - query: GetDidsCountQuery, -} -fn count_distinct_dids( - accept: ExtractAccept, - query: Query, - store: impl LinkReader, -) -> Result { - let total = store - .get_distinct_did_count(&query.target, &query.collection, &query.path) - .map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)?; - Ok(acceptable( - accept, - GetDidsCountResponse { - total, - query: (*query).clone(), - }, - )) -} - -#[derive(Clone, Deserialize)] -struct GetLinkItemsQuery { - target: String, - collection: String, - path: String, - cursor: Option, - limit: Option, - // TODO: allow reverse (er, forward) order as well -} -#[derive(Template, Serialize)] -#[template(path = "links.html.j2")] -struct GetLinkItemsResponse { - // what does staleness mean? - // - new links have appeared. would be nice to offer a `since` cursor to fetch these. and/or, - // - links have been deleted. hmm. - total: u64, - linking_records: Vec, - cursor: Option, - #[serde(skip_serializing)] - query: GetLinkItemsQuery, -} -fn get_links( - accept: ExtractAccept, - query: Query, - store: impl LinkReader, -) -> Result { - let until = query - .cursor - .clone() - .map(|oc| ApiCursor::try_from(oc).map_err(|_| http::StatusCode::BAD_REQUEST)) - .transpose()? - .map(|c| c.next); - - let limit = query.limit.unwrap_or(DEFAULT_CURSOR_LIMIT); - if limit > DEFAULT_CURSOR_LIMIT_MAX { - return Err(http::StatusCode::BAD_REQUEST); - } - - let paged = store - .get_links(&query.target, &query.collection, &query.path, limit, until) - .map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)?; - - let cursor = paged.next.map(|next| { - ApiCursor { - version: paged.version, - next, - } - .into() - }); - - Ok(acceptable( - accept, - GetLinkItemsResponse { - total: paged.total, - linking_records: paged.items, - cursor, - query: (*query).clone(), - }, - )) -} - -#[derive(Clone, Deserialize)] -struct GetDidItemsQuery { - target: String, - collection: String, - path: String, - cursor: Option, - limit: Option, - // TODO: allow reverse (er, forward) order as well -} -#[derive(Template, Serialize)] -#[template(path = "dids.html.j2")] -struct GetDidItemsResponse { - // what does staleness mean? - // - new links have appeared. would be nice to offer a `since` cursor to fetch these. and/or, - // - links have been deleted. hmm. - total: u64, - linking_dids: Vec, - cursor: Option, - #[serde(skip_serializing)] - query: GetDidItemsQuery, -} -fn get_distinct_dids( - accept: ExtractAccept, - query: Query, - store: impl LinkReader, -) -> Result { - let until = query - .cursor - .clone() - .map(|oc| ApiCursor::try_from(oc).map_err(|_| http::StatusCode::BAD_REQUEST)) - .transpose()? - .map(|c| c.next); - - let limit = query.limit.unwrap_or(DEFAULT_CURSOR_LIMIT); - if limit > DEFAULT_CURSOR_LIMIT_MAX { - return Err(http::StatusCode::BAD_REQUEST); - } - - let paged = store - .get_distinct_dids(&query.target, &query.collection, &query.path, limit, until) - .map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)?; - - let cursor = paged.next.map(|next| { - ApiCursor { - version: paged.version, - next, - } - .into() - }); - - Ok(acceptable( - accept, - GetDidItemsResponse { - total: paged.total, - linking_dids: paged.items, - cursor, - query: (*query).clone(), - }, - )) -} - -#[derive(Clone, Deserialize)] -struct GetAllLinksQuery { - target: String, -} -#[derive(Template, Serialize)] -#[template(path = "links-all-count.html.j2")] -struct GetAllLinksResponse { - links: HashMap>, - #[serde(skip_serializing)] - query: GetAllLinksQuery, -} -fn count_all_links( - accept: ExtractAccept, - query: Query, - store: impl LinkReader, -) -> Result { - let links = store - .get_all_record_counts(&query.target) - .map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)?; - Ok(acceptable( - accept, - GetAllLinksResponse { - links, - query: (*query).clone(), - }, - )) -} - -#[derive(Clone, Deserialize)] -struct ExploreLinksQuery { - target: String, -} -#[derive(Template, Serialize)] -#[template(path = "explore-links.html.j2")] -struct ExploreLinksResponse { - links: HashMap>, - #[serde(skip_serializing)] - query: ExploreLinksQuery, -} -fn explore_links( - accept: ExtractAccept, - query: Query, - store: impl LinkReader, -) -> Result { - let links = store - .get_all_counts(&query.target) - .map_err(|_| http::StatusCode::INTERNAL_SERVER_ERROR)?; - Ok(acceptable( - accept, - ExploreLinksResponse { - links, - query: (*query).clone(), - }, - )) -} - -#[serde_as] -#[derive(Clone, Serialize, Deserialize)] // for json -struct OpaqueApiCursor(#[serde_as(as = "serde_with::hex::Hex")] Vec); - -#[derive(Serialize, Deserialize)] // for bincode -struct ApiCursor { - version: (u64, u64), // (collection length, deleted item count) - next: u64, -} - -impl TryFrom for ApiCursor { - type Error = bincode::Error; - - fn try_from(item: OpaqueApiCursor) -> Result { - bincode::DefaultOptions::new().deserialize(&item.0) - } -} - -impl From for OpaqueApiCursor { - fn from(item: ApiCursor) -> Self { - OpaqueApiCursor(bincode::DefaultOptions::new().serialize(&item).unwrap()) - } -} diff --git a/constellation/src/storage/mem_store.rs b/constellation/src/storage/mem_store.rs deleted file mode 100644 index 2b7281a..0000000 --- a/constellation/src/storage/mem_store.rs +++ /dev/null @@ -1,394 +0,0 @@ -use super::{LinkReader, LinkStorage, PagedAppendingCollection, StorageStats}; -use crate::{ActionableEvent, CountsByCount, Did, RecordId}; -use anyhow::Result; -use links::CollectedLink; -use std::collections::{HashMap, HashSet}; -use std::sync::{Arc, Mutex}; - -// hopefully-correct simple hashmap version, intended only for tests to verify disk impl -#[derive(Debug, Clone)] -pub struct MemStorage(Arc>); - -type Linkers = Vec>; // optional because we replace with None for deleted links to keep cursors stable - -#[derive(Debug, Default)] -struct MemStorageData { - dids: HashMap, // bool: active or nah - targets: HashMap>, // target -> (collection, path) -> (did, rkey)?[] - links: HashMap>>, // did -> collection:rkey -> (path, target)[] -} - -impl MemStorage { - pub fn new() -> Self { - Self(Arc::new(Mutex::new(MemStorageData::default()))) - } - - fn add_links(&mut self, record_id: &RecordId, links: &[CollectedLink]) { - let mut data = self.0.lock().unwrap(); - for link in links { - data.dids.entry(record_id.did()).or_insert(true); // if they are inserting a link, presumably they are active - data.targets - .entry(Target::new(link.target.as_str())) - .or_default() - .entry(Source::new(&record_id.collection, &link.path)) - .or_default() - .push(Some((record_id.did(), RKey(record_id.rkey())))); - data.links - .entry(record_id.did()) - .or_default() - .entry(RepoId::from_record_id(record_id)) - .or_insert(Vec::with_capacity(1)) - .push(( - RecordPath::new(&link.path), - Target::new(link.target.as_str()), - )) - } - } - - fn remove_links(&mut self, record_id: &RecordId) { - let mut data = self.0.lock().unwrap(); - let repo_id = RepoId::from_record_id(record_id); - if let Some(Some(link_targets)) = data.links.get(&record_id.did).map(|cr| cr.get(&repo_id)) - { - let link_targets = link_targets.clone(); // satisfy borrowck - for (record_path, target) in link_targets { - data.targets - .get_mut(&target) - .expect("must have the target if we have a link saved") - .get_mut(&Source::new(&record_id.collection, &record_path.0)) - .expect("must have the target at this path if we have a link to it saved") - .iter_mut() - .rfind(|d| **d == Some((record_id.did(), RKey(record_id.rkey())))) - .expect("must be in dids list if we have a link to it") - .take(); - } - } - data.links - .get_mut(&record_id.did) - .map(|cr| cr.remove(&repo_id)); - } - - fn update_links(&mut self, record_id: &RecordId, new_links: &[CollectedLink]) { - self.remove_links(record_id); - self.add_links(record_id, new_links); - } - - fn set_account(&mut self, did: &Did, active: bool) { - let mut data = self.0.lock().unwrap(); - if let Some(account) = data.dids.get_mut(did) { - *account = active; - } - } - - fn delete_account(&mut self, did: &Did) { - let mut data = self.0.lock().unwrap(); - if let Some(links) = data.links.get(did) { - let links = links.clone(); - for (repo_id, targets) in links { - let targets = targets.clone(); - for (record_path, target) in targets { - data.targets - .get_mut(&target) - .expect("must have the target if we have a link saved") - .get_mut(&Source::new(&repo_id.collection, &record_path.0)) - .expect("must have the target at this path if we have a link to it saved") - .iter_mut() - .find(|d| **d == Some((did.clone(), repo_id.rkey.clone()))) - .expect("lkasjdlfkj") - .take(); - } - } - } - data.links.remove(did); // nb: this is removing by a whole prefix in kv context - data.dids.remove(did); - } -} - -impl Default for MemStorage { - fn default() -> Self { - Self::new() - } -} - -impl LinkStorage for MemStorage { - fn push(&mut self, event: &ActionableEvent, _cursor: u64) -> Result<()> { - match event { - ActionableEvent::CreateLinks { record_id, links } => self.add_links(record_id, links), - ActionableEvent::UpdateLinks { - record_id, - new_links, - } => self.update_links(record_id, new_links), - ActionableEvent::DeleteRecord(record_id) => self.remove_links(record_id), - ActionableEvent::ActivateAccount(did) => self.set_account(did, true), - ActionableEvent::DeactivateAccount(did) => self.set_account(did, false), - ActionableEvent::DeleteAccount(did) => self.delete_account(did), - } - Ok(()) - } - - fn to_readable(&mut self) -> impl LinkReader { - self.clone() - } -} - -impl LinkReader for MemStorage { - fn get_count(&self, target: &str, collection: &str, path: &str) -> Result { - let data = self.0.lock().unwrap(); - let Some(paths) = data.targets.get(&Target::new(target)) else { - return Ok(0); - }; - let Some(linkers) = paths.get(&Source::new(collection, path)) else { - return Ok(0); - }; - Ok(linkers.iter().flatten().count() as u64) - } - - fn get_distinct_did_count(&self, target: &str, collection: &str, path: &str) -> Result { - let data = self.0.lock().unwrap(); - let Some(paths) = data.targets.get(&Target::new(target)) else { - return Ok(0); - }; - let Some(linkers) = paths.get(&Source::new(collection, path)) else { - return Ok(0); - }; - Ok(linkers - .iter() - .flatten() - .map(|(did, _)| did) - .collect::>() - .len() as u64) - } - - fn get_links( - &self, - target: &str, - collection: &str, - path: &str, - limit: u64, - until: Option, - ) -> Result> { - let data = self.0.lock().unwrap(); - let Some(paths) = data.targets.get(&Target::new(target)) else { - return Ok(PagedAppendingCollection { - version: (0, 0), - items: Vec::new(), - next: None, - total: 0, - }); - }; - let Some(did_rkeys) = paths.get(&Source::new(collection, path)) else { - return Ok(PagedAppendingCollection { - version: (0, 0), - items: Vec::new(), - next: None, - total: 0, - }); - }; - - let total = did_rkeys.len(); - let end = until - .map(|u| std::cmp::min(u as usize, total)) - .unwrap_or(total); - let begin = end.saturating_sub(limit as usize); - let next = if begin == 0 { None } else { Some(begin as u64) }; - - let alive = did_rkeys.iter().flatten().count(); - let gone = total - alive; - - let items: Vec<_> = did_rkeys[begin..end] - .iter() - .rev() - .flatten() - .filter(|(did, _)| *data.dids.get(did).expect("did must be in dids")) - .map(|(did, rkey)| RecordId { - did: did.clone(), - rkey: rkey.0.clone(), - collection: collection.to_string(), - }) - .collect(); - - Ok(PagedAppendingCollection { - version: (total as u64, gone as u64), - items, - next, - total: alive as u64, - }) - } - - fn get_distinct_dids( - &self, - target: &str, - collection: &str, - path: &str, - limit: u64, - until: Option, - ) -> Result> { - let data = self.0.lock().unwrap(); - let Some(paths) = data.targets.get(&Target::new(target)) else { - return Ok(PagedAppendingCollection { - version: (0, 0), - items: Vec::new(), - next: None, - total: 0, - }); - }; - let Some(did_rkeys) = paths.get(&Source::new(collection, path)) else { - return Ok(PagedAppendingCollection { - version: (0, 0), - items: Vec::new(), - next: None, - total: 0, - }); - }; - - let dids: Vec> = { - let mut seen = HashSet::new(); - did_rkeys - .iter() - .map(|o| { - o.clone().and_then(|(did, _)| { - if seen.contains(&did) { - None - } else { - seen.insert(did.clone()); - Some(did) - } - }) - }) - .collect() - }; - - let total = dids.len(); - let end = until - .map(|u| std::cmp::min(u as usize, total)) - .unwrap_or(total); - let begin = end.saturating_sub(limit as usize); - let next = if begin == 0 { None } else { Some(begin as u64) }; - - let alive = dids.iter().flatten().count(); - let gone = total - alive; - - let items: Vec = dids[begin..end] - .iter() - .rev() - .flatten() - .filter(|did| *data.dids.get(did).expect("did must be in dids")) - .cloned() - .collect(); - - Ok(PagedAppendingCollection { - version: (total as u64, gone as u64), - items, - next, - total: alive as u64, - }) - } - - fn get_all_record_counts(&self, target: &str) -> Result>> { - let data = self.0.lock().unwrap(); - let mut out: HashMap> = HashMap::new(); - if let Some(asdf) = data.targets.get(&Target::new(target)) { - for (Source { collection, path }, linkers) in asdf { - let count = linkers.iter().flatten().count() as u64; - out.entry(collection.to_string()) - .or_default() - .insert(path.to_string(), count); - } - } - Ok(out) - } - - fn get_all_counts( - &self, - target: &str, - ) -> Result>> { - let data = self.0.lock().unwrap(); - let mut out: HashMap> = HashMap::new(); - if let Some(asdf) = data.targets.get(&Target::new(target)) { - for (Source { collection, path }, linkers) in asdf { - let records = linkers.iter().flatten().count() as u64; - let distinct_dids = linkers - .iter() - .flatten() - .map(|(did, _)| did) - .collect::>() - .len() as u64; - out.entry(collection.to_string()).or_default().insert( - path.to_string(), - CountsByCount { - records, - distinct_dids, - }, - ); - } - } - Ok(out) - } - - fn get_stats(&self) -> Result { - let data = self.0.lock().unwrap(); - let dids = data.dids.len() as u64; - let targetables = data - .targets - .values() - .map(|sources| sources.len()) - .sum::() as u64; - let linking_records = data.links.values().map(|recs| recs.len()).sum::() as u64; - Ok(StorageStats { - dids, - targetables, - linking_records, - }) - } -} - -#[derive(Debug, PartialEq, Hash, Eq, Clone)] -struct Target(String); - -impl Target { - fn new(t: &str) -> Self { - Self(t.into()) - } -} - -#[derive(Debug, PartialEq, Hash, Eq, Clone)] -struct Source { - collection: String, - path: String, -} - -impl Source { - fn new(collection: &str, path: &str) -> Self { - Self { - collection: collection.into(), - path: path.into(), - } - } -} - -#[derive(Debug, PartialEq, Hash, Eq, Clone)] -struct RKey(String); - -#[derive(Debug, PartialEq, Hash, Eq, Clone)] -struct RepoId { - collection: String, - rkey: RKey, -} - -impl RepoId { - fn from_record_id(record_id: &RecordId) -> Self { - Self { - collection: record_id.collection.clone(), - rkey: RKey(record_id.rkey.clone()), - } - } -} - -#[derive(Debug, PartialEq, Hash, Eq, Clone)] -struct RecordPath(String); - -impl RecordPath { - fn new(rp: &str) -> Self { - Self(rp.into()) - } -} diff --git a/constellation/src/storage/mod.rs b/constellation/src/storage/mod.rs deleted file mode 100644 index 332099a..0000000 --- a/constellation/src/storage/mod.rs +++ /dev/null @@ -1,1085 +0,0 @@ -use crate::{ActionableEvent, CountsByCount, Did, RecordId}; -use anyhow::Result; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; - -pub mod mem_store; -pub use mem_store::MemStorage; - -#[cfg(feature = "rocks")] -pub mod rocks_store; -#[cfg(feature = "rocks")] -pub use rocks_store::RocksStorage; - -#[derive(Debug, PartialEq)] -pub struct PagedAppendingCollection { - pub version: (u64, u64), // (collection length, deleted item count) // TODO: change to (total, active)? since dedups isn't "deleted" - pub items: Vec, - pub next: Option, - pub total: u64, -} - -#[derive(Debug, Deserialize, Serialize, PartialEq)] -pub struct StorageStats { - /// estimate of how many accounts we've seen create links. the _subjects_ of any links are not represented here. - /// for example: new user A follows users B and C. this count will only increment by one, for A. - pub dids: u64, - - /// estimate targets * distinct (collection, path)s to reference them. - /// distinct targets alone are currently challenging to estimate. - pub targetables: u64, - - /// estimate of the count of atproto records seen that contain links. - /// records with multiple links are single-counted. - /// for LSM stores, deleted links don't decrement this, and updated records with any links will likely increment it. - pub linking_records: u64, -} - -pub trait LinkStorage: Send + Sync { - /// jetstream cursor from last saved actions, if available - fn get_cursor(&mut self) -> Result> { - Ok(None) - } - - fn push(&mut self, event: &ActionableEvent, cursor: u64) -> Result<()>; - - // readers are off from the writer instance - fn to_readable(&mut self) -> impl LinkReader; -} - -pub trait LinkReader: Clone + Send + Sync + 'static { - fn get_count(&self, target: &str, collection: &str, path: &str) -> Result; - - fn get_distinct_did_count(&self, target: &str, collection: &str, path: &str) -> Result; - - fn get_links( - &self, - target: &str, - collection: &str, - path: &str, - limit: u64, - until: Option, - ) -> Result>; - - fn get_distinct_dids( - &self, - target: &str, - collection: &str, - path: &str, - limit: u64, - until: Option, - ) -> Result>; // TODO: reflect dedups in cursor - - fn get_all_record_counts(&self, _target: &str) - -> Result>>; - - fn get_all_counts( - &self, - _target: &str, - ) -> Result>>; - - /// assume all stats are estimates, since exact counts are very challenging for LSMs - fn get_stats(&self) -> Result; -} - -#[cfg(test)] -mod tests { - use super::*; - use links::{CollectedLink, Link}; - use std::ops::RangeBounds; - - macro_rules! test_each_storage { - ($test_name:ident, |$storage_label:ident| $test_code:block) => { - #[test] - fn $test_name() -> Result<()> { - { - println!("=> testing with memstorage backend"); - #[allow(unused_mut)] - let mut $storage_label = MemStorage::new(); - $test_code - } - - #[cfg(feature = "rocks")] - { - println!("=> testing with rocksdb backend"); - let rocks_db_path = tempfile::tempdir()?; - #[allow(unused_mut)] - let mut $storage_label = RocksStorage::new(rocks_db_path.path())?; - $test_code - } - - Ok(()) - } - }; - } - - fn assert_stats( - stats: StorageStats, - dids: impl RangeBounds, - targetables: impl RangeBounds, - linking_records: impl RangeBounds, - ) { - fn check(name: &str, stat: u64, rb: impl RangeBounds) { - assert!( - rb.contains(&stat), - "{name:?}: {stat:?} not in range {:?}–{:?}", - rb.start_bound(), - rb.end_bound() - ); - } - check("dids", stats.dids, dids); - check("targetables", stats.targetables, targetables); - check("linking_records", stats.linking_records, linking_records); - } - - test_each_storage!(test_empty, |storage| { - assert_eq!(storage.get_count("", "", "")?, 0); - assert_eq!(storage.get_count("a", "b", "c")?, 0); - assert_eq!( - storage.get_count( - "at://did:plc:b3rzzkblqsxhr3dgcueymkqe/app.bsky.feed.post/3lf6yc4drhk2f", - "app.t.c", - ".reply.parent.uri" - )?, - 0 - ); - assert_eq!(storage.get_distinct_did_count("", "", "")?, 0); - assert_eq!( - storage.get_links("a.com", "app.t.c", ".abc.uri", 100, None)?, - PagedAppendingCollection { - version: (0, 0), - items: vec![], - next: None, - total: 0, - } - ); - assert_eq!( - storage.get_distinct_dids("a.com", "app.t.c", ".abc.uri", 100, None)?, - PagedAppendingCollection { - version: (0, 0), - items: vec![], - next: None, - total: 0, - } - ); - assert_eq!(storage.get_all_counts("bad-example.com")?, HashMap::new()); - assert_eq!( - storage.get_all_record_counts("bad-example.com")?, - HashMap::new() - ); - - assert_stats(storage.get_stats()?, 0..=0, 0..=0, 0..=0); - }); - - test_each_storage!(test_add_link, |storage| { - storage.push( - &ActionableEvent::CreateLinks { - record_id: RecordId { - did: "did:plc:asdf".into(), - collection: "app.t.c".into(), - rkey: "fdsa".into(), - }, - links: vec![CollectedLink { - target: Link::Uri("e.com".into()), - path: ".abc.uri".into(), - }], - }, - 0, - )?; - assert_eq!(storage.get_count("e.com", "app.t.c", ".abc.uri")?, 1); - assert_eq!( - storage.get_distinct_did_count("e.com", "app.t.c", ".abc.uri")?, - 1 - ); - assert_eq!(storage.get_count("bad.com", "app.t.c", ".abc.uri")?, 0); - assert_eq!(storage.get_count("e.com", "app.t.c", ".bad.uri")?, 0); - assert_eq!( - storage.get_distinct_did_count("e.com", "app.t.c", ".bad.uri")?, - 0 - ); - assert_stats(storage.get_stats()?, 1..=1, 1..=1, 1..=1); - }); - - test_each_storage!(test_links, |storage| { - storage.push( - &ActionableEvent::CreateLinks { - record_id: RecordId { - did: "did:plc:asdf".into(), - collection: "app.t.c".into(), - rkey: "fdsa".into(), - }, - links: vec![CollectedLink { - target: Link::Uri("e.com".into()), - path: ".abc.uri".into(), - }], - }, - 0, - )?; - - // delete under the wrong collection - storage.push( - &ActionableEvent::DeleteRecord(RecordId { - did: "did:plc:asdf".into(), - collection: "app.test.wrongcollection".into(), - rkey: "fdsa".into(), - }), - 0, - )?; - assert_eq!(storage.get_count("e.com", "app.t.c", ".abc.uri")?, 1); - - // delete under the wrong rkey - storage.push( - &ActionableEvent::DeleteRecord(RecordId { - did: "did:plc:asdf".into(), - collection: "app.t.c".into(), - rkey: "wrongkey".into(), - }), - 0, - )?; - assert_eq!(storage.get_count("e.com", "app.t.c", ".abc.uri")?, 1); - - // finally actually delete it - storage.push( - &ActionableEvent::DeleteRecord(RecordId { - did: "did:plc:asdf".into(), - collection: "app.t.c".into(), - rkey: "fdsa".into(), - }), - 0, - )?; - assert_eq!(storage.get_count("e.com", "app.t.c", ".abc.uri")?, 0); - - // put it back - storage.push( - &ActionableEvent::CreateLinks { - record_id: RecordId { - did: "did:plc:asdf".into(), - collection: "app.t.c".into(), - rkey: "fdsa".into(), - }, - links: vec![CollectedLink { - target: Link::Uri("e.com".into()), - path: ".abc.uri".into(), - }], - }, - 0, - )?; - assert_eq!(storage.get_count("e.com", "app.t.c", ".abc.uri")?, 1); - assert_eq!( - storage.get_distinct_did_count("e.com", "app.t.c", ".abc.uri")?, - 1 - ); - - // add another link from this user - storage.push( - &ActionableEvent::CreateLinks { - record_id: RecordId { - did: "did:plc:asdf".into(), - collection: "app.t.c".into(), - rkey: "fdsa2".into(), - }, - links: vec![CollectedLink { - target: Link::Uri("e.com".into()), - path: ".abc.uri".into(), - }], - }, - 0, - )?; - assert_eq!(storage.get_count("e.com", "app.t.c", ".abc.uri")?, 2); - assert_eq!( - storage.get_distinct_did_count("e.com", "app.t.c", ".abc.uri")?, - 1 - ); - - // add a link from someone else - storage.push( - &ActionableEvent::CreateLinks { - record_id: RecordId { - did: "did:plc:asdfasdf".into(), - collection: "app.t.c".into(), - rkey: "fdsa".into(), - }, - links: vec![CollectedLink { - target: Link::Uri("e.com".into()), - path: ".abc.uri".into(), - }], - }, - 0, - )?; - assert_eq!(storage.get_count("e.com", "app.t.c", ".abc.uri")?, 3); - assert_eq!( - storage.get_distinct_did_count("e.com", "app.t.c", ".abc.uri")?, - 2 - ); - - // aaaand delete the first one again - storage.push( - &ActionableEvent::DeleteRecord(RecordId { - did: "did:plc:asdf".into(), - collection: "app.t.c".into(), - rkey: "fdsa".into(), - }), - 0, - )?; - assert_eq!(storage.get_count("e.com", "app.t.c", ".abc.uri")?, 2); - assert_eq!( - storage.get_distinct_did_count("e.com", "app.t.c", ".abc.uri")?, - 2 - ); - assert_stats(storage.get_stats()?, 2..=2, 1..=1, 2..=2); - }); - - test_each_storage!(test_two_user_links_delete_one, |storage| { - // create the first link - storage.push( - &ActionableEvent::CreateLinks { - record_id: RecordId { - did: "did:plc:asdf".into(), - collection: "app.t.c".into(), - rkey: "A".into(), - }, - links: vec![CollectedLink { - target: Link::Uri("e.com".into()), - path: ".abc.uri".into(), - }], - }, - 0, - )?; - assert_eq!(storage.get_count("e.com", "app.t.c", ".abc.uri")?, 1); - assert_eq!( - storage.get_distinct_did_count("e.com", "app.t.c", ".abc.uri")?, - 1 - ); - - // create the second link (same user, different rkey) - storage.push( - &ActionableEvent::CreateLinks { - record_id: RecordId { - did: "did:plc:asdf".into(), - collection: "app.t.c".into(), - rkey: "B".into(), - }, - links: vec![CollectedLink { - target: Link::Uri("e.com".into()), - path: ".abc.uri".into(), - }], - }, - 0, - )?; - assert_eq!(storage.get_count("e.com", "app.t.c", ".abc.uri")?, 2); - assert_eq!( - storage.get_distinct_did_count("e.com", "app.t.c", ".abc.uri")?, - 1 - ); - - // aaaand delete the first link - storage.push( - &ActionableEvent::DeleteRecord(RecordId { - did: "did:plc:asdf".into(), - collection: "app.t.c".into(), - rkey: "A".into(), - }), - 0, - )?; - assert_eq!(storage.get_count("e.com", "app.t.c", ".abc.uri")?, 1); - assert_eq!( - storage.get_distinct_did_count("e.com", "app.t.c", ".abc.uri")?, - 1 - ); - assert_stats(storage.get_stats()?, 1..=1, 1..=1, 1..=1); - }); - - test_each_storage!(test_accounts, |storage| { - // create two links - storage.push( - &ActionableEvent::CreateLinks { - record_id: RecordId { - did: "did:plc:asdf".into(), - collection: "app.t.c".into(), - rkey: "A".into(), - }, - links: vec![CollectedLink { - target: Link::Uri("a.com".into()), - path: ".abc.uri".into(), - }], - }, - 0, - )?; - storage.push( - &ActionableEvent::CreateLinks { - record_id: RecordId { - did: "did:plc:asdf".into(), - collection: "app.t.c".into(), - rkey: "B".into(), - }, - links: vec![CollectedLink { - target: Link::Uri("b.com".into()), - path: ".abc.uri".into(), - }], - }, - 0, - )?; - assert_eq!(storage.get_count("a.com", "app.t.c", ".abc.uri")?, 1); - assert_eq!(storage.get_count("b.com", "app.t.c", ".abc.uri")?, 1); - - // and a third from a different account - storage.push( - &ActionableEvent::CreateLinks { - record_id: RecordId { - did: "did:plc:fdsa".into(), - collection: "app.t.c".into(), - rkey: "A".into(), - }, - links: vec![CollectedLink { - target: Link::Uri("a.com".into()), - path: ".abc.uri".into(), - }], - }, - 0, - )?; - assert_eq!(storage.get_count("a.com", "app.t.c", ".abc.uri")?, 2); - - // delete the first account - storage.push(&ActionableEvent::DeleteAccount("did:plc:asdf".into()), 0)?; - assert_eq!(storage.get_count("a.com", "app.t.c", ".abc.uri")?, 1); - assert_eq!(storage.get_count("b.com", "app.t.c", ".abc.uri")?, 0); - assert_stats(storage.get_stats()?, 1..=2, 2..=2, 1..=1); - }); - - test_each_storage!(multi_link, |storage| { - storage.push( - &ActionableEvent::CreateLinks { - record_id: RecordId { - did: "did:plc:asdf".into(), - collection: "app.t.c".into(), - rkey: "fdsa".into(), - }, - links: vec![ - CollectedLink { - target: Link::Uri("e.com".into()), - path: ".abc.uri".into(), - }, - CollectedLink { - target: Link::Uri("f.com".into()), - path: ".xyz[].uri".into(), - }, - CollectedLink { - target: Link::Uri("g.com".into()), - path: ".xyz[].uri".into(), - }, - ], - }, - 0, - )?; - assert_eq!(storage.get_count("e.com", "app.t.c", ".abc.uri")?, 1); - assert_eq!( - storage.get_distinct_did_count("e.com", "app.t.c", ".abc.uri")?, - 1 - ); - assert_eq!(storage.get_count("f.com", "app.t.c", ".xyz[].uri")?, 1); - assert_eq!( - storage.get_distinct_did_count("f.com", "app.t.c", ".xyz[].uri")?, - 1 - ); - assert_eq!(storage.get_count("g.com", "app.t.c", ".xyz[].uri")?, 1); - assert_eq!( - storage.get_distinct_did_count("g.com", "app.t.c", ".xyz[].uri")?, - 1 - ); - - storage.push( - &ActionableEvent::DeleteRecord(RecordId { - did: "did:plc:asdf".into(), - collection: "app.t.c".into(), - rkey: "fdsa".into(), - }), - 0, - )?; - assert_eq!(storage.get_count("e.com", "app.t.c", ".abc.uri")?, 0); - assert_eq!( - storage.get_distinct_did_count("e.com", "app.t.c", ".abc.uri")?, - 0 - ); - assert_eq!(storage.get_count("f.com", "app.t.c", ".xyz[].uri")?, 0); - assert_eq!(storage.get_count("g.com", "app.t.c", ".xyz[].uri")?, 0); - assert_stats(storage.get_stats()?, 1..=1, 3..=3, 0..=0); - }); - - test_each_storage!(update_link, |storage| { - // create the links - storage.push( - &ActionableEvent::CreateLinks { - record_id: RecordId { - did: "did:plc:asdf".into(), - collection: "app.t.c".into(), - rkey: "fdsa".into(), - }, - links: vec![ - CollectedLink { - target: Link::Uri("e.com".into()), - path: ".abc.uri".into(), - }, - CollectedLink { - target: Link::Uri("f.com".into()), - path: ".xyz[].uri".into(), - }, - CollectedLink { - target: Link::Uri("g.com".into()), - path: ".xyz[].uri".into(), - }, - ], - }, - 0, - )?; - assert_eq!(storage.get_count("e.com", "app.t.c", ".abc.uri")?, 1); - assert_eq!(storage.get_count("f.com", "app.t.c", ".xyz[].uri")?, 1); - assert_eq!(storage.get_count("g.com", "app.t.c", ".xyz[].uri")?, 1); - - // update them - storage.push( - &ActionableEvent::UpdateLinks { - record_id: RecordId { - did: "did:plc:asdf".into(), - collection: "app.t.c".into(), - rkey: "fdsa".into(), - }, - new_links: vec![ - CollectedLink { - target: Link::Uri("h.com".into()), - path: ".abc.uri".into(), - }, - CollectedLink { - target: Link::Uri("f.com".into()), - path: ".xyz[].uri".into(), - }, - CollectedLink { - target: Link::Uri("i.com".into()), - path: ".xyz[].uri".into(), - }, - ], - }, - 0, - )?; - assert_eq!(storage.get_count("e.com", "app.t.c", ".abc.uri")?, 0); - assert_eq!(storage.get_count("h.com", "app.t.c", ".abc.uri")?, 1); - assert_eq!(storage.get_count("f.com", "app.t.c", ".xyz[].uri")?, 1); - assert_eq!(storage.get_count("g.com", "app.t.c", ".xyz[].uri")?, 0); - assert_eq!(storage.get_count("i.com", "app.t.c", ".xyz[].uri")?, 1); - assert_stats(storage.get_stats()?, 1..=1, 5..=5, 1..=1); - }); - - test_each_storage!(update_no_links_to_links, |storage| { - // update without prior create (consumer would have filtered out the original) - storage.push( - &ActionableEvent::UpdateLinks { - record_id: RecordId { - did: "did:plc:asdf".into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - }, - new_links: vec![CollectedLink { - target: Link::Uri("a.com".into()), - path: ".abc.uri".into(), - }], - }, - 0, - )?; - assert_eq!(storage.get_count("a.com", "app.t.c", ".abc.uri")?, 1); - assert_stats(storage.get_stats()?, 1..=1, 1..=1, 1..=1); - }); - - test_each_storage!(delete_multi_link_same_target, |storage| { - storage.push( - &ActionableEvent::CreateLinks { - record_id: RecordId { - did: "did:plc:asdf".into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - }, - links: vec![ - CollectedLink { - target: Link::Uri("a.com".into()), - path: ".abc.uri".into(), - }, - CollectedLink { - target: Link::Uri("a.com".into()), - path: ".def.uri".into(), - }, - ], - }, - 0, - )?; - assert_eq!(storage.get_count("a.com", "app.t.c", ".abc.uri")?, 1); - assert_eq!(storage.get_count("a.com", "app.t.c", ".def.uri")?, 1); - - storage.push( - &ActionableEvent::DeleteRecord(RecordId { - did: "did:plc:asdf".into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - }), - 0, - )?; - assert_eq!(storage.get_count("a.com", "app.t.c", ".abc.uri")?, 0); - assert_eq!(storage.get_count("a.com", "app.t.c", ".def.uri")?, 0); - assert_stats(storage.get_stats()?, 1..=1, 2..=2, 0..=0); - }); - - test_each_storage!(get_links_basic, |storage| { - storage.push( - &ActionableEvent::CreateLinks { - record_id: RecordId { - did: "did:plc:asdf".into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - }, - links: vec![CollectedLink { - target: Link::Uri("a.com".into()), - path: ".abc.uri".into(), - }], - }, - 0, - )?; - assert_eq!( - storage.get_links("a.com", "app.t.c", ".abc.uri", 100, None)?, - PagedAppendingCollection { - version: (1, 0), - items: vec![RecordId { - did: "did:plc:asdf".into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - }], - next: None, - total: 1, - } - ); - assert_eq!( - storage.get_distinct_dids("a.com", "app.t.c", ".abc.uri", 100, None)?, - PagedAppendingCollection { - version: (1, 0), - items: vec!["did:plc:asdf".into()], - next: None, - total: 1, - } - ); - assert_stats(storage.get_stats()?, 1..=1, 1..=1, 1..=1); - }); - - test_each_storage!(get_links_paged, |storage| { - for i in 1..=5 { - storage.push( - &ActionableEvent::CreateLinks { - record_id: RecordId { - did: format!("did:plc:asdf-{i}").into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - }, - links: vec![CollectedLink { - target: Link::Uri("a.com".into()), - path: ".abc.uri".into(), - }], - }, - 0, - )?; - } - let links = storage.get_links("a.com", "app.t.c", ".abc.uri", 2, None)?; - let dids = storage.get_distinct_dids("a.com", "app.t.c", ".abc.uri", 2, None)?; - assert_eq!( - links, - PagedAppendingCollection { - version: (5, 0), - items: vec![ - RecordId { - did: "did:plc:asdf-5".into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - }, - RecordId { - did: "did:plc:asdf-4".into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - }, - ], - next: Some(3), - total: 5, - } - ); - assert_eq!( - dids, - PagedAppendingCollection { - version: (5, 0), - items: vec!["did:plc:asdf-5".into(), "did:plc:asdf-4".into()], - next: Some(3), - total: 5, - } - ); - let links = storage.get_links("a.com", "app.t.c", ".abc.uri", 2, links.next)?; - let dids = storage.get_distinct_dids("a.com", "app.t.c", ".abc.uri", 2, dids.next)?; - assert_eq!( - links, - PagedAppendingCollection { - version: (5, 0), - items: vec![ - RecordId { - did: "did:plc:asdf-3".into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - }, - RecordId { - did: "did:plc:asdf-2".into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - }, - ], - next: Some(1), - total: 5, - } - ); - assert_eq!( - dids, - PagedAppendingCollection { - version: (5, 0), - items: vec!["did:plc:asdf-3".into(), "did:plc:asdf-2".into()], - next: Some(1), - total: 5, - } - ); - let links = storage.get_links("a.com", "app.t.c", ".abc.uri", 2, links.next)?; - let dids = storage.get_distinct_dids("a.com", "app.t.c", ".abc.uri", 2, dids.next)?; - assert_eq!( - links, - PagedAppendingCollection { - version: (5, 0), - items: vec![RecordId { - did: "did:plc:asdf-1".into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - },], - next: None, - total: 5, - } - ); - assert_eq!( - dids, - PagedAppendingCollection { - version: (5, 0), - items: vec!["did:plc:asdf-1".into()], - next: None, - total: 5, - } - ); - assert_stats(storage.get_stats()?, 5..=5, 1..=1, 5..=5); - }); - - test_each_storage!(get_links_exact_multiple, |storage| { - for i in 1..=4 { - storage.push( - &ActionableEvent::CreateLinks { - record_id: RecordId { - did: format!("did:plc:asdf-{i}").into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - }, - links: vec![CollectedLink { - target: Link::Uri("a.com".into()), - path: ".abc.uri".into(), - }], - }, - 0, - )?; - } - let links = storage.get_links("a.com", "app.t.c", ".abc.uri", 2, None)?; - assert_eq!( - links, - PagedAppendingCollection { - version: (4, 0), - items: vec![ - RecordId { - did: "did:plc:asdf-4".into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - }, - RecordId { - did: "did:plc:asdf-3".into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - }, - ], - next: Some(2), - total: 4, - } - ); - let links = storage.get_links("a.com", "app.t.c", ".abc.uri", 2, links.next)?; - assert_eq!( - links, - PagedAppendingCollection { - version: (4, 0), - items: vec![ - RecordId { - did: "did:plc:asdf-2".into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - }, - RecordId { - did: "did:plc:asdf-1".into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - }, - ], - next: None, - total: 4, - } - ); - assert_stats(storage.get_stats()?, 4..=4, 1..=1, 4..=4); - }); - - test_each_storage!(page_links_while_new_links_arrive, |storage| { - for i in 1..=4 { - storage.push( - &ActionableEvent::CreateLinks { - record_id: RecordId { - did: format!("did:plc:asdf-{i}").into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - }, - links: vec![CollectedLink { - target: Link::Uri("a.com".into()), - path: ".abc.uri".into(), - }], - }, - 0, - )?; - } - let links = storage.get_links("a.com", "app.t.c", ".abc.uri", 2, None)?; - assert_eq!( - links, - PagedAppendingCollection { - version: (4, 0), - items: vec![ - RecordId { - did: "did:plc:asdf-4".into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - }, - RecordId { - did: "did:plc:asdf-3".into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - }, - ], - next: Some(2), - total: 4, - } - ); - storage.push( - &ActionableEvent::CreateLinks { - record_id: RecordId { - did: "did:plc:asdf-5".into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - }, - links: vec![CollectedLink { - target: Link::Uri("a.com".into()), - path: ".abc.uri".into(), - }], - }, - 0, - )?; - let links = storage.get_links("a.com", "app.t.c", ".abc.uri", 2, links.next)?; - assert_eq!( - links, - PagedAppendingCollection { - version: (5, 0), - items: vec![ - RecordId { - did: "did:plc:asdf-2".into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - }, - RecordId { - did: "did:plc:asdf-1".into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - }, - ], - next: None, - total: 5, - } - ); - assert_stats(storage.get_stats()?, 5..=5, 1..=1, 5..=5); - }); - - test_each_storage!(page_links_while_some_are_deleted, |storage| { - for i in 1..=4 { - storage.push( - &ActionableEvent::CreateLinks { - record_id: RecordId { - did: format!("did:plc:asdf-{i}").into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - }, - links: vec![CollectedLink { - target: Link::Uri("a.com".into()), - path: ".abc.uri".into(), - }], - }, - 0, - )?; - } - let links = storage.get_links("a.com", "app.t.c", ".abc.uri", 2, None)?; - assert_eq!( - links, - PagedAppendingCollection { - version: (4, 0), - items: vec![ - RecordId { - did: "did:plc:asdf-4".into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - }, - RecordId { - did: "did:plc:asdf-3".into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - }, - ], - next: Some(2), - total: 4, - } - ); - storage.push( - &ActionableEvent::DeleteRecord(RecordId { - did: "did:plc:asdf-2".into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - }), - 0, - )?; - let links = storage.get_links("a.com", "app.t.c", ".abc.uri", 2, links.next)?; - assert_eq!( - links, - PagedAppendingCollection { - version: (4, 1), - items: vec![RecordId { - did: "did:plc:asdf-1".into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - },], - next: None, - total: 3, - } - ); - assert_stats(storage.get_stats()?, 4..=4, 1..=1, 3..=3); - }); - - test_each_storage!(page_links_accounts_inactive, |storage| { - for i in 1..=4 { - storage.push( - &ActionableEvent::CreateLinks { - record_id: RecordId { - did: format!("did:plc:asdf-{i}").into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - }, - links: vec![CollectedLink { - target: Link::Uri("a.com".into()), - path: ".abc.uri".into(), - }], - }, - 0, - )?; - } - let links = storage.get_links("a.com", "app.t.c", ".abc.uri", 2, None)?; - assert_eq!( - links, - PagedAppendingCollection { - version: (4, 0), - items: vec![ - RecordId { - did: "did:plc:asdf-4".into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - }, - RecordId { - did: "did:plc:asdf-3".into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - }, - ], - next: Some(2), - total: 4, - } - ); - storage.push( - &ActionableEvent::DeactivateAccount("did:plc:asdf-1".into()), - 0, - )?; - let links = storage.get_links("a.com", "app.t.c", ".abc.uri", 2, links.next)?; - assert_eq!( - links, - PagedAppendingCollection { - version: (4, 0), - items: vec![RecordId { - did: "did:plc:asdf-2".into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - },], - next: None, - total: 4, - } - ); - assert_stats(storage.get_stats()?, 4..=4, 1..=1, 4..=4); - }); - - test_each_storage!(get_all_counts, |storage| { - storage.push( - &ActionableEvent::CreateLinks { - record_id: RecordId { - did: "did:plc:asdf".into(), - collection: "app.t.c".into(), - rkey: "asdf".into(), - }, - links: vec![ - CollectedLink { - target: Link::Uri("a.com".into()), - path: ".abc.uri".into(), - }, - CollectedLink { - target: Link::Uri("a.com".into()), - path: ".def.uri".into(), - }, - ], - }, - 0, - )?; - assert_eq!(storage.get_all_record_counts("a.com")?, { - let mut counts = HashMap::new(); - let mut t_c_counts = HashMap::new(); - t_c_counts.insert(".abc.uri".into(), 1); - t_c_counts.insert(".def.uri".into(), 1); - counts.insert("app.t.c".into(), t_c_counts); - counts - }); - assert_eq!(storage.get_all_counts("a.com")?, { - let mut counts = HashMap::new(); - let mut t_c_counts = HashMap::new(); - t_c_counts.insert( - ".abc.uri".into(), - CountsByCount { - records: 1, - distinct_dids: 1, - }, - ); - t_c_counts.insert( - ".def.uri".into(), - CountsByCount { - records: 1, - distinct_dids: 1, - }, - ); - counts.insert("app.t.c".into(), t_c_counts); - counts - }); - assert_stats(storage.get_stats()?, 1..=1, 2..=2, 1..=1); - }); -} diff --git a/constellation/src/storage/rocks_store.rs b/constellation/src/storage/rocks_store.rs deleted file mode 100644 index 73fa4e8..0000000 --- a/constellation/src/storage/rocks_store.rs +++ /dev/null @@ -1,1293 +0,0 @@ -use super::{ActionableEvent, LinkReader, LinkStorage, PagedAppendingCollection, StorageStats}; -use crate::{CountsByCount, Did, RecordId}; -use anyhow::{bail, Result}; -use bincode::Options as BincodeOptions; -use links::CollectedLink; -use metrics::{counter, describe_counter, describe_histogram, histogram, Unit}; -use ratelimit::Ratelimiter; -use rocksdb::backup::{BackupEngine, BackupEngineOptions}; -use rocksdb::{ - AsColumnFamilyRef, ColumnFamilyDescriptor, DBWithThreadMode, IteratorMode, MergeOperands, - MultiThreaded, Options, PrefixRange, ReadOptions, WriteBatch, -}; -use serde::{Deserialize, Serialize}; -use std::collections::{HashMap, HashSet}; -use std::io::Read; -use std::marker::PhantomData; -use std::path::{Path, PathBuf}; -use std::sync::{ - atomic::{AtomicU64, Ordering}, - Arc, -}; -use std::thread; -use std::time::{Duration, Instant}; -use tokio_util::sync::CancellationToken; - -static DID_IDS_CF: &str = "did_ids"; -static TARGET_IDS_CF: &str = "target_ids"; -static TARGET_LINKERS_CF: &str = "target_links"; -static LINK_TARGETS_CF: &str = "link_targets"; - -static JETSTREAM_CURSOR_KEY: &str = "jetstream_cursor"; - -// todo: actually understand and set these options probably better -fn rocks_opts_base() -> Options { - let mut opts = Options::default(); - opts.set_level_compaction_dynamic_level_bytes(true); - opts.create_if_missing(true); - opts.set_compression_type(rocksdb::DBCompressionType::Lz4); - opts.set_bottommost_compression_type(rocksdb::DBCompressionType::Zstd); // this probably doesn't work because it hasn't been enabled - // TODO: actually enable the bottommost compression. but after other changes run for a bit in case zstd is cpu- or mem-expensive. - opts -} -fn get_db_opts() -> Options { - let mut opts = rocks_opts_base(); - opts.create_missing_column_families(true); - opts.increase_parallelism(4); // todo: make configurable if anyone else actually runs a different instance. start at # of cores - // consider doing optimize_level_style_compaction or optimize_universal_style_compaction - opts -} -fn get_db_read_opts() -> Options { - let mut opts = Options::default(); - opts.optimize_for_point_lookup(16_384); // mb (run this on big machines) - opts -} - -#[derive(Debug, Clone)] -pub struct RocksStorage { - pub db: Arc>, // TODO: mov seqs here (concat merge op will be fun) - did_id_table: IdTable, - target_id_table: IdTable, - is_writer: bool, - backup_task: Arc>>>, -} - -trait IdTableValue: ValueFromRocks + Clone { - fn new(v: u64) -> Self; - fn id(&self) -> u64; -} -#[derive(Debug, Clone)] -struct IdTableBase -where - Orig: KeyFromRocks, - for<'a> &'a Orig: AsRocksKey, -{ - _key_marker: PhantomData, - _val_marker: PhantomData, - name: String, - id_seq: Arc, -} -impl IdTableBase -where - Orig: KeyFromRocks, - for<'a> &'a Orig: AsRocksKey, -{ - fn cf_descriptor(&self) -> ColumnFamilyDescriptor { - ColumnFamilyDescriptor::new(&self.name, rocks_opts_base()) - } - fn init( - self, - db: &DBWithThreadMode, - ) -> Result> { - if db.cf_handle(&self.name).is_none() { - bail!("failed to get cf handle from db -- was the db open with our .cf_descriptor()?"); - } - let priv_id_seq = if let Some(seq_bytes) = db.get(self.seq_key())? { - if seq_bytes.len() != 8 { - bail!( - "reading bytes for u64 id seq {:?}: found the wrong number of bytes", - self.seq_key() - ); - } - let mut buf: [u8; 8] = [0; 8]; - seq_bytes.as_slice().read_exact(&mut buf)?; - let last_seq = u64::from_le_bytes(buf); - last_seq + 1 - } else { - 1 - }; - self.id_seq.store(priv_id_seq, Ordering::SeqCst); - Ok(IdTable { - base: self, - priv_id_seq, - }) - } - fn seq_key(&self) -> Vec { - let mut k = b"__id_seq_key_plz_be_unique:".to_vec(); - k.extend(self.name.as_bytes()); - k - } -} -#[derive(Debug, Clone)] -struct IdTable -where - Orig: KeyFromRocks, - for<'a> &'a Orig: AsRocksKey, -{ - base: IdTableBase, - priv_id_seq: u64, -} -impl IdTable -where - Orig: KeyFromRocks, - for<'v> &'v IdVal: AsRocksValue, - for<'k> &'k Orig: AsRocksKey, -{ - #[must_use] - fn setup(name: &str) -> IdTableBase { - IdTableBase:: { - _key_marker: PhantomData, - _val_marker: PhantomData, - name: name.into(), - id_seq: Arc::new(AtomicU64::new(0)), // zero is "uninint", first seq num will be 1 - } - } - fn get_id_val( - &self, - db: &DBWithThreadMode, - orig: &Orig, - ) -> Result> { - let cf = db.cf_handle(&self.base.name).unwrap(); - if let Some(_id_bytes) = db.get_cf(&cf, _rk(orig))? { - Ok(Some(_vr(&_id_bytes)?)) - } else { - Ok(None) - } - } - fn __get_or_create_id_val( - &mut self, - cf: &CF, - db: &DBWithThreadMode, - batch: &mut WriteBatch, - orig: &Orig, - ) -> Result - where - CF: AsColumnFamilyRef, - { - Ok(self.get_id_val(db, orig)?.unwrap_or_else(|| { - let prev_priv_seq = self.priv_id_seq; - self.priv_id_seq += 1; - let prev_public_seq = self.base.id_seq.swap(self.priv_id_seq, Ordering::SeqCst); - assert_eq!( - prev_public_seq, prev_priv_seq, - "public seq may have been modified??" - ); - let id_value = IdVal::new(self.priv_id_seq); - batch.put(self.base.seq_key(), self.priv_id_seq.to_le_bytes()); - batch.put_cf(cf, _rk(orig), _rv(&id_value)); - id_value - })) - } - fn estimate_count(&self) -> u64 { - self.base.id_seq.load(Ordering::SeqCst) - 1 // -1 because seq zero is reserved - } -} -impl IdTable -where - Orig: KeyFromRocks, - for<'v> &'v IdVal: AsRocksValue, - for<'k> &'k Orig: AsRocksKey, -{ - fn get_or_create_id_val( - &mut self, - db: &DBWithThreadMode, - batch: &mut WriteBatch, - orig: &Orig, - ) -> Result { - let cf = db.cf_handle(&self.base.name).unwrap(); - let id_val = self.__get_or_create_id_val(&cf, db, batch, orig)?; - // TODO: assert that the original is never a u64 that could collide - batch.put_cf(&cf, id_val.id().to_be_bytes(), _rk(orig)); // reversed rk/rv on purpose here :/ - Ok(id_val) - } - - fn get_val_from_id( - &self, - db: &DBWithThreadMode, - id: u64, - ) -> Result> { - let cf = db.cf_handle(&self.base.name).unwrap(); - if let Some(orig_bytes) = db.get_cf(&cf, id.to_be_bytes())? { - // HACK ish - Ok(Some(_kr(&orig_bytes)?)) - } else { - Ok(None) - } - } -} -impl IdTable -where - Orig: KeyFromRocks, - for<'v> &'v IdVal: AsRocksValue, - for<'k> &'k Orig: AsRocksKey, -{ - fn get_or_create_id_val( - &mut self, - db: &DBWithThreadMode, - batch: &mut WriteBatch, - orig: &Orig, - ) -> Result { - let cf = db.cf_handle(&self.base.name).unwrap(); - self.__get_or_create_id_val(&cf, db, batch, orig) - } -} - -impl IdTableValue for DidIdValue { - fn new(v: u64) -> Self { - DidIdValue(DidId(v), true) - } - fn id(&self) -> u64 { - self.0 .0 - } -} -impl IdTableValue for TargetId { - fn new(v: u64) -> Self { - TargetId(v) - } - fn id(&self) -> u64 { - self.0 - } -} - -impl RocksStorage { - pub fn new(path: impl AsRef) -> Result { - Self::describe_metrics(); - RocksStorage::open_readmode(path, false) - } - - pub fn open_readonly(path: impl AsRef) -> Result { - RocksStorage::open_readmode(path, true) - } - - fn open_readmode(path: impl AsRef, readonly: bool) -> Result { - let did_id_table = IdTable::<_, _, true>::setup(DID_IDS_CF); - let target_id_table = IdTable::<_, _, false>::setup(TARGET_IDS_CF); - - let cfs = vec![ - // id reference tables - did_id_table.cf_descriptor(), - target_id_table.cf_descriptor(), - // the reverse links: - ColumnFamilyDescriptor::new(TARGET_LINKERS_CF, { - let mut opts = rocks_opts_base(); - opts.set_merge_operator_associative( - "merge_op_extend_did_ids", - Self::merge_op_extend_did_ids, - ); - opts - }), - // unfortunately we also need forward links to handle deletes - ColumnFamilyDescriptor::new(LINK_TARGETS_CF, rocks_opts_base()), - ]; - - let db = if readonly { - DBWithThreadMode::open_cf_descriptors_read_only(&get_db_read_opts(), path, cfs, false)? - } else { - DBWithThreadMode::open_cf_descriptors(&get_db_opts(), path, cfs)? - }; - - let db = Arc::new(db); - let did_id_table = did_id_table.init(&db)?; - let target_id_table = target_id_table.init(&db)?; - Ok(Self { - db, - did_id_table, - target_id_table, - is_writer: !readonly, - backup_task: None.into(), - }) - } - - pub fn start_backup( - &mut self, - path: PathBuf, - auto: Option<(u64, Option)>, - stay_alive: CancellationToken, - ) -> Result<()> { - let task = if let Some((interval_hrs, copies)) = auto { - eprintln!("backups: starting background task..."); - self.backup_task(path, interval_hrs, copies, stay_alive) - } else { - eprintln!("backups: starting a one-off backup..."); - thread::spawn({ - let db = self.db.clone(); - move || Self::do_backup(db, path) - }) - }; - self.backup_task = Arc::new(Some(task)); - Ok(()) - } - - fn backup_task( - &self, - path: PathBuf, - interval_hrs: u64, - copies: Option, - stay_alive: CancellationToken, - ) -> std::thread::JoinHandle> { - let db = self.db.clone(); - thread::spawn(move || { - let limit = - Ratelimiter::builder(1, Duration::from_secs(interval_hrs * 60 * 60)).build()?; - let minimum_sleep = Duration::from_secs(1); - - 'quit: loop { - if let Err(sleep) = limit.try_wait() { - eprintln!("backups: background: next backup scheduled in {sleep:?}"); - let waiting = Instant::now(); - loop { - let remaining = sleep - waiting.elapsed(); - if stay_alive.is_cancelled() { - break 'quit; - } else if remaining <= Duration::ZERO { - break; - } else if remaining < minimum_sleep { - thread::sleep(remaining); - break; - } else { - thread::sleep(minimum_sleep); - } - } - } - eprintln!("backups: background: starting backup..."); - if let Err(e) = Self::do_backup(db.clone(), &path) { - eprintln!("backups: background: backup failed: {e:?}"); - // todo: metrics - } else { - eprintln!("backups: background: backup succeeded yay"); - } - if let Some(copies) = copies { - eprintln!("backups: background: trimming to {copies} saved backups..."); - if let Err(e) = Self::trim_backups(copies, &path) { - eprintln!("backups: background: failed to trim backups: {e:?}"); - } else { - eprintln!("backups: background: trimming worked!") - } - } - } - - Ok(()) - }) - } - - fn do_backup(db: Arc>, path: impl AsRef) -> Result<()> { - let mut engine = - BackupEngine::open(&BackupEngineOptions::new(path)?, &rocksdb::Env::new()?)?; - eprintln!("backups: starting backup..."); - let t0 = Instant::now(); - if let Err(e) = engine.create_new_backup(&db) { - eprintln!("backups: oh no, backup failed: {e:?}"); - } else { - eprintln!("backups: yay, backup worked?"); - } - eprintln!( - "backups: backup finished after {:.2}s", - t0.elapsed().as_secs_f32() - ); - Ok(()) - } - - fn trim_backups(num_backups_to_keep: usize, path: impl AsRef) -> Result<()> { - let mut engine = - BackupEngine::open(&BackupEngineOptions::new(path)?, &rocksdb::Env::new()?)?; - engine.purge_old_backups(num_backups_to_keep)?; - Ok(()) - } - - fn describe_metrics() { - describe_histogram!( - "storage_rocksdb_read_seconds", - Unit::Seconds, - "duration of the read stage of actions" - ); - describe_histogram!( - "storage_rocksdb_action_seconds", - Unit::Seconds, - "duration of read + write of actions" - ); - describe_counter!( - "storage_rocksdb_batch_ops_total", - Unit::Count, - "total batched operations from actions" - ); - describe_histogram!( - "storage_rocksdb_delete_account_ops", - Unit::Count, - "total batched ops for account deletions" - ); - } - - fn merge_op_extend_did_ids( - key: &[u8], - existing: Option<&[u8]>, - operands: &MergeOperands, - ) -> Option> { - let mut linkers: Vec<_> = if let Some(existing_bytes) = existing { - match _vr(existing_bytes) { - Ok(TargetLinkers(mut existing_linkers)) => { - existing_linkers.reserve(operands.len()); - existing_linkers - } - Err(e) => { - eprintln!("bug? could not deserialize existing target linkers: {e:?}. key={key:?}. continuing, but data will be lost!"); - if existing_bytes.len() < 1000 { - eprintln!("dropping: {existing_bytes:?}"); - } else { - eprintln!("(too long to print)"); - } - Vec::with_capacity(operands.len()) - } - } - } else { - Vec::with_capacity(operands.len()) - }; - for new_linkers in operands { - match _vr(new_linkers) { - Ok(TargetLinkers(new_linkers)) => linkers.extend(new_linkers), - Err(e) => { - eprintln!("bug? could not deserialize new target linkers: {e:?}. key={key:?}. continuing, but data will be lost!"); - if new_linkers.len() < 1000 { - eprintln!("skipping: {new_linkers:?}"); - } else { - eprintln!("(too long to print)"); - } - } - } - } - Some(_rv(&TargetLinkers(linkers))) - } - - fn prefix_iter_cf( - &self, - cf: &CF, - pre: P, - ) -> impl Iterator + use<'_, K, V, CF, P> - where - K: KeyFromRocks, - V: ValueFromRocks, - CF: AsColumnFamilyRef, - for<'a> &'a P: AsRocksKeyPrefix, - { - let mut read_opts = ReadOptions::default(); - read_opts.set_iterate_range(PrefixRange(_rkp(&pre))); // TODO verify: inclusive bounds? - self.db - .iterator_cf_opt(cf, read_opts, IteratorMode::Start) - .map_while(Result::ok) - .map_while(|(k, v)| Some((_kr(&k).ok()?, _vr(&v).ok()?))) - } - - fn update_did_id_value(&self, batch: &mut WriteBatch, did: &Did, update: F) -> Result - where - F: FnOnce(DidIdValue) -> Option, - { - let cf = self.db.cf_handle(DID_IDS_CF).unwrap(); - let Some(did_id_value) = self.did_id_table.get_id_val(&self.db, did)? else { - return Ok(false); - }; - let Some(new_did_id_value) = update(did_id_value) else { - return Ok(false); - }; - batch.put_cf(&cf, _rk(did), _rv(&new_did_id_value)); - Ok(true) - } - fn delete_did_id_value(&self, batch: &mut WriteBatch, did: &Did) { - let cf = self.db.cf_handle(DID_IDS_CF).unwrap(); - batch.delete_cf(&cf, _rk(did)); - } - - fn get_target_linkers(&self, target_id: &TargetId) -> Result { - let cf = self.db.cf_handle(TARGET_LINKERS_CF).unwrap(); - let Some(linkers_bytes) = self.db.get_cf(&cf, _rk(target_id))? else { - return Ok(TargetLinkers::default()); - }; - _vr(&linkers_bytes) - } - /// zero out every duplicate did. bit of a hack, looks the same as deleted, but eh - fn get_distinct_target_linkers(&self, target_id: &TargetId) -> Result { - let mut seen = HashSet::new(); - let mut linkers = self.get_target_linkers(target_id)?; - for (did_id, _) in linkers.0.iter_mut() { - if seen.contains(did_id) { - did_id.0 = 0; - } else { - seen.insert(*did_id); - } - } - Ok(linkers) - } - fn merge_target_linker( - &self, - batch: &mut WriteBatch, - target_id: &TargetId, - linker_did_id: &DidId, - linker_rkey: &RKey, - ) { - let cf = self.db.cf_handle(TARGET_LINKERS_CF).unwrap(); - batch.merge_cf( - &cf, - _rk(target_id), - _rv(&TargetLinkers(vec![(*linker_did_id, linker_rkey.clone())])), - ); - } - fn update_target_linkers( - &self, - batch: &mut WriteBatch, - target_id: &TargetId, - update: F, - ) -> Result - where - F: FnOnce(TargetLinkers) -> Option, - { - let cf = self.db.cf_handle(TARGET_LINKERS_CF).unwrap(); - let existing_linkers = self.get_target_linkers(target_id)?; - let Some(new_linkers) = update(existing_linkers) else { - return Ok(false); - }; - batch.put_cf(&cf, _rk(target_id), _rv(&new_linkers)); - Ok(true) - } - - fn put_link_targets( - &self, - batch: &mut WriteBatch, - record_link_key: &RecordLinkKey, - targets: &RecordLinkTargets, - ) { - // todo: we are almost idempotent to link creates with this blind write, but we'll still - // merge in the reverse index. we could read+modify+write here but it'll be SLOWWWWW on - // the path that we need to be fast. we could go back to a merge op and probably be - // consistent. or we can accept just a littttttle inconsistency and be idempotent on - // forward links but not reverse, slightly messing up deletes :/ - // _maybe_ we could run in slow idempotent r-m-w mode during firehose catch-up at the start, - // then switch to the fast version? - let cf = self.db.cf_handle(LINK_TARGETS_CF).unwrap(); - batch.put_cf(&cf, _rk(record_link_key), _rv(targets)); - } - fn get_record_link_targets( - &self, - record_link_key: &RecordLinkKey, - ) -> Result> { - let cf = self.db.cf_handle(LINK_TARGETS_CF).unwrap(); - if let Some(bytes) = self.db.get_cf(&cf, _rk(record_link_key))? { - Ok(Some(_vr(&bytes)?)) - } else { - Ok(None) - } - } - fn delete_record_link(&self, batch: &mut WriteBatch, record_link_key: &RecordLinkKey) { - let cf = self.db.cf_handle(LINK_TARGETS_CF).unwrap(); - batch.delete_cf(&cf, _rk(record_link_key)); - } - fn iter_links_for_did_id( - &self, - did_id: &DidId, - ) -> impl Iterator + use<'_> { - let cf = self.db.cf_handle(LINK_TARGETS_CF).unwrap(); - self.prefix_iter_cf(&cf, RecordLinkKeyDidIdPrefix(*did_id)) - } - fn iter_targets_for_target( - &self, - target: &Target, - ) -> impl Iterator + use<'_> { - let cf = self.db.cf_handle(TARGET_IDS_CF).unwrap(); - self.prefix_iter_cf(&cf, TargetIdTargetPrefix(target.clone())) - } - - // - // higher-level event action handlers - // - - fn add_links( - &mut self, - record_id: &RecordId, - links: &[CollectedLink], - batch: &mut WriteBatch, - ) -> Result<()> { - let DidIdValue(did_id, _) = - self.did_id_table - .get_or_create_id_val(&self.db, batch, &record_id.did)?; - - let record_link_key = RecordLinkKey( - did_id, - Collection(record_id.collection()), - RKey(record_id.rkey()), - ); - let mut record_link_targets = RecordLinkTargets::with_capacity(links.len()); - - for CollectedLink { target, path } in links { - let target_key = TargetKey( - Target(target.clone().into_string()), - Collection(record_id.collection()), - RPath(path.clone()), - ); - let target_id = - self.target_id_table - .get_or_create_id_val(&self.db, batch, &target_key)?; - self.merge_target_linker(batch, &target_id, &did_id, &RKey(record_id.rkey())); - - record_link_targets.add(RecordLinkTarget(RPath(path.clone()), target_id)) - } - - self.put_link_targets(batch, &record_link_key, &record_link_targets); - Ok(()) - } - - fn remove_links(&mut self, record_id: &RecordId, batch: &mut WriteBatch) -> Result<()> { - let Some(DidIdValue(linking_did_id, _)) = - self.did_id_table.get_id_val(&self.db, &record_id.did)? - else { - return Ok(()); // we don't know her: nothing to do - }; - - let record_link_key = RecordLinkKey( - linking_did_id, - Collection(record_id.collection()), - RKey(record_id.rkey()), - ); - let Some(record_link_targets) = self.get_record_link_targets(&record_link_key)? else { - return Ok(()); // we don't have these links - }; - - // we do read -> modify -> write here: could merge-op in the deletes instead? - // otherwise it's another single-thread-constraining thing. - for RecordLinkTarget(_, target_id) in record_link_targets.0 { - self.update_target_linkers(batch, &target_id, |mut linkers| { - if linkers.0.is_empty() { - eprintln!("bug? linked target was missing when removing links"); - } - if !linkers.remove_linker(&linking_did_id, &RKey(record_id.rkey.clone())) { - eprintln!("bug? linked target was missing a link when removing links"); - } - Some(linkers) - })?; - } - - self.delete_record_link(batch, &record_link_key); - Ok(()) - } - - fn set_account(&mut self, did: &Did, active: bool, batch: &mut WriteBatch) -> Result<()> { - // this needs to be read-modify-write since the did_id needs to stay the same, - // which has a benefit of allowing to avoid adding entries for dids we don't - // need. reading on dids needs to be cheap anyway for the current design, and - // did active/inactive updates are low-freq in the firehose so, eh, it's fine. - self.update_did_id_value(batch, did, |current_value| { - Some(DidIdValue(current_value.did_id(), active)) - })?; - Ok(()) - } - - fn delete_account(&mut self, did: &Did, batch: &mut WriteBatch) -> Result { - let mut total_batched_ops = 0; - let Some(DidIdValue(did_id, _)) = self.did_id_table.get_id_val(&self.db, did)? else { - return Ok(total_batched_ops); // ignore updates for dids we don't know about - }; - self.delete_did_id_value(batch, did); - // TODO: also delete the reverse!! - - // use a separate batch for all their links, since it can be a lot and make us crash at around 1GiB batch size. - // this should still hopefully be crash-safe: as long as we don't actually delete the DidId entry until after all links are cleared. - // the above .delete_did_id_value is batched, so it shouldn't be written until we've returned from this fn successfully - // TODO: queue a background delete task or whatever - // TODO: test delete account with more links than chunk size - let stuff: Vec<_> = self.iter_links_for_did_id(&did_id).collect(); - for chunk in stuff.chunks(1024) { - let mut mini_batch = WriteBatch::default(); - - for (record_link_key, links) in chunk { - self.delete_record_link(&mut mini_batch, record_link_key); // _could_ use delete range here instead of individual deletes, but since we have to scan anyway it's not obvious if it's better - - for RecordLinkTarget(_, target_link_id) in links.0.iter() { - self.update_target_linkers(&mut mini_batch, target_link_id, |mut linkers| { - if !linkers.remove_linker(&did_id, &record_link_key.2) { - eprintln!("bug? could not find linker when removing links while deleting an account"); - } - Some(linkers) - })?; - } - } - total_batched_ops += mini_batch.len(); - self.db.write(mini_batch)?; // todo - } - Ok(total_batched_ops) - } -} - -impl Drop for RocksStorage { - fn drop(&mut self) { - if self.is_writer { - println!("rocksdb writer: cleaning up for shutdown..."); - if let Err(e) = self.db.flush_wal(true) { - eprintln!("rocks: flushing wal failed: {e:?}"); - } - if let Err(e) = self.db.flush_opt(&{ - let mut opt = rocksdb::FlushOptions::default(); - opt.set_wait(true); - opt - }) { - eprintln!("rocks: flushing memtables failed: {e:?}"); - } - match Arc::get_mut(&mut self.backup_task) { - Some(maybe_task) => { - if let Some(task) = maybe_task.take() { - eprintln!("waiting for backup task to complete..."); - if let Err(e) = task.join() { - eprintln!("failed to join backup task: {e:?}"); - } - } - } - None => eprintln!("rocks: failed to get backup task, likely a bug."), - } - self.db.cancel_all_background_work(true); - } - } -} - -impl AsRocksValue for u64 {} -impl ValueFromRocks for u64 {} - -impl LinkStorage for RocksStorage { - fn get_cursor(&mut self) -> Result> { - self.db - .get(JETSTREAM_CURSOR_KEY)? - .map(|b| _vr(&b)) - .transpose() - } - - fn push(&mut self, event: &ActionableEvent, cursor: u64) -> Result<()> { - // normal ops - let mut batch = WriteBatch::default(); - let t0 = Instant::now(); - if let Some(action) = match event { - ActionableEvent::CreateLinks { record_id, links } => { - self.add_links(record_id, links, &mut batch)?; - Some("create_links") - } - ActionableEvent::UpdateLinks { - record_id, - new_links, - } => { - self.remove_links(record_id, &mut batch)?; - self.add_links(record_id, new_links, &mut batch)?; - Some("update_links") - } - ActionableEvent::DeleteRecord(record_id) => { - self.remove_links(record_id, &mut batch)?; - Some("delete_record") - } - ActionableEvent::ActivateAccount(did) => { - self.set_account(did, true, &mut batch)?; - Some("set_account_status") - } - ActionableEvent::DeactivateAccount(did) => { - self.set_account(did, false, &mut batch)?; - Some("set_account_status") - } - ActionableEvent::DeleteAccount(_) => None, // delete account is handled specially - } { - let t_read = t0.elapsed(); - batch.put(JETSTREAM_CURSOR_KEY.as_bytes(), _rv(cursor)); - let batch_ops = batch.len(); - self.db.write(batch)?; - let t_total = t0.elapsed(); - - histogram!("storage_rocksdb_read_seconds", "action" => action) - .record(t_read.as_secs_f64()); - histogram!("storage_rocksdb_action_seconds", "action" => action) - .record(t_total.as_secs_f64()); - counter!("storage_rocksdb_batch_ops_total", "action" => action) - .increment(batch_ops as u64); - } - - // special metrics for account deletion which can be arbitrarily expensive - let mut outer_batch = WriteBatch::default(); - let t0 = Instant::now(); - if let ActionableEvent::DeleteAccount(did) = event { - let inner_batch_ops = self.delete_account(did, &mut outer_batch)?; - let total_batch_ops = inner_batch_ops + outer_batch.len(); - self.db.write(outer_batch)?; - let t_total = t0.elapsed(); - - histogram!("storage_rocksdb_action_seconds", "action" => "delete_account") - .record(t_total.as_secs_f64()); - counter!("storage_rocksdb_batch_ops_total", "action" => "delete_account") - .increment(total_batch_ops as u64); - histogram!("storage_rocksdb_delete_account_ops").record(total_batch_ops as f64); - } - - Ok(()) - } - - fn to_readable(&mut self) -> impl LinkReader { - let mut readable = self.clone(); - readable.is_writer = false; - readable - } -} - -impl LinkReader for RocksStorage { - fn get_count(&self, target: &str, collection: &str, path: &str) -> Result { - let target_key = TargetKey( - Target(target.to_string()), - Collection(collection.to_string()), - RPath(path.to_string()), - ); - if let Some(target_id) = self.target_id_table.get_id_val(&self.db, &target_key)? { - let (alive, _) = self.get_target_linkers(&target_id)?.count(); - Ok(alive) - } else { - Ok(0) - } - } - - fn get_distinct_did_count(&self, target: &str, collection: &str, path: &str) -> Result { - let target_key = TargetKey( - Target(target.to_string()), - Collection(collection.to_string()), - RPath(path.to_string()), - ); - if let Some(target_id) = self.target_id_table.get_id_val(&self.db, &target_key)? { - Ok(self.get_target_linkers(&target_id)?.count_distinct_dids()) - } else { - Ok(0) - } - } - - fn get_links( - &self, - target: &str, - collection: &str, - path: &str, - limit: u64, - until: Option, - ) -> Result> { - let target_key = TargetKey( - Target(target.to_string()), - Collection(collection.to_string()), - RPath(path.to_string()), - ); - - let Some(target_id) = self.target_id_table.get_id_val(&self.db, &target_key)? else { - return Ok(PagedAppendingCollection { - version: (0, 0), - items: Vec::new(), - next: None, - total: 0, - }); - }; - - let linkers = self.get_target_linkers(&target_id)?; - - let (alive, gone) = linkers.count(); - let total = alive + gone; - let end = until.map(|u| std::cmp::min(u, total)).unwrap_or(total) as usize; - let begin = end.saturating_sub(limit as usize); - let next = if begin == 0 { None } else { Some(begin as u64) }; - - let did_id_rkeys = linkers.0[begin..end].iter().rev().collect::>(); - - let mut items = Vec::with_capacity(did_id_rkeys.len()); - // TODO: use get-many (or multi-get or whatever it's called) - for (did_id, rkey) in did_id_rkeys { - if did_id.is_empty() { - continue; - } - if let Some(did) = self.did_id_table.get_val_from_id(&self.db, did_id.0)? { - let Some(DidIdValue(_, active)) = self.did_id_table.get_id_val(&self.db, &did)? - else { - eprintln!("failed to look up did_value from did_id {did_id:?}: {did:?}: data consistency bug?"); - continue; - }; - if !active { - continue; - } - items.push(RecordId { - did, - collection: collection.to_string(), - rkey: rkey.0.clone(), - }); - } else { - eprintln!("failed to look up did from did_id {did_id:?}"); - } - } - - Ok(PagedAppendingCollection { - version: (total, gone), - items, - next, - total: alive, - }) - } - - fn get_distinct_dids( - &self, - target: &str, - collection: &str, - path: &str, - limit: u64, - until: Option, - ) -> Result> { - let target_key = TargetKey( - Target(target.to_string()), - Collection(collection.to_string()), - RPath(path.to_string()), - ); - - let Some(target_id) = self.target_id_table.get_id_val(&self.db, &target_key)? else { - return Ok(PagedAppendingCollection { - version: (0, 0), - items: Vec::new(), - next: None, - total: 0, - }); - }; - - let linkers = self.get_distinct_target_linkers(&target_id)?; - - let (alive, gone) = linkers.count(); - let total = alive + gone; - let end = until.map(|u| std::cmp::min(u, total)).unwrap_or(total) as usize; - let begin = end.saturating_sub(limit as usize); - let next = if begin == 0 { None } else { Some(begin as u64) }; - - let did_id_rkeys = linkers.0[begin..end].iter().rev().collect::>(); - - let mut items = Vec::with_capacity(did_id_rkeys.len()); - // TODO: use get-many (or multi-get or whatever it's called) - for (did_id, _) in did_id_rkeys { - if did_id.is_empty() { - continue; - } - if let Some(did) = self.did_id_table.get_val_from_id(&self.db, did_id.0)? { - let Some(DidIdValue(_, active)) = self.did_id_table.get_id_val(&self.db, &did)? - else { - eprintln!("failed to look up did_value from did_id {did_id:?}: {did:?}: data consistency bug?"); - continue; - }; - if !active { - continue; - } - items.push(did); - } else { - eprintln!("failed to look up did from did_id {did_id:?}"); - } - } - - Ok(PagedAppendingCollection { - version: (total, gone), - items, - next, - total: alive, - }) - } - - fn get_all_record_counts(&self, target: &str) -> Result>> { - let mut out: HashMap> = HashMap::new(); - for (target_key, target_id) in self.iter_targets_for_target(&Target(target.into())) { - let TargetKey(_, Collection(ref collection), RPath(ref path)) = target_key; - let (count, _) = self.get_target_linkers(&target_id)?.count(); - out.entry(collection.into()) - .or_default() - .insert(path.clone(), count); - } - Ok(out) - } - - fn get_all_counts( - &self, - target: &str, - ) -> Result>> { - let mut out: HashMap> = HashMap::new(); - for (target_key, target_id) in self.iter_targets_for_target(&Target(target.into())) { - let TargetKey(_, Collection(ref collection), RPath(ref path)) = target_key; - let target_linkers = self.get_target_linkers(&target_id)?; - let (records, _) = target_linkers.count(); - let distinct_dids = target_linkers.count_distinct_dids(); - out.entry(collection.into()).or_default().insert( - path.clone(), - CountsByCount { - records, - distinct_dids, - }, - ); - } - Ok(out) - } - - fn get_stats(&self) -> Result { - let dids = self.did_id_table.estimate_count(); - let targetables = self.target_id_table.estimate_count(); - let lr_cf = self.db.cf_handle(LINK_TARGETS_CF).unwrap(); - let linking_records = self - .db - .property_value_cf(&lr_cf, rocksdb::properties::ESTIMATE_NUM_KEYS)? - .map(|s| s.parse::()) - .transpose()? - .unwrap_or(0); - Ok(StorageStats { - dids, - targetables, - linking_records, - }) - } -} - -trait AsRocksKey: Serialize {} -trait AsRocksKeyPrefix: Serialize {} -trait AsRocksValue: Serialize {} -trait KeyFromRocks: for<'de> Deserialize<'de> {} -trait ValueFromRocks: for<'de> Deserialize<'de> {} - -// did_id table -impl AsRocksKey for &Did {} -impl AsRocksValue for &DidIdValue {} -impl ValueFromRocks for DidIdValue {} - -// temp -impl KeyFromRocks for Did {} -impl AsRocksKey for &DidId {} - -// target_ids table -impl AsRocksKey for &TargetKey {} -impl AsRocksKeyPrefix for &TargetIdTargetPrefix {} -impl AsRocksValue for &TargetId {} -impl KeyFromRocks for TargetKey {} -impl ValueFromRocks for TargetId {} - -// target_links table -impl AsRocksKey for &TargetId {} -impl AsRocksValue for &TargetLinkers {} -impl ValueFromRocks for TargetLinkers {} - -// record_link_targets table -impl AsRocksKey for &RecordLinkKey {} -impl AsRocksKeyPrefix for &RecordLinkKeyDidIdPrefix {} -impl AsRocksValue for &RecordLinkTargets {} -impl KeyFromRocks for RecordLinkKey {} -impl ValueFromRocks for RecordLinkTargets {} - -pub fn _bincode_opts() -> impl BincodeOptions { - bincode::DefaultOptions::new().with_big_endian() // happier db -- numeric prefixes in lsm -} -fn _rk(k: impl AsRocksKey) -> Vec { - _bincode_opts().serialize(&k).unwrap() -} -fn _rkp(kp: impl AsRocksKeyPrefix) -> Vec { - _bincode_opts().serialize(&kp).unwrap() -} -fn _rv(v: impl AsRocksValue) -> Vec { - _bincode_opts().serialize(&v).unwrap() -} -fn _kr(bytes: &[u8]) -> Result { - Ok(_bincode_opts().deserialize(bytes)?) -} -fn _vr(bytes: &[u8]) -> Result { - Ok(_bincode_opts().deserialize(bytes)?) -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] -pub struct Collection(pub String); - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] -pub struct RPath(pub String); - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct RKey(pub String); - -impl RKey { - fn empty() -> Self { - RKey("".to_string()) - } -} - -// did ids -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub struct DidId(pub u64); - -impl DidId { - fn empty() -> Self { - DidId(0) - } - fn is_empty(&self) -> bool { - self.0 == 0 - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -struct DidIdValue(DidId, bool); // active or not - -impl DidIdValue { - fn did_id(&self) -> DidId { - let Self(id, _) = self; - *id - } -} - -// target ids -#[derive(Debug, Clone, Serialize, Deserialize)] -struct TargetId(u64); // key - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Target(pub String); // the actual target/uri - -// targets (uris, dids, etc.): the reverse index -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TargetKey(pub Target, pub Collection, pub RPath); - -#[derive(Debug, Default, Serialize, Deserialize)] -pub struct TargetLinkers(pub Vec<(DidId, RKey)>); - -impl TargetLinkers { - fn remove_linker(&mut self, did: &DidId, rkey: &RKey) -> bool { - if let Some(entry) = self.0.iter_mut().rfind(|d| **d == (*did, rkey.clone())) { - *entry = (DidId::empty(), RKey::empty()); - true - } else { - false - } - } - pub fn count(&self) -> (u64, u64) { - // (linkers, deleted links) - let total = self.0.len() as u64; - let alive = self.0.iter().filter(|(DidId(id), _)| *id != 0).count() as u64; - let gone = total - alive; - (alive, gone) - } - fn count_distinct_dids(&self) -> u64 { - self.0 - .iter() - .filter_map(|(DidId(id), _)| if *id == 0 { None } else { Some(id) }) - .collect::>() - .len() as u64 - } -} - -// forward links to targets so we can delete links -#[derive(Debug, Serialize, Deserialize)] -struct RecordLinkKey(DidId, Collection, RKey); - -// does this even work???? -#[derive(Debug, Serialize, Deserialize)] -struct RecordLinkKeyDidIdPrefix(DidId); - -#[derive(Debug, Serialize, Deserialize)] -struct TargetIdTargetPrefix(Target); - -#[derive(Debug, Serialize, Deserialize)] -struct RecordLinkTarget(RPath, TargetId); - -#[derive(Debug, Default, Serialize, Deserialize)] -struct RecordLinkTargets(Vec); - -impl RecordLinkTargets { - fn with_capacity(cap: usize) -> Self { - Self(Vec::with_capacity(cap)) - } - fn add(&mut self, target: RecordLinkTarget) { - self.0.push(target) - } -} - -#[cfg(test)] -mod tests { - use super::super::ActionableEvent; - use super::*; - use links::Link; - use tempfile::tempdir; - - #[test] - fn rocks_delete_iterator_regression() -> Result<()> { - let mut store = RocksStorage::new(tempdir()?)?; - - // create a link from the deleter account - store.push( - &ActionableEvent::CreateLinks { - record_id: RecordId { - did: "did:plc:will-shortly-delete".into(), - collection: "a.b.c".into(), - rkey: "asdf".into(), - }, - links: vec![CollectedLink { - target: Link::Uri("example.com".into()), - path: ".uri".into(), - }], - }, - 0, - )?; - - // and a different link from a separate, new account (later in didid prefix iteration) - store.push( - &ActionableEvent::CreateLinks { - record_id: RecordId { - did: "did:plc:someone-else".into(), - collection: "a.b.c".into(), - rkey: "asdf".into(), - }, - links: vec![CollectedLink { - target: Link::Uri("another.example.com".into()), - path: ".uri".into(), - }], - }, - 0, - )?; - - // now delete the first account (this is where the buggy version explodes) - store.push( - &ActionableEvent::DeleteAccount("did:plc:will-shortly-delete".into()), - 0, - )?; - - Ok(()) - } - - #[test] - fn rocks_prefix_iteration_helper() -> Result<()> { - #[derive(Serialize, Deserialize)] - struct Key(u8, u8); - - #[derive(Serialize)] - struct KeyPrefix(u8); - - #[derive(Serialize, Deserialize)] - struct Value(()); - - impl AsRocksKey for &Key {} - impl AsRocksKeyPrefix for &KeyPrefix {} - impl AsRocksValue for &Value {} - - impl KeyFromRocks for Key {} - impl ValueFromRocks for Value {} - - let data = RocksStorage::new(tempdir()?)?; - let cf = data.db.cf_handle(DID_IDS_CF).unwrap(); - let mut batch = WriteBatch::default(); - - // not our prefix - batch.put_cf(&cf, _rk(&Key(0x01, 0x00)), _rv(&Value(()))); - batch.put_cf(&cf, _rk(&Key(0x01, 0xFF)), _rv(&Value(()))); - - // our prefix! - for i in 0..=0xFF { - batch.put_cf(&cf, _rk(&Key(0x02, i)), _rv(&Value(()))); - } - - // not our prefix - batch.put_cf(&cf, _rk(&Key(0x03, 0x00)), _rv(&Value(()))); - batch.put_cf(&cf, _rk(&Key(0x03, 0xFF)), _rv(&Value(()))); - - data.db.write(batch)?; - - let mut okays: [bool; 256] = [false; 256]; - for (i, (k, Value(_))) in data.prefix_iter_cf(&cf, KeyPrefix(0x02)).enumerate() { - assert!(i < 256); - assert_eq!(k.0, 0x02, "prefix iterated key has the right prefix"); - assert_eq!(k.1 as usize, i, "prefixed keys are iterated in exact order"); - okays[k.1 as usize] = true; - } - assert!(okays.iter().all(|b| *b), "every key was iterated"); - - Ok(()) - } - - // TODO: add tests for key prefixes actually prefixing (bincode encoding _should_...) -} diff --git a/constellation/templates/base.html.j2 b/constellation/templates/base.html.j2 deleted file mode 100644 index 0a06610..0000000 --- a/constellation/templates/base.html.j2 +++ /dev/null @@ -1,66 +0,0 @@ - - - - - {% block title %}{% endblock %} — Constellation - - - - - -

This is a constellation 🌌 API server from microcosm ✨

- {% block content %}{% endblock %} - - - - diff --git a/constellation/templates/dids-count.html.j2 b/constellation/templates/dids-count.html.j2 deleted file mode 100644 index af690ba..0000000 --- a/constellation/templates/dids-count.html.j2 +++ /dev/null @@ -1,30 +0,0 @@ -{% extends "base.html.j2" %} -{% import "try-it-macros.html.j2" as try_it %} - -{% block title %}Distinct DIDs{% endblock %} -{% block description %}Count of distinct DIDs with {{ query.collection }} records linking to {{ query.target }} at JSON path {{ query.path }}{% endblock %} - -{% block content %} - - {% call try_it::dids_count(query.target, query.collection, query.path) %} - -

- Total DIDs linking to {{ query.target }} - {% if let Some(browseable_uri) = query.target|to_browseable %} - browse record - {% endif %} -

- -

{{ total|human_number }} total linking DIDs from {{ query.collection }} at {{ query.path }}

- - - -
- Raw JSON response -
{{ self|tojson }}
-
- -{% endblock %} diff --git a/constellation/templates/dids.html.j2 b/constellation/templates/dids.html.j2 deleted file mode 100644 index 12df459..0000000 --- a/constellation/templates/dids.html.j2 +++ /dev/null @@ -1,50 +0,0 @@ -{% extends "base.html.j2" %} -{% import "try-it-macros.html.j2" as try_it %} - -{% block title %}DIDs{% endblock %} -{% block description %}All distinct DIDs with {{ query.collection }} records linking to {{ query.target }} at JSON path {{ query.path }}{% endblock %} - -{% block content %} - - {% call try_it::dids(query.target, query.collection, query.path) %} - -

- DIDs with links to {{ query.target }} - {% if let Some(browseable_uri) = query.target|to_browseable %} - browse record - {% endif %} -

- -

{{ total|human_number }} dids from {{ query.collection }} at {{ query.path }}

- - - -

DIDs, most recent first:

- - {% for did in linking_dids %} -
DID: {{ did.0 }}
-  -> see links to this DID
-  -> browse this DID record
- {% endfor %} - - {% if let Some(c) = cursor %} -
- - - - - -
- {% else %} - - {% endif %} - -
- Raw JSON response -
{{ self|tojson }}
-
- -{% endblock %} diff --git a/constellation/templates/explore-links.html.j2 b/constellation/templates/explore-links.html.j2 deleted file mode 100644 index a023ddf..0000000 --- a/constellation/templates/explore-links.html.j2 +++ /dev/null @@ -1,36 +0,0 @@ -{% extends "base.html.j2" %} -{% import "try-it-macros.html.j2" as try_it %} - -{% block title %}Explore links{% endblock %} -{% block description %}All link sources of atproto records linking to {{ query.target }}{% endblock %} - -{% block content %} - - {% call try_it::explore_links(query.target) %} - -

- All links to {{ query.target }} - {% if let Some(browseable_uri) = query.target|to_browseable %} - browse record - {% endif %} -

- -

Links by collection and path:

- -
-{%- for (collection, collection_links) in links -%}
-  {{ collection }}
-  {%- for (path, counts) in collection_links %}
-  {{ path }}: {{ counts.records|human_number }} links from {{ counts.distinct_dids|human_number }} distinct DIDs
-  {%- endfor %}
-
-{% else -%}
-  No links indexed for this target
-{% endfor -%}
-
-
- Raw JSON response -
{{ self|tojson }}
-
- -{% endblock %} diff --git a/constellation/templates/hello.html.j2 b/constellation/templates/hello.html.j2 deleted file mode 100644 index ce75a6a..0000000 --- a/constellation/templates/hello.html.j2 +++ /dev/null @@ -1,146 +0,0 @@ -{% extends "base.html.j2" %} -{% import "try-it-macros.html.j2" as try_it %} - -{% block title %}Hello!{% endblock %} -{% block body_classes %}home{% endblock %} - -{% block content %} - -

Constellation is a self-hosted JSON API to an atproto-wide index of PDS record back-links, so you can query social interactions in real time. It can answer questions like:

- - - -

It works by recursively walking all records coming through the firehose, searching for anything that looks like a link. Links are indexed by the target they point at, the collection the record came from, and the JSON path to the link in that record.

- -

- This server has indexed {{ stats.linking_records|human_number }} links between {{ stats.targetables|human_number }} targets and sources from {{ stats.dids|human_number }} identities over {{ days_indexed|human_number }} days.
- (indexing new records in real time, backfill still TODO) -

- -

The API is currently unstable. But feel free to use it! If you want to be nice, put your project name and bsky username (or email) in your user-agent header for api requests.

- - -

API Endpoints

- -

GET /links

- -

A list of records linking to a target.

- -

Query parameters:

- -
    -
  • target: required, must url-encode. Example: at://did:plc:vc7f4oafdgxsihk4cry2xpze/app.bsky.feed.post/3lgwdn7vd722r
  • -
  • collection: required. Example: app.bsky.feed.like
  • -
  • path: required, must url-encode. Example: .subject.uri
  • -
- -

Try it:

- {% call try_it::links("at://did:plc:vc7f4oafdgxsihk4cry2xpze/app.bsky.feed.post/3lgwdn7vd722r", "app.bsky.feed.like", ".subject.uri") %} - - -

GET /links/distinct-dids

- -

A list of distinct DIDs (identities) with links to a target.

- -

Query parameters:

- -
    -
  • target: required, must url-encode. Example: at://did:plc:vc7f4oafdgxsihk4cry2xpze/app.bsky.feed.post/3lgwdn7vd722r
  • -
  • collection: required. Example: app.bsky.feed.like
  • -
  • path: required, must url-encode. Example: .subject.uri
  • -
- -

Try it:

- {% call try_it::dids("at://did:plc:vc7f4oafdgxsihk4cry2xpze/app.bsky.feed.post/3lgwdn7vd722r", "app.bsky.feed.like", ".subject.uri") %} - - -

GET /links/count

- -

The total number of links pointing at a given target.

- -

Query parameters:

- -
    -
  • target: required, must url-encode. Example: did:plc:vc7f4oafdgxsihk4cry2xpze
  • -
  • collection: required. Example: app.bsky.graph.block
  • -
  • path: required, must url-encode. Example: .subject
  • -
  • cursor: optional, see Definitions.
  • -
- -

Try it:

- {% call try_it::links_count("did:plc:vc7f4oafdgxsihk4cry2xpze", "app.bsky.graph.block", ".subject") %} - - -

GET /links/count/distinct-dids

- -

The total number of DIDs (identities) with links to at a given target.

- -

Query parameters:

- -
    -
  • target: required, must url-encode. Example: did:plc:vc7f4oafdgxsihk4cry2xpze
  • -
  • collection: required. Example: app.bsky.graph.block
  • -
  • path: required, must url-encode. Example: .subject
  • -
  • cursor: optional, see Definitions.
  • -
- -

Try it:

- {% call try_it::dids_count("did:plc:vc7f4oafdgxsihk4cry2xpze", "app.bsky.graph.block", ".subject") %} - - -

GET /links/all

- -

Show all sources with links to a target, including linking record counts and distinct linking DIDs

- -

Query parameters:

- -
    -
  • target: required, must url-encode. Example: did:plc:oky5czdrnfjpqslsw2a5iclo
  • -
- -

Try it:

- {% call try_it::explore_links("did:plc:oky5czdrnfjpqslsw2a5iclo") %} - - -

[deprecated] GET /links/all/count

- -

The total counts of all links pointing at a given target, by collection and path.

- -

DEPRECATED: Use GET /links/all instead.

- -

Query parameters:

- -
    -
  • target: required, must url-encode. Example: did:plc:oky5czdrnfjpqslsw2a5iclo
  • -
- -

Try it:

- {% call try_it::links_all_count("did:plc:oky5czdrnfjpqslsw2a5iclo") %} - - -

Definitions

- -

Target

- -

A DID like did:plc:hdhoaan3xa3jiuq4fg4mefid, or an AT-URI like at://did:plc:z72i7hdynmk6r22z27h6tvur/app.bsky.feed.post/3lgu4lg6j2k2v, or a URI like https://example.com.

- -

Collection

- -

A record NSID like app.bsky.feed.like.

- -

Path

- -

A (currently-very-very-hacky) json-path-ish representation of the source of a link in a record. Records may contain multiple links with different meanings, so this specifies which specific link is of interest. Like .subject.uri.

- -

Cursor

- -

Paged responses include a cursor property. When it's null, no more data is available. If it's not null, you can repeat the request with &cursor=<cursor> in the URL query to get the next set of responses.

- - -{% endblock %} diff --git a/constellation/templates/links-all-count.html.j2 b/constellation/templates/links-all-count.html.j2 deleted file mode 100644 index ed85159..0000000 --- a/constellation/templates/links-all-count.html.j2 +++ /dev/null @@ -1,39 +0,0 @@ -{% extends "base.html.j2" %} -{% import "try-it-macros.html.j2" as try_it %} - -{% block title %}[deprecated] All link counts{% endblock %} - -{% block content %} - - {% call try_it::links_all_count(query.target) %} - -

- All links to {{ query.target }} - {% if let Some(browseable_uri) = query.target|to_browseable %} - browse record - {% endif %} -

- - - -

Links by collection and path:

- -
-{%- for (collection, collection_links) in links -%}
-  {{ collection }}
-  {%- for (path, count) in collection_links %}
-  {{ path }}: {{ count|human_number }} links
-  {%- endfor %}
-
-{% else -%}
-  No links indexed for this target
-{% endfor -%}
-
-
- Raw JSON response -
{{ self|tojson }}
-
- -{% endblock %} diff --git a/constellation/templates/links-count.html.j2 b/constellation/templates/links-count.html.j2 deleted file mode 100644 index 45671ee..0000000 --- a/constellation/templates/links-count.html.j2 +++ /dev/null @@ -1,30 +0,0 @@ -{% extends "base.html.j2" %} -{% import "try-it-macros.html.j2" as try_it %} - -{% block title %}Link count{% endblock %} -{% block description %}Count of {{ query.collection }} records linking to {{ query.target }} at JSON path {{ query.path }}{% endblock %} - -{% block content %} - - {% call try_it::links_count(query.target, query.collection, query.path) %} - -

- Total links to {{ query.target }} - {% if let Some(browseable_uri) = query.target|to_browseable %} - browse record - {% endif %} -

- -

{{ total|human_number }} total links from {{ query.collection }} at {{ query.path }}

- - - -
- Raw JSON response -
{{ self|tojson }}
-
- -{% endblock %} diff --git a/constellation/templates/links.html.j2 b/constellation/templates/links.html.j2 deleted file mode 100644 index b293f02..0000000 --- a/constellation/templates/links.html.j2 +++ /dev/null @@ -1,51 +0,0 @@ -{% extends "base.html.j2" %} -{% import "try-it-macros.html.j2" as try_it %} - -{% block title %}Links{% endblock %} -{% block description %}All {{ query.collection }} records with links to {{ query.target }} at JSON path {{ query.path }}{% endblock %} - -{% block content %} - - {% call try_it::links(query.target, query.collection, query.path) %} - -

- Links to {{ query.target }} - {% if let Some(browseable_uri) = query.target|to_browseable %} - browse record - {% endif %} -

- -

{{ total|human_number }} links from {{ query.collection }} at {{ query.path }}

- - - -

Links, most recent first:

- - {% for record in linking_records %} -
DID:        {{ record.did().0 }} (DID links)
-Collection: {{ record.collection }}
-RKey:       {{ record.rkey }}
--> browse record
- {% endfor %} - - {% if let Some(c) = cursor %} -
- - - - - -
- {% else %} - - {% endif %} - -
- Raw JSON response -
{{ self|tojson }}
-
- -{% endblock %} diff --git a/constellation/templates/try-it-macros.html.j2 b/constellation/templates/try-it-macros.html.j2 deleted file mode 100644 index a9da1f0..0000000 --- a/constellation/templates/try-it-macros.html.j2 +++ /dev/null @@ -1,52 +0,0 @@ -{% macro links(target, collection, path) %} -
-
GET /links
-  ?target=     
-  &collection= 
-  &path=        
-
-{% endmacro %} - - -{% macro dids(target, collection, path) %} -
-
GET /links/distinct-dids
-  ?target=     
-  &collection= 
-  &path=        
-
-{% endmacro %} - - -{% macro links_count(target, collection, path) %} -
-
GET /links/count
-  ?target=     
-  &collection= 
-  &path=        
-
-{% endmacro %} - - -{% macro dids_count(target, collection, path) %} -
-
GET /links/count/distinct-dids
-  ?target=     
-  &collection= 
-  &path=        
-
-{% endmacro %} - - -{% macro links_all_count(target) %} -
-
GET /links/all/count?target= 
-
-{% endmacro %} - - -{% macro explore_links(target) %} -
-
GET /links/all?target= 
-
-{% endmacro %} diff --git a/constellation/zstd/dictionary b/constellation/zstd/dictionary deleted file mode 100644 index 106847e7007b4ac409fd58912fed85d69aedb352..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 112640 zcmXqCV(=!{O*BDG-~!hqzW*T5Jk#gUiDfQbJ}pxwiOl@Lf7aZ)EO^t-8w}jijK-`C z3=9m14kQ~GOgYX}%d)_sjh&^vm5G_9nW=j^D_hS54t5TnCWinKu@<(48xjmzN_bu} zSu!v%q?}e{Jji&MJ)=p2g^_{5Q9?mVpn;izfq{{Mfq{jAfq{d8fk7`VF*&tFIX5-0 zBr`uxNk=IqGeyY?L|PT(BwH1iR~eb7mZv2qrwtN|C5c5PN>)Y&MztW7If;4c#Y$GuN~w8Du{ug6sTCzk zR!WZ>`kYGh6nyf_GV{_OH}rFbXC#&=WF!_8q~;YXlxL*oDI_YCXXNLk7U$=tDr6+) zr0FIVrKYB&7AfSWrj`^dlxLP?C?qNr<>w~mm1HI>B$ejbD}<-!2DrDrBC?w{kC={2cCYDqx6sJ}wWELx=q^4!&WtODoR4U}>DU@WSDkNv5CTHhl z7MJMNaXFWkq$)U-=2M3WNJDvMGxGb%E&GEJ&V64O!>lZsN!jdF776El)aO>%Nglxl0aszFhL6fv0<1?eVcl?7Ef#@VH*iRFpq z#$^>56{$)(N+p@Osqv-7N>+yEMrIZk2Igj_7KRqaCgwUyC6xuKN>)nA`MF9uATl?z z1QdTosbxx5O2#=Rxn;&FMOn!vMj0S;po+mtb8=FX!70Q_DY2jc;>EPo)D%5X;40}T z6=kPZBAJnxSqe6zC^b31C4P(%r`G9F-tWzO0Ot3H?J%% zO--uGPs&U)Hcv@S%c)8$D>pI8F;UV{DlN)XvQkPcvC`L1$xH#K_o__e>=NV3UQqI^X~MMZmih161ol2nDn+PW=Z)4rDo-+#(7Ce#b$}cMJbv2<>tnP1r?PgW=1B(sihf}nW<(OmDs$W zl#*R(Txp(KXlhcKQdF6kXqH@7VpNia&HJVX=7tt#mZn5`zbrkcDAC9aWDdCr-KZ=* zy|fZ+2IUFe)XLOI&%g+8LeD5kFDx@EGd0f3GRa6XF*hnM$jr;hHLomAE6Xr9D$Fn` zEXgV_EzU|WDJsd$HA^%p0p)Zo3B97I$TZ6+KReGnwcMzxEV;b6qO{mJJqcGrFHOnF z%goL(D#J|Z$>wH>rr9QCM!ES(xke^t<&~wyImJ1p#VLjP$(81&8O9k&MwMpS1?f3? zrDf?AMLAhm6S`SNu}NxHWrmS)VPC#Hm zTr<-gqe_rDWX5!9rD;x8QVQ4%%46EZ%FtZTz`zn;k(OFemYMnr~K+lu~JK zQdVG`pIVZbTTx(MY*w0AnVFW8S7~HYQIVceU}BV!3@ZDv#dKw^NkxuHNlH#yWm#r= zWs*sONs@U{9wer%3{0$y%=L_ojqtf9xiGgpJ3l!mtF)>hH?=%5r?@mVxg@`=FsURf zs~|Z!!>BMVC#BRhuQI7LIV;nstSY?_djXqTUTI!sRG3m>T9#T_k!@UMR8*c;T2%&d z4XA1b*DCM=wzML(ptL9zTe?d(E;lbUs;VeW$E zWyTdrMtN0b*%f(}mBq&9=4Pgqr5Wa_n79(Aw8A94+z4DJ5h`KQixLYm^wRQka`MYj+fAjFrR5db`Cw!4 zL^hfMc$!WoRwl-J2F9p03aA|kZt7$e<(8D^ftpyRR>lT;1_pRbfDDs@q{Ni+#LA@X z^xUfClBB}a4AZhoqmtB|f+F+c+(MI-qQugyq_V=4f~>;a(p=-*Ozb5q zrU(NAJqrsHd}V#Qd4X|KQC@y-X-0*KsY#AWWnxycNt#i*SwUq%L2+?ea#ngtMqz4x zd1jhvL2-JPX$Fq6-l!-mCAloas4A}@C!@SPKQk*eC!?YQXJb0GINQ9cFxSW!Go~~0 zvkD3e(+di6vyDsBlk(Dwvr^O3%9C=-@{CF6w|D;*06z?4*L6{N!v<2M=Ed%*a5`z!0_8fR`dkrlv;brR61A zspSRPr5TxNDaA#_1^E>i{eh(Hg2c?Eg4Ck&;;Ouo{H$!FjN&TOiRv?8snDm@cy z2BA2BmbQ4}z!=mgGqW@V#R0CthorP*6VsHEybQC-^c=JDs)BMee)8edr^GwsS>=GlRG~=wyBJ=$6^1?LZ)SRr0(!{(%vr>}+6XR5@Jxr6_tn!M2 zyo~(R#O%C^!o;NVj5MP>Gpwb*si}#fk);7~Ln&2d#!1;mplBtSZqb@=$m3>4Riz~v z5M%HpT{Hvmq+U}iQ)4|tW7OInUUHXK7-yTMHg7*}RxBo~_$mggqsCYfQB8Y%fn zS!Mb8CMITP1^J2PsYWL0899Yn$vIhh8TpkJ#+gNlNg2t;nUxt8xmm@zrbVgXHatcv z%fvLLFfp+(BPTsSGdr)`+$1ljJU1f~dq2z45ImGcM9E!dT$quak!(~9Dt*XoWr55{ z%_#<(fhXvRY-O2Q8CvLBni$|KHBw45igMC3l1)-f3QbbXD~*$KOfpPOaw<)X)6DZy zbMsP+Q;jN3ij&ffOU$whs_W53;)?8Sqf&4_Gc7PpG|$h?F-^)#Og1V^%1+5PGs!AT&&VstE6z{OOEXT&Gb_Yp zb$)tLPI^&peo|hkc}0?0VNPyRs!3)g&MaO~TAoo_W>f&S8g&Rax3HipH`y$?EXT~W z&@|t;B+1CABGo7@Ik7k=CC$9b*rYPiD61;RC^OYG%dD^<(HLt}-aIwMthgXAwJ5)` zq#~ubyx26|v@kOTJj{nyHJO{48yFgy5H*NYW}I4PUQS^H&p5TrsH&7o4Lma|12a8y zBfOoq^i1QT(!7$AvdW6WjNHuPjQq+x)579(qly%heB+XmDl?PZ^wO%Nywb|7yt0zw z%=BbXS%@W(6_uu#ROV#l=BJjJC8lR47i8uZW)!EV;*9AW)BGf}B%@R;F<*Qf!i0T9907QdVLLnm{NmLY?lbEY2=0$;?VgF)J~y%uLV9Of@#oPfh_3VWH(7 zGh;IoLo-8DVk*eeipV^dtYC$BuY(!8h)+mJ_kNm7+*L20RJeo|?Ea#lfE zR&hpt0jS}em2X~@X_Q$|oNa7ml2uY#lA2SJnwwQn1j>ULxhFr>s3Il3vY;R1roPYy zJyQ~k%TtO|le5k9^E1=)a!gH2OA@n;3r$KgN|Ot6i_*$Ws`8A}it|&AO$w`WlGD?& zunu~rniiC185~}WGRrnCNK4HsEjGF#2OGeIZ2fn1;)u4rsZZu*%|qz1^H!Z>E+<5ZuBH? zY+z()Vs1=Kg;83OoKawwVq^-+I%Kv#L1tv6Ld>8%$s1dl8t56A;H@yy%uPzn3yl&@ za>_GO3oCB_+L$;rmK1x2MeW4gSeA}I&dl&VFaGEd2`%t|WEGfyqeN;R)4DoZyt zt4K9X%t*{kOv}ix$TKY|Pb$mKugoyZH!3$xEz8d=#M%hVt1>puFe)fW%&*KgEz3*J zHcKo^F-}hc#SmIdo0}P18W|Xy5;HzgW}H+}oNsDW05XSAOrv$vkZLlJF*zk^U}Nw^ zHktu=`gvwn2F7}3W~ieS@P1x-Nq%l>eqM@Ek*RTFYGz_`Mqy@om1&j|IP}rhKP0A< z=2s+^m6_z_B&U|-o0^oQml~&K72%A`ssiJbBqO6N%xs&HR#;h{TAWi-R#Z@0k(*(f zZJt_TmYCnpTpOZC;p|ZB~(;T4o zxu`O?AhoIpYdg=v#K6$d+?=?`EHln2Dl$qTCNgofNy?1Nj5A6hX5fiQWZih9(a2cO z&qLxYRhcxV*?T$tXR)GOHrD$TX`UsjQ?tBRL~I zu{^B`)W61(Z3`>&vx`#`D^rcqb4zp5GfmRXtBNX8DsaSfdbvqnib+-pwA+Z@a zHZe6eDabM|DL2k4GtSPeN-In+$OU--V`$d^6w?;QM72rEvof;sbB#dbRAjeF%Cj<( zOG?3JP#)99Rz^m8mX;Ry1_&}z%F@bms!|J+i%ZfHOH#^ms|rj~E0c|r%8N|Pa?GpJ zO)8R8%yW}7&60Aea*fh6^C2yIj5STg1tulQ<%LyA##LFBrs+A^rG=U48EGltmb=sY)#= z$;r>nt1!tar~-vImV{SSnx0mbmz$E2oLp6uQJzv#l%Hafl819#FuyP*zpx52ri9)^ z$}3OFEGe%r&8Nlhs!GdDHLG%+D{_j=&C&}p!Hrn7`ohA%#MIEp$dsrR04AAvnMR35-~>#l zzCbP{kShz5%)BJ?Ot2Z0Cpm{(AiW|W#y zRFPkrXUSZfQkgW_C(uiBWQ9 zO0sEsVnvBrftjgsKK4Na5=MYcEltb}jg3u+ng1&-Ps}VU0IgC%^pSART%c9#NRbUP zrl6$M2(%ypPh_JRfTv0^wlXx*Gc&_60$g5Zl0{%Ss7Zc$URi|^sAM6x=qxkIPtP>U z0GolwQ^@HbkB`hiZCgVNyko10g~e$m#wBJ+#)-xzd8wJHDOp7+=|<^Q#sx;EiKf}c z>7}_D#>Ewh#yN$VY3aE|=82#ziX{bCRvMY46qn~$nWSfBnHQ(0rIqER7iK}%PGMQC zRGOYrVGe2`Vzh0OGLy24OY<|$Gc)t^5=%;ovI=u6v-6D$6LX3RsMUFlHeuZ-~hDPS*#MDZqRVgNwWm!g`fGDX{|Yz3dpDuu%<6>>{6OY#*`a|`lQQi~E(5_J>`@{0;fQxy^w z(o^$FQsGiCl{twz3VEr;#i>B4G7=TiGIKJ(UP;c+OU^0HEY(xs zQgAFT&MYoTwBqu~FG|fzRPf9#D9SHMRLIL$2+B{+R>;g#2+9PF?IsmvrleQ;7iFer z<|XEY7Ud{eDP@$D6ck(O>ldY#fea}uP0iIy&&f~9*GtaN)lV%;%`3?-)-TFW&eqM$ z(=E!(*9T>;rl~n*rdj#PC23iv#aI^tWM||V7n>KA8E2Oy zrIZw=n;8|R6zAuIS^?-qtGT(Mk)eTwB{4&RWyYB$=~ao~c^|TyOU9WcNv4HhGbpb_ z&8!S8^h}Mwi->SlqKTEpRcS@#DT!Id85K#!+1VLM<&}v^rKXh`W%=nv6&dDLMyZu$ z$>lkw=DDUO>6PZV#uQSrD$^3P)68;BD+(%&i;QyiqPf`xi3QoEMc77< z60>v6GAfcva?-PkOv}nM^Nr0-GR$(aOEL;GQ!6skk_!s+(oCyLvT~~O(hJJ+3kym> z6WSP+XjMg8es*PHMpc=yQGr=nN=jy7UU^~`_Ca5BOG8sj;+Bh5WEGX9BpFp8noFR8 zq-3m1mC)Mj@CLteQfYp9QDS)^c+?xi=|xGE#TgZOrfKPgDXHnnB_?I2d1i%K;1z0U zy;3tXBXdI&L!wrzlvXBXWMvqGhjU9RGc(IUD{gd@it_U@H&&$-XD1h!m*ksdb6?=hC#>ld{8@~RG}cXD7`eVBwt4XRH^5JiVRRY zq%5^aq4@C1qRdo#dwY9@ki!c>1yylka$+8&jt1AUpbbwsCEx-TqzSb93ACTEI5!`o zyh$o5Pc6wSsH#XU&o$3XNzE=a%E(VC&QD9s%PlHSG)gJUFUl}AN;63*PfE*4HBZS+ z!I}z7vr7wd3X{r8jnk?!D=N)W%X10}l1#BirIER*p^>qv38E*7bGKe;g?VY3Sq^by zn7C4*NiN73vy>9BF?fm`Gy`zex`tK;##ScgdWIILBbo34>8ixk%<_sVQ?r~xGt=Cv ziq!0s(hRfWWX#Y{PR&j&HA_iPEy*j&uBu2WFDfmmFf-3AC`(N=POD0-Fv`g*F{`LD zD>o@AGO9FBHNiT(lUP|^Qc_T!S6otBT9%S&npu&WpIlIkb;+oqF{smTZbVGzmlx+I zmZln|Bgz2Kh!JUJfJts?d2z0Br3u&=DuuqWv7V_BxXlAE^WmYNUSOVIY*ta8SZZ#P zl%J|poLQBsWMye)X>M)-%4wM;IiN9t;KK`(5>xUON>YmsF9dA_R7lM$DN0qyD=hz1SzL3p~w`8m2^7wAGxrM7MJGe8tNIE8k-uMLl%-_Z`1fI_~jSnrxur_>VUdEsl^4Upe?%aEK{tdI+8{)3vR#X1TFMTa+6WP;lL`K1bp`3gm; zDfvYTDfz_;#s-IWIb)>fl$6ZWr0nFZ%oOv?yyX0nx)Qht7AD)vFFtRmyW^2CzDBIC-+^s<7?%*+gvoT40Ttrs%`V4D$vmN(*u`Gcq$u)AN(cb5b&^@^ULO%}q@*L8DoaB#N^4H#s>a&nUM% zrL3qdy{f`Iqqv~TxUAHy0$1zBtT4y4GOx_21as~yrJyRiDy^)nyr9H5rM$?vAhSFz zB_%)AEZMB8ILFv1BR#LQEUzFdx!fqZBD1O_IXMsODh<=rg5pARlaxFoqqH)!oWwli zN|TbrbZmn?<|Zb_h9(x4#LS4485NiqnU{c?0F|H)Q9)u+Y940opOKrDmQ_|#m6wu} zSy7czURaotX_}N;S(cKWn4FrPoS#!^Y*tX7o^GC(S!`Thl4+U_8dSg%v}R_>=4F|Q zX_aY7na1Xod4(Cdg(Zb0<~V{j+bG>At+=4V2wW#%Z<*w!W|o)c8Yd-{XQw0;m{t{* z7bX^#msFKimYJC6q*Rs`mYWslRuq}#n5UZ?C1vFr;~1qjDoV>t&&W(pD=#U|GBr=k zO3MSSmBO`%F{?P=tje^i2)xD>(K0Cn_3%@Za}tZ7!zn3=3I#>^1(}Ho0d5M#`AJ2o zR$Sn5q{M;({eq(Ww9K4T{gTS!3_U$PjFKX$z{ohaup~Xryf7stts*b0sv<4DusAoT zs46kNB*Q$@G`G|^JKZcRKc}G37&JkYjkV@WNl!}8s>;qSDNC=)FG@}{O{*|Ut1t(J z1bRQm)Y8)2(9q0?xRN3*)hNdtI)nr{Qa}lGpa3Zg^~=*zN(&0WW>8*I7=xGGLi(`q zIuC7?Pr7kRc3ELYqG@_oN>WC8hFPVNk$Gi#N=ja8L271}aambzu2EJ&K}uO=Syf?K zQfWmBs7S?C`D>oNm;&WQDu2)zL7~z zSx!k(W`QD$0IO1g=$QBiVcQI<)0VyUTVa!FcB9*#1!qN>8&xGX8JxWc3& zu{0??(=-IMQ%z-VtG}nQD#X=W_DIuSwW?7No7S+ zZenJ6W>reQsZphAdR|#(QF@taMm}hU4a-RxMdbx0Ny#NeNmUspMJ1*w6=_vwCOJuk zpjI1tH_Fh+%+So3sPzM-{SxF_C zm3c`fDV3?_DHX|;B}K*Nd1hq=MI|M9MLEeP>8Yi;#(6nOm64}lVyl@jiDKDcEAnk7Kp8U`}MG$R>o2BDaS>c-Q*GP5!^*0Zz(FYwBN z9j$}1HSEdmu4mJizTY$X+03TLjjLa5-Tt_-_i^L0$Bok;H|~4fxcG77{Kt*mkDH+9 zn3N_aW|ifES_Bx=8ObSS$t7j!Da9sLd1*POC1uG)1!g6QMR}#=X~q>fC8ZUaCPnE< z>1oDQrFjKKIeEp{V<*4J*gVZR+oU9;GS55(w7;psq&zJX+enPLp^2%XiKRJl6IaD0 z$)=D^cVy?os^a3xv<&d(KSHrXxYcTEWoW8rXl#yebw^5Cc4lfpRgPIkVNzjsa%ECl zYFb5#X@zM;VR1>NNqJdmR#sVYd46RX*19UdvjB~F`bm1o@H8IUYTf= zQe0+gYFd$1W?q$ATwGpWVU%iCmYQXpTA5pxo}QRhnV(Zqnw^tdf}?ItO*YOh&deyz zDmO|?&nPmkDk~_-NG`(JKPjz9H#bkoFsi_e>7>NM!ko%t(=-zk6FnmXa6y8z*_4`N zmSmi4o^EPdnpBWknO2^iQ<$G;YF1X9ZE94STvl38Rh(00mXlamo|9yhmy=t7quErR zTVS47nqHQkkyM(Qo}ZPOo>fwinT2zBxU@8>C@aGR+cNmn{KT~Ml>EwyETe+Na`W=! zLSwVk!kmh-#DWy_g3?SQv*gOOGE=jnvXo-uWYdg_#3HPV3rq4+a!o7qO;e1_E6t6~ zOAF1*@{E#;KzR*4shU|>T7cS3#4Ii>tujtd%%HFA6|OC1r^w6-L>ZNi{t$Gpne| zsMIJoEu*A7)7&I2Eu+%BJj*yKqr%8Iv%t7Azbq>;F}J9!%&e%e(kM5t9P5;iscE5! zSxHf*SxQDmR#{bYdWm_qafUH?EiYQ*z|6qV(9qb@h^Qgh^7MiN(Y5mu*{6_uwLm1JiYW>@6rWt3#4<|O5rSEhj)A?S(J+}PC6(9G0~sCJD} zdbTO@QEMik)j}4Qc(+O7np;##bQY#CMIQtS(%w>C7D&mMxbUvc}i(nNixo^LTRy)NoINu zc&-F}9iVxVX+e34QDtU9MV4_^PF`MWRc?Atg_%iqj&WjTQ7UNDLwcflPGM0=S$>Ln zzEKI*wKX}WX=%oVd8TP)X=R0_C7DH~rG?39*`NqT4`?$JOLIeWGvWr;DoU%W3d_KY zN1E(HaCdsCzsYTgY zMU_cLX&HIuSd)2%QI=^^N?B5>X>LiHS%y)5ijhf%aSFDw*xbOv*wD<#z?_KYdudf} zMOHRsxjn(q2RGlLO+5HKok=dpj8y2#e>_2ttQ$}B-4wK5-_Qi_ux3hSc6o_$Qd(k_ zNn$}-MMZj6T1l2^zIk50agtGDv2k90MS5ygPI^j3Nm7wfsYzvyF{nq2C7%_faZH%ltaEHFtkGA*qzFUZJA%B#$* z!qqk{EhsN3swl;F08El`MOHykN=j;Big{XkT1j42MrL|WWmZaxsZnJ~PI9?%PI7Xx zX=ZUrR;F21YJsU)2G$YP?9|+(!mMoboD|cXJo8kuBGVM(l!{!eeMB>JOG6U_6QV{` zjnj%tjVr->I*3bT&|V%=A}a%J=1R;2n?Wd%L3QIvWZ?b51}5Oq0C*n}WmQLZVo`pM zLON*jG~@8flGLJndyKrDUTU0VT54=+oRU>#R#lo`S)QJlQIThsW0ad!l~`JmZ=PRR zWn@~JR+?p8lv`SsYnqC+OJG!4WL%h?kzQ(&o1I%+SX@<=nw^zb1Zvx$m#pT-CI*J) zrpClVi%4Acc{1oFVlcI`}43q2%lj4f>6CNnEAPERi=GJ+ggnpjzoRFPqlRGOWilU7ht zl%JHGnwe8sm0?y=Vw8(@ca?E!L5@*crb&KbdI{)&5R=@*s@(iU@cuXSmXxWnrJ<35 zIWcirT2WbLY*vcglEPg}BA2vCFBw_8yXXcw0mFAk3q*j#YW||Zh z7?)OMC6^^9=b0xJCl?fz7UUHcnUxt;nI`2X=9;D%m*s&rpkk~*Nlr2{(S(cGsVxE~&R*;r$ zng|*x%d5!AG)^wgC^I%LPRh(Pt1!+oHOr_n%Q8zWG{u_FjB*PzD$PqPQ$by?^hDFV zJmbnl({gO5(peZ<7#W&am=V)sDKja{EioS7=(2SW;k= zV_aBPo>5qpo0M*xl$e@kY;Ka8fweuJU6Gobot$o*R+4IFo>gIgOHD$hzQ%gQj#$SN#0DmP0uH8(O(Nh{AU&Pgi*1r(N& zrZO!huPQCEG%F=9vnbE3*rY1o*wnZ@6<3X6USSG4YS<_pOYgkgEIBW;ASbOP$K2dJ zKhrcVu_U=7Kcz6KATJ}&xU4L*A}uL3E7LR|l%|Ij3V<~)3S7lS< zPQEH}1QATl6RaK=?zDZh;asZm*BQhr`#sZnYYXowd*VVj$nni`rL5jE^o zW>lD7o|10_T5UyK5r(bVTbN#$X$CffP_l;V#*?T`txQbyj4Z&NT6lj0a@|8wYC#UZ zbtXv_xrtRp=D8IaX?c~V##yGxNv0XeRhjvPr3K~L=9%d^IVQ!G>1HNL#ipt0<)!H< zxca)e<>h(CMW#79RmEwE#l`vMMX3cvg{3%894)OVOEE7k08KyRNc)LZ6{g8KW%-$D zl^G_f#+gNBxygAMDds5|$=O+%`T3O@$*K9d1xeY)1xCq5Mv2B{DLBSNNp)LERdRA! zWv-DCa@u#yQwYsV&Ph!yQV34YNKGltNma;CQ}E5tOHWNLPE{z+FU?6&NJ>>GE=>mA zjFFU>lapGckds-GT9jB+sgRdnmYSnb0$$_?-v1B2^`W%VJg+>z5`2UO!3WNw;SU~HO{ zYhqrJmtI+xmRXdBqogq}%_%L*$V@iLPb)|%ODd~M$*M>*FDbxL(&Uy_q!g#8lz`3| z$B`0JlalfZ@=6O#P4lyp64OodD$0_pj4JaB%2V@9jq}R$vx-vlOH)kp(@IUsj0*E} z^7C-izg4A0W=2I7MpeeSrI`g4MwzLZ$!Uc+_nPJAR+X4nm6aQrqof2!IKXIuW>ln? z7Z;gimzEWnres$YmK3I%7L;UUnP!(3RaK-FXOH{UACN? zlx1FAlwFXUn39{4V_cS;ZJLo}l83DYYG!O`X=q|X)M>e;6{*IBW@W^+ztEcRDAjVR zac&aC7(A&K%>X<_m@#;c#1hA{<=pI~;*84dl%&$+in8+bsd!PugpkJ zO3KPKNlMKvD@rY_$j>Xt$garDFDNZfF32dzEjCU|E6gv?$Sox=BVZ*2RCuhGxd524+OHevMMh^HZz9$Iz8jLiYcg80i^8 zGA#HyAkbAk)k!6{Ri`&AVVW#lzV~<4GmH634#1loB&RtNa!6^0Po2?sOEAQZus>ixd(S zlJb)i6;i+(40BWSQZm!?6+o-`K^yFnK|2b;3wet{%?0qLzf^_f#L{B${=37A@`^JP z6^g-I^zxI6GSd^m`;kC92$WIWk2ckinx3AVm}XQ7y0{=CB`>Gkv?M7fH@zq|v9hSD zDmT?UC%vjNC#kTg%qTe}KfT81Hq`30#t>8WPLnU%%KxQhLflGLoCiejT;aM6mg{wys$ zDKEP)BegKgJg=~*$~@P&FfT1VyRyRAG$lJDr?k>6H94a+-MF&Q*tFcVs=%le`=PT{ zrWyHpg%ugbMuqu_iK*$CnT3UBS(#WzoDB^u3{4FUh*=F$T2+uzo>K(wlt5d6pm{3n zlS<&Yf_F_2WoTL&`uV>Z8Ac^Z#pXt-RjKJ|skz4Km8N+a=0)a(6@@uvDMh)t#wN+8 z>ACskDdi#2bl%JlSS!$ALT2_)( zlxB-yQxJwvopH_*oa6ElqylZ+Eh za&t`*Q;W)zO-jnjD{~8Ts!G#~lGC!2Q&LhYOskC2j0&OtK|a8fRsc zrxd0ql@ui>n-`jtWM`KdC*#a#MTwau<|*J^f3-SFkow=yM9;_q?>Kk5QE5q9MOCFy ziFtlSWlCjwWl~ypa*=7ac}ZeMT6uQ4QCfa-R(YCfVMSTDDK7Vtcq>FsNm*H9 zS(=eiK4uY}l$vK!U}kKRpJ$q!n4Ml(k(5@HlW%Hfnwy!ERFPerT9sawpIBsUYMy0Y zlw(wAY+{BrWf~P%6dETQ6%>`3Rwm_D6;!3<78#j<&v-y+jhwPMDRaTyE23clIdCD{e-8^M#g4)7GOPQ&uWmy#&X8B1eiDh|Jrm3a*Ddpv5 zX^9yX1*T<1C28gvW(65V<+=H3g_(s(Ch27*xRwZK7ABXOB$|~MCRLOfr{$QImX?FOrj29h3m&#CK?&&nOmaHETE;ljJ&eMtOC;t@pn9z|6wRT%)RTqcrp6V^Wz_oSl7%)(g+&Ey)-E&w>&>1G2PT8H?gcNFA@8V zQh7yHRmo;mIhiT>iIwF=plj~3%}sM~oj{sXoReglpOp;ST?QTlLpoq0J>T5K$h@jB zqarcM%q-P7vC=#@%hWilqOveKF~1@?)ubRNy&$o;+^9G$DJQKmE2|Rgdk3aJ&kPEJk= zsS25SX*s2-dC8fHDWI+QDVgb+C5btZvqn-fK_~l^q!xj9%9drOB!bS#QAkWmEh+&W z8&m-rG)~k39isy}-liC2T5^7FehTym5YP^M$YCfcsR{+DIjMRI!HJnkpp#xQixo-| zlk)R(5)}eV!5#L(QqcJ*`FWsjI(Rpo0_f0?g8br4$mvBX`3i}}#gH=rKt}_mRutr9 zCMQ}c#3;FzDkLUnKqdnqr{#d%nss;~Xu=>VH5ck+Jq6cNh4lR567az#DXEziiA4$} zr4W-db3x+{xe7U;Gg%Z0iV_t-yN?xeOY>6l^%Me%KnL;UD&&F28V;{iC;`!FhZiPi z=73IfNi2fw08~g+NKDQH9ftxwHYv4KA+IzqIZ>e`vn*9l!4q_{Pg;JaLTRo-UTXT` z&B>V%M<*ud=ar-;gOr1;0iE{)@_`QMCRUIyN>V|`uoxLbj(bT-RmcSGq}G8Pc2KNP zkXl@vpIEGrmtT~dm;;_JQOL|q1fBJh3m#a>1X)~`nyZkPsF0Wfx^6l7@G4N8>nVWd zT0jPY?s-iuR!B@K%_&I)MNV$g;gz`xpp&3-!JDN)X6VHzDTF104tz>f@X0So1zD+3 zTmrgvIRzAvARADQ9#RNOOiN77QE)CTPAtjS4=hd11;tk$B={g9QIKDnoUf3QnRa+B zC^Qq3iZY8}=T3n_1QgPc<7LV+ixP_<$LAzMCU-!AlbD=eTvD2wnh8m?kn^9wu?jxa z33`GRIO0&{z{vqD4o(x`(9VUY;e5!166nC2!&?-RN{f@h$ut){vy@l_3W}iA#GK43 zuy9FHVhMPnDj$6KO};`R=%Aj&ybP#d?%|awnaTM&;4`p5v7HExnB3A7(9Uv&#GFh} zv_W^4>gp;KrIuuZcD#dAYH=y(2JDi|Jkar3$r<^{`8kQ8UF`+=MGC1EsksF?kOQ(H zsS^}s;Pa!B^K+9j^HQqPO*8ULN-Gk}3o?s~3QH=g%F`2#vU76F3-gn6t8&bZjkEI0 zO>9_Vhctdf$dq|}m(+*D(e(xS}5?2HPtl+^U3id-`zcUR<= zSLGDtnCB$u0KCMO#wX6NLXY`u zYGj_AnQKyDR$`WFUTj)Xl$~ajmsyfwUS(`*oMe_;g>_9tNk&eQNkwL|c~NOrj%iU! zxp_{RsaYQQ&MdTEw7Hq3k)g2#ahq$5GeGlK;06$)%~4B}t~H=4P2$W;r?LRh8KpMpY?BRTajG#udid#>R=s z>4^n7#d+Wc7nac~6VtR3lfr)N@PAsw$IA3b9>flaga% zVp3_A2wLToS(=n?nwyi7ZJr!$Amlb3s z85NnDWaj1;q!m}D8YP=o<`$V^TQdpTvubE!LEKrT6LfJgqX$G0%mVL3zS5 zu`;pLGcv*36-lo!E>BEJGc7MN%S<&lGA%LBPEJh9Oe#q$NKH&J%}>osG|4kbF3G9N zG)YRzN=wc#!7&VKTAX82n3|E4Wn7q)YFw3*Us03^%C-79Zj>+0H8x4fsl>KdJSjQR z#I(4oA~Q87Gd07!Dl^F>udtvr*}Sqm$GkKr&p5~2q&Uf}B+sn0q_iqEIlC$gYva_| zyf`(@#5^x4tuWENAU(@G%P6h1stRi-*uuyVbe0NnYoJR~jY+!h6yB9Uit94tl9X~H zFFb|o$I}%twK6f%voOTl6-hJ8Eyyj%GcGDJOEXF=HZ#h~OD`)c$}%ggtSHP*FUw6y zHO))TsZ6RgF-y0u)N~*{$C^RuQE=tVFD5y%# zNUh3DF3c;-$SK0Qh_kBFxFk6wIsVNq31 zMR8tgT2WGBig8s*d4)+)zFC!dR+ec}R-#dQd17`^a&k7-VV}&rO0yiJ?6e%y-0aN! zyrQIX^8&M^Vr=JcnHyS|8k$)UxwWJ`zbq*eI+aSyIT-L1iQX!@gD&64v z@uqk~3q2#q?r@wHXNGyHk!eX`V@LB2^@Nq!l$ftr(*nwpeYQI%JaZS-An z#>vU4X~j7e#i?0lW=0vtMMWt^X33SQr9~B~<;EFV`5C5VMunwm#<}@9$%VR@d9HC{Zf;(pxlx*NVtIaLR%xDbS%pbbSz>0gQC?YchPhEia!yWq zT6%J2VpdXV73dsT%mozXIaw81Cg$ZSg=V?=xkd#=#px-f#wFkzxY7GT=4Qr*7G?&f z#Ej${=M|S_WEg>#ZV?yTn9bEPg8)cYe6_lr!rezdlB_>srW)v0`rxuwNl%<+y=VzFh!*Az6X>+8N z<`k8crFLI1#rdg;g_)^FS*BH) z*?GAYMk!_mNvS!xRr%(o>4lj|#>quVDaKXVIr)X<8A+vB8v#`rNks)!#m1@GCTZ!J z<)&HY`9)bpkm4LI#h96ySr{4_84xx5UzwJiY?=X{w;?yhlvbuCn>!U)?+W$3UUcqBSEwFJCQ5PS@O zK6vO4G>)EGlvu1#T$-d{WT4=U4xr5|B}D(KG_R^4w;;dJq$ndlHKRN^EhoRADy=Fp zyD+)1Dl5GJbVf;~d4^e;S!RZLalT1rX)dJsj4V7GB-*rDJn|L&CbSp zNtU^Rk%6J5IguB4rkAG`C4z6jCN6TIH5YQiu1YUYEUg5aK`3&dy75Ggsg;4Dp1HXx zzEUi?EH5W5C9|jkbX}Oac~)kjNlI~9fk|GaX`WG1R&h#pK~Y&=eqKgVqG^tCRW>Ma zvCknEmY8LOhSUqq%&T&=a&WFy$jD4FFDj~puE|EJL^HC}QnSi3l1t0V zj1n`=t4d3fjg5`+Q%W+E%uLdAa#Jcwt1|M-%`-C!O|$be63eP`v93x<%`P#{E6+-; zDlIoHN-N9D%E>RQtgOH`(rad5W?^JtO5DQ1@?5jR+zi;l!Q^stW0UOEO5^;j@?x_} zvs`m*<3MJXhL(n=mP9RvD6KM0&M?acFQy_tDVZiGr5J(Dpgbv=fG!QPG{d_F6?80K zc5ZHFYEp?=d45uXd6s!rzG-4Xm1&lFNk(C@SwU_|mYKOpYJOr~R+@21W{w$-#&T96 z=pMW5l8WSf(~Nx6DigEv#G;f8Toqz&X;x{Od3rInq?DRdT9{;JTwGBcnx0*dla-Q}m{M9;l44qAT##y1oRXWCY@Th7eGDi+IXgMqG^?_rIL$1%urM>( z+_bzZu>k8vUeKlJ1}4O9udB?=E+~WCXij!aS7v5sSLK4oP^b~pMwWUe_!l0e6r~g! znHLn7r>5m(l;)(Q6q=i56{VF`W)+tul^SK5R1}sMR;8EZ=4P9km1b3#7USA_PwG{6 zr8xzKd8WBWDVW`&q>RGsDs%Is(h}nmljQR3oNRN`;`|EZ+@$Q{s`7%2inP+~;i_83L57C)x4?6`9;|8Oo22{ax#lcphn;=%E1>5f)^u!bb!|MmssiRr(~vB z733sa6`K`j7N!^Fl;u^H<>#B26y}s>nHp!7BSuXTw&@$^7#mlar01uX8XJN7_V7Lt zyv386o|#ctmY7^-T9sLzlx>uqSeRW=o@|0qC#U756&IUU6`5zJr6r}LXPV`trW7Qa zl%|!WCuLNcn59)D<(E~M7-eRqm>A_IXD63pA9*k}E>Fub$;dP}%}cM!$YMu#rJ`I|^KxH%bm93BvO3Tm5$p_UCND*3Em70~A zmjhnojVvvbQT zjEq3$5NaxeTWin;0g}7RjMK9-D+uTO^hl{%PR_sQj<%~ zaj6{BC{m(%Djq{)I_61@BzSy z#ir)QMoC5`g+{4aMFrVKrlooLW~oMHxk<&T<>l!W#TCgWMd`%_#buQ_W~NDr`M4(A z3kwT#jB^u9O|wi=i}ETmGt;U{l8Z|+5m^}=+n~0&zHyF8ZfRwrsd+^<_y#nzalxdt zEYkwhoZM3L%Ivh1tn5;=%#z$(6SI`OwDQctyp)onH1nj~yj-J<(v)KJid>^ybF9-{ zNvQ=T*~WRsm1*T=nTdsJs!K(bT}w(8S!3m^GiJ73C)7Wd-1_2sC$+ zzV5uD+@#P5a=#+wN!-NB&`{5Sz-f;8##KhrshW3MJ8!gRVHRe zCMEf0{N5}bkJeP7^{2J&5SDwO3li%Gczly%1lfOQYs5fQwq!~OR`Lj(kgPz%&Ia?@^eeG zjnndy@-ovhP0X=YeicbMrD?f^Nm)fX`FTYt#RVBfrP*b<*arU14Ghc-EzC@b+iH`R zlbsLQ5Jh%OSEc2omZX4N-IT|)sg;3&p0O#o;|j0;&_;hVa*KX5%`=PBlG2k)EAmpaKo=)rWWB1S((GK*^1O=Tf~>^6+ziw5iln5p z0<%)%)XKt)ioC>}!jzri=mI%x40xBw-yZZ9oQPb*GMH3FG|C+NX-1+-;= zC)^FKOilF+%q{WN?TJN26&dAaRVhivB`MieMa3zJx#ksSMQQ0q70KC2d1Xa8g-K~f zhuhgbo2b|vUe-DhWM1Rb%n)U$+aR6^US4;}vkU67TL z4;tt?yiy@wp*SPITp=Y|867$TAl&q@CoT~Jq{6h1z;NaOp}bN+(h%djH=YUirhTo(n3(19CK>N%&amwFRwB!4|Ix6rCE7Og;`-{2KXFN z^mJ=(VPI)!WJc8ZW0`TWX>M65xCo}8)BqWiXr2o;22au@zSJ-SpO;{S+A@Wg8io0} zMoA_WrK#mrnW@F)r8!jv#f8bqMoQ>|=V>WbY39jgDJ6y3IVF|JrYR|91$k*^rIlso zmHAoaNrl-NWl7m4>7`{Ux#q@INo6M3mz`CZCYP00WfT-8l@_I!rDYW+SEUx28iN+R zV1&N0k&&Ufr4cdBr81+G+(hG2BT(mrVCaMMJ*bTc>VsydRw`M6RgO3A56&IOx+ zC+LxNqm~+mX1az3x<-~Eh89-F#-QVFp?8Bqle@1cXmNF3esOW4LViw0BIF+2w9+Eb zvh$Qwg(T35^pt!B-$YOgBUQoAQNhL4%{AB&BQ`P;Qwz%ybBZcU(@HbVt8$H!vy0R6 zD)Y0=)AGxc@-wo`a?*0Dva-wzQ%X$BjEYQiDzWD5%Jkyo+|tVA9FxMr%IvbT)Re5s zoWgW$Iora}%+k=%%#xVH_R363GmMQ=jgl%V(oKys(lgCW%#%y<^UF=mt1@t%sZo|@ zTAWp0Qe>Q(Wl~g-WtN+2RGv{#o`j>tms^}tnwy&pIU}<~ioc zrb$Kxg~fSE$%%=@nZ~$Uqq+Iz*}3MKu%kwy(O;Yix;?cRwAMNmvRWFxv=_P^0JO|q zp&+riII&0}DX}Co1+-?n6qHu-!J7i0#}4GADx@kDXXcfFW(%_NA?xTtyA8nG7!EIn ztlLg40$op-S`50pI3-a5bmwcTLP};)YGz5I4!UaaN_6n5@#K-inVVsqlon)4jR+(p>kzJKvQk+`}x<<7uzbGXq zFVU#5veF2=Oao)ApgglOCojLExU@X6ETbSly(rh%JkvNA>kSU3CPs#47RJO(tCbm5 z7UkxpgC{?UD;c0I0_3p*qspR;A~UcVgh~deZq$+iCa5>)p*62lOEXP! z(~67=^Rmlw%~DIt(lX1-vvLb6Q?imWO!6u#3kx%|5|hm`bJHtQb1M>yQgPkjkeFDU zoS9-$WDdGxryx5sBiksmq9iQ~SGFpxN~x$a%>`ZCiBShsnWh<+R+bo>=A@)%7Z+q_ z87JnO6jW7JRpnI_mgJjfC7YM!7MJB`CKl(Wq?sFm7Wfn^Sw&YXCFYbUSt*6)DuAx! z%}mQo2HpNwmRVF%nwSIKv;fK_`3f;g&iT2ix%rU&2Zja;xtTc`hqtB{gA#LTap~cO zMVXLWhn;dtQ;V}Jbrh7%^o-05^h}J+bQGKt^AhtC6?_#QHLP+-0&{Zpz!zhI_aEgv zYFMMAkeQcW1lqMy3c8xl$O2??VtT$Fhz8v*9iybAqm&4`v9Ba3u_QAebcb0+YG!&y ziISC}nSp_hQh8=dNrsY@k%56ht&S3Cw@WHW5$qI`q?~-v*mqHC8dx+ZGcOx5QNZo?Mhxkeh6hol|UHkeHjElAlqO zUI;4xb2D>OLm=&8u!Z_r1*xDzQ6M)g8XK5cSb$DBsnt=!-qr%`wSngPBJg};N@i7R zkpg(j6FB$hCKl^KfP%jQG;kFX!FQs9$|>m4X9~EYL`NYNv|}i(G*clTDM}O+QGmjS z`IA~d%Mr)U-U&Jg*?J zB)2@PEX6dns@yc!I1TFnReEAos##ICNltQpN>X}JRfTzCdYWk#))hTQ=0=95#zc+N zlo^>-q?lEJr`yRdUd$@Yk_*6QP@{M;GSjm(N9`t}6)!18$>l~xB`Nupx#<;2rllsy z#Gz!cOpGRD7qI4wOdH>n~!B`v3@A|)rc zEVVc#J3Y71+_*R=x3nS&bVsgPZcar?xp`h$2I!Zasyz&EW;KJP0)W{Gt(GDND$OMhjXQrg)m1LHH z&%H|luj_`*cHl8PBQY-peEn)-UP*plW`0&_US@t#b{cX4sidP+oLZ=4WoT(^X=H9{ zU<}$OoS6%{v=t*3O-=O7L3e&&S>$14W?EX3o{?%?R$87|Rb`rPWLA`AWRBH`7NAYB zMnvr{DFYp7m7hk@s3^#cv|{2Gc@VD5%|NT6EDZ7PBT6bwF*mQOGB3?3D66Q*D9X=I zu1ZQSO0G0YGc!%hPb$vJC`d2OEGy1WHcL-3H_k4sz>(WZQ!Dezs!Gx-(^AXQ%S;N> z@^cEaG7E8?aG6`0np9aX}KjCnIDa@|%qYh=ue_?TEU&~QF|V>Lv!b9lzpM&08idgSHa9jjG&C}Y4yGtsDJ6rK$tCCK z=4O_F+7?BrWw6exaaK`PE@jJ~%8av$N(&*mi%>lT?LOma=9+=#3rrv}otm4JihZIN zETdPJnUb21r`2TuI<^$r>M}JqtJP70=&-6*BC64qk(iaAnv|25ZEBL2kyMs#lv($w;@!t^B5{JiXn zqN=Ql^7Qn~q^v^I-0YO3T;udyT$ewYj6;$PyV_iy>o|KeaZkk$>3);zCP?%I!oLrD!mWBQ3DKj%eQ&TgdCWgw(vXat3 zg*UwJ$2|pzHXw-9c`Yq3%QDF_qv{RtW_rdJs7vSIBZ|33`Q?S?g(ZcVl}R}nSt%8{ zX{G6&aE^_OEfZ1 z&aSM=&rT^fN-{Mvt1K?T8v14CS&1p-l{scSQl%v?|%g_7km z$})2^jPsKV(sPVUQVNsuDvC|3jEhYQ3$u)K&6A7MlgrK0ib^swjM6IeKo{4W;W~@a zC?&BdE#KIzBBj_QzbdaRv%tKvs_)=JSdyF8^R4RohkN=`vua$bH~ zVqsox38=_KF9*zwEsYH=3@t%@8$zL9T9IRJlvQ8^T8>FD^ucoj&`tw#5nPdDR+3)< zHUm%4BkRUf0vKDFTIiV>qm}?@9sIPMlDyoMoYc}Jv)syZ^VICjyt0Cnoc#QZ?2M$M zlI)7gvb2H%^U4Bqqr$3;O4E|0Qc%MNOA|WFI4iHHyeOkQDbXk|y`ng$qOda8Bn9{Q zVOd^QR$@Nr(DPayNN?Z3LeJa?Zx1QGvOFy=->AYk zF*&2CJgcHKD?L9mttj8jJPUMY5Dw>Nq-IoP8W$EDXC)@5mn5eYXC$SiCYj>C#xyrM zH^(FubUy+5aA#6^Wo5B(MQOfyNlHO_Qhs?wT1H`6l~HNH$PDMeHiCJ=a zdP;U_m2q}{64r#7QfOYDnUR`QWtx&yW}KT{mS~)uQw3faiIFgkj0_FU4Gf8D9#j|= z6_)2gZ(1Q{doZ?yX;hSLTn09S8VS?HP|wm7@6Mo9qwK1zk| zjC7;Y(ws`OqSV~7#GJg$eB-SA{ESTFoTB3V%p4psZCsIAT#{T|mR*!mWLi<0oo||& zm6e%{^Jc1ivy$v2lQe86J*JgrSC*P2mFHyUW>h7mCncp7=b5CI6(lF+=b0O4mSm-x znO782s?0U5gp}N57eQ5(IRz<@N`&&5HU&*88=2r=5|Nx(1R-hviEKw(*(Tc0o z%;enoL!pe<{1TGWAG#lGz0KdPo`F&^Jy(nr?=qE z0+U?hl&pNy6!Q|J^4#p?yu?%!Qxnt542)($Vs2q+idlI|N<~#+Zf;p#hH+6rdSz~z zSxI)bNkN%ua$;^#Mpa39WnxKkQK4}`X_6V%2C7MBd3kP1PI6+INv3&5X|`!fN@9VL zNfK!3FxptFnTe4FXh#!Kd9oruBPSWUzZ2SqP|{JtzIqy5u|r2pkb=InB0nQNJqx@? z6i?72>&6rACRPSUdZv*3LLO~!%N;(*Q@ z%PdP(aP#zY^zn7|3vpFQOioP7F9jWmR+Okv20GJTA+0newFt6yAvd)s1$0KfLQ;Nl z37C3JEZnW<@M73Qg#rN)^#iB%>g=Ek6* zFDyAZEhQL~J=ZEhRfE)2Of_E5oGJ zG%ckpJ+G=LztkkjtimL}#Hc*QtUR&GG{-2_D5tD2KO@VyDy;--O|u=^RmM2}@00UQudhoC-FBP+EfO z#*>zet&A=7Eby1v>A7WDl{uNIg%y=XW~nBb$!5uU<>rM|RawQx+38iKNf~*@re+yM z#YRS@X=W+O85Nm0YV!1)Ec4W=ax)Y2{8Z!Yq*RmiN)w~hRGejYX=z@SSuSYlK4|O> zrGJ^4RZ*3alvG)knpA3*Q3NX7vW$z<($Xr@i}Q_(%}R>%D$G(6P4jZiD~hwtKurv+ zF>RKSm|K`vY?hapTbP<|l%JTLUr=g>^(=RD&{^(AMn*)H8dZ6fpeCd#D8-N)(^Ywu zIaQF7hw_*<1>MqYXpXmb24%ctW6-EfdP;6#c2Q<}ZiQ)1K~{2^X>MM+S#Dx>l396v zYG#>vj#-XzX>xHvI`lXfJuHP@WubXhZYq`Tg)*@+HrKN-1~){|3Oz!%0VO3R8WkCv zo2DC8RuttGl;s-drx&HBmlmbugN~7}NUth1OD#<+G_A@=O-#(qGcv}tB`+l{KQ+-j zJ3G<1pe(g2tIFIsEv?inI|Emnqtv`4Gug}tbOa!d;|Y^e3M+EV)6=T5iVKShN(+p$ z@=MB$%JR}vDl_wqtFjY~l9Tf?64SDi%0Wld<(ieG;mBJhCE3YFSrxfO8KpVd$>pZG znOQ}N1?9Ni20EI%4BCBD0$l#S3TadUH1149#2qP9SR zdSiL1kj0jiG@*=hvoo?xsJCX*+{)Nc&%_d(sxeo37iXoF=b4oyXBFj{Bo(LTR3zsY znIz?#VuXHbYO+zTk#SyHQd*vIW?p%@v3a>^Vqs2URc>-oT4`~Lxmj^$dRlgNp|PoP zK}u?Jg&EeuFu5o@r!29+sJJTC$gHZkw7l5JI6Xfd`yC^u7KSFKum&ua$$v8|BO^Uy zBe35{&9OxpnaSy8sYxa|sm1B0rg>$VN##lD=DCUKrKx4bm6^sFNkv7;sm3OGl}4p` zrIlr6IC=?HIi+c-W+@r@$rY&?DMh6vd8TG&#o0Iq4@)vi3NnpyjM6}1T$G;=o_s^8 zc#@33-7C}N@+7k~Q}eQ-;@raY@}ktV#EdFq(~Oclfe`8RTLV zm!?#hWaZ>k6eNR2D65iFOf$=iO)83t^Q!U^D~ruatMZF0%W^9cbF#{d%t4oLR+V8* z73pcl85w3Kx#^WA<=LjmX6cz_rB%jhpfyS8!y4uWre=m_mgYo_w^d~)XBMP_w@nbt zli;Q?v^j&^u&&BZPA!I*L6s`fM9;_)?|55ENoitwiBWo1L9UTea)ohfVp>{3QA&kT zZfR+Gfk~Q4QkhwLZfTj3d2W7TX?A95X(EmSyRbB~EH|~hG%dp@CpobosjM(PxhyFq z9cN`yTv%mVRA`iqxm+bVqsTlfBiYQjFefJ|)uf`ZsKU71B+aY}G*)h2oRwFVo>-n{ zQf6*kQecu&SyE7*m5A+7h|1*BjEvIUg5rXLl+4W3?Bp^d^QuG>tfjAsp^>4XrI`_A z$_jo91GYHI&o@d-A*vk!)s3gY3U0=jSmGUe$}mk!GOa95G%Cm{O3JB9&oeSJ&Pp~l zs;tVWs>~}bHA^$jNjEc2FRL&sPdCX)PtGiWHe=A21E-{zSDBj>R%H~V85flq<(8DB zniQrO;ogQ=o|I&6Vv6kMPiCss| zVQNWvVPQd8c4<{{VwGuP9_UUUEK94hvW!a;vkQxIjf!(hj8fAIimQyOGO{bMwS|l= zjSbB$iCaKol%1JgOymL*w9z1>qPNT_t1LMwAG|`VD76H<3pXXrJgcf0YoMeRnidyV z6_llzl~yH}6jv2zn&f0BWf~c!CK;C+mzo%rXB8Gy80Y4K4s0sPOi9d50Z8e4O0MsLaGDH>VI{ z2Ia}$40P+3sS)1eg%gv~3d)o6Q_Ks~Qc}}X(~>fhQWH&*5{pvHsxm67s`5%rD>L&7 zQcFw=(<}1Iv-1mbakNE9yiF;$)Fe02ytn|=vq7ItNz2VF$TTT0E-3?zuVk9#Cnn{k zGR`q8H8;sJttc%q zP0z^Bz;>j71!yyxxv>Q?1znj*WnN|p?k%jH>%3X!Pe|ZF*PbqDz7L@%P)qn|4AxO zFECCn%*iOsFiI;dD=DkWHcCx5$w@9OsVX)}DmE@lE66EHEKg0%FeyvPD>tc31La*T z!-6EA2wPlWR+M3eZT(Mra*Ao9X=YANR%vdzxp75aaehH%Wl?HjNk(a2nsHL5d0MJz zae01aT1q)+omzfX71ktKQdD77XjYn0Xq=l@S&?N_S!SM_m0W~vv8$Phskx!0i5W3_ z+Da>mk_wE|i7R|?m&hPvvdzuF#^6bnXa?YEYnfOXnCclB;~3yiEGa1}E&;Wj3QLp9 zl2Xg6(vp)iGNA{;r6|FZL`rF5R#}=^MGok&-O@61v*hf8{KTBxG}D}tlFCA};*`9? ztiqzKtaM}JwB$;oLgOT?t1b!(l1eIz3zD)E%gb`Ba`H_Ii&G1$azMalz?}Tp!5PW3eq#p^URVeOR~#S@-vM~vy#j+^An3IlhVs` z^ApR9jH=3uOe@SQN{o{Wl1hxylCrT@ys5c~#U_=h*@b4N1)xnVIY!B8DLIwUH817) z<{4!<6`8526~-xL=EW&hnPxaQ%^4+So0e9X8&zTUWKxSu^OLet%L@w3%*_j{vNBBb zinFqeazXCQ%*{17GEFHf$S+PbOE1bSOis+ot0>0mT4S^1QqW-%Nd>6|S?NXDITd+j z>BZ)t+7F|dU|?x%XlOv(p7pA{(u#si60YLIdWdgTUTINDCD;@^85P{hhU>?ZQowy8 zLo>WB<&3nPto)+%oTSRUqO{z+qVh~Lld^Kts@(KKqmra@!SQsWe(Zh?9Ae{;u37*Ru+bamWJlwi=$z+KklX0CIy8iCMlF&FkWU-kXBd$ zS-*iNx=AS+Obzu+OmLJ8$(hNO#)&CL>E;zx#YJgZY5Cb@xrG@?7~?~!Wu=8hMp-3Q zW#*+OMODVh$r;8;CZ_q7iG@XZMagM}#Yx49IVq``x#_va+46l?=z7KSFqMy5t)M9uq~6dRcp7J+LNXj%oWb;UmOjFx(kDyHOKrk!eM49=4NJ%uUP;4UJ4JiE_G8VP$q< zA-E|5bvkHy9(JdL%Q@)C50bmfj0!7L%M!t6;K??~y788g#uj>}MyPEWw8be&SxM&R z#^!})={cp9=85S>xn{X$=E()6>4l(6ZBom!lTu77&5F#D)67c?Qp~b(cBe8;b2HO2 zvx?12Q}PO{Qj0T$&2g<=&oEBP%F0Lu9SMw4MyBT_nH85MCYP3GmQ-YBC+Aiq zrIn_D&Ra{aGO0+)HZRI4Gc7C2NK7)zEHNr9G0DZgEi%)ru)++K)02y`(u_d+3QDt* z3b2-uW}p(%#KeTSiB40KlyVcqA$(=Y=4mGRiD@b66~&nu*(S+Fl|~f>RXDH7Dy_`Q zOij+hc6CW=RbFv!YDR8FVPQ#OK}Jz(Mnz6hW?52hUJ7{P&Mc=WKP#muvohbT+$hmF zC8HnYlPVK43)0PV%gRhmD@}6?iz})!ii$FG%Tx2R&62Zll*DF9Wk#my<~c=KSs96G zg~eG$l~v|uNjMjb=9`wKrj>v)G)5v%D@#nvHA~7%&dRJxt}-brG|4O}sLHM?H7`gi zFs-UA%TG%#O*PG|%E>4wPAf4s%`C>+5GXgVFe%RlE!Qf@D$7eWGcHX^PRd9FZ5Tyw z_L*52ni(1!5O?`hWl2U}sj(4w4>6%G0=PheH`|a3w9?9wjP$A$uqlM%8m=Etk!E6L zVyI_eig*8SMpa=7_+Y<^^wP4j@{-i70;8(LqLPBD@;vkG?7aNMRI{{{q)Zdj?DCw7 zl+4l`@Wo(Q>fNmL)UpEe#EkNkbmOw@+%)rw#ER0g6k|xU57FC8%}FaZEy&9*C@MC} z%PLAWFE2AGtx76SF32}7C^bziHciS+E-6h;F-y+O$STP%$jJd^L@YK|WmFX#6;+jF znUh+Xmu_BCo|ToDW|o&% z3c7l^1k_u^64Qm|Nv1g_d3mO#Mi~`3CFaSdCRurji8(l9I^WE=DmfW6ON&1Gm{^f& zR+*KYmu!@oUzKB?m{gT&W|EU*oRgeioK~EXmYGpZBN>JEKlCsK5 z)6C+sl$6wxY)P=2C9Mp0^-ZH!VzNl8JmmA-y%VsS}+N`9VR zeqK&yUaEdcVtTQDZfZ$lT26jZVvLePa!z7#v2BbJ=oFpI{5*w>#Nv#S#Pk>?g`(6P z+ZZJ<8)Ahrm`W}#NX!GXQW55WoRqAOkO9l1xmCXaEDEY5l8Q1@(kuOoGC{Z6=7bjI zC|Tiju|sl3W>I2dy1t>Iv7xziSf=d#Mz=xk0*VgJFw%k(5N#OHQbd(?k04yPZ<695p!m9lI zL_N^%C>)Li%{3!C(%7sPq%tQlDK)1Ur}FQY@Sw_)0QVUX&Q_YLh%ZiIDj1qIq%Cd_pE0c}V@)A?i3QJQn(yJ=dQ;f>YObgPh z3W}4;D$~*sZbRGuZ)|01u4jaHh$UKwE2FG9wJg0TE!WH}HOHvDtfDeGGsU#9vNSWR zusl1{Bq`C@EF-VXv?|fGsKVSR!!#S$LCNVkIaS8xWhPm9M%gB1$!S$)+3DsbnYbqE zb8<~f%99h5jj}L@AX7>ci!*Zaij%7f(({b-%#+LtGEK6Q3yiaqtBew}b4v@$K_?2P z=O*Tro0pfQn-!H|tw9Pa&C7C<^NkD5DwFcfG7ItxO_EEpOtIA<<|gJAhDMea#C5HU zGs;r)jldVG5^9Tr6SW?;)4a=!N{fxtE5V%?Jk13(1Mrk;rdB4FdY0y>gOKpy>eT#- zbkpprib9jj#Io$7s)Dq%l0@_LB1r9lG+B^Zo|;&kn`By$)ADT{GlMO8szWm>XvL26lQc1ebrd0B2^ z3Dy~+m@*Ov}v9H>yOsNB~~qLwX*inPye_#@J@V z6VsB5jY@MXi?UNqQi>9DN-|6e(kjanqyLIo>yL!59*qu_iW8UlV28QrbMOZs+|11d{Cbpp7?Q( zF@ik}k1a%O8&#&47Nk@ZgWKa6LyTEfrAeu#6{Q8mrTO`#`GpxJxoHJC>EP)$^d-HP zW`>5w7DV;Q%S;mUO4E#p8=MEb9NHE}xH}hQMovyK*bF?`5Lq{#641=b*hJ6L0zAZs zy(OKLS(R8+l&=74`6}d==B4B-B!iB{1u+tniW0$x<>looBqbIpfKGGFOjIaTC@D?J zS4hh*E>2Y_0PX83C`!dFH8aYKi!$(?rb^9w+|WQ@4Vs7$NKE6B`FDN3(2HOfoO%gi%3E-nK#Xwg%dsimQr zp^2d>WC0-T_>Yw2WV7^i(>#;RbmP?Gk^$>rI_pks)NEAots zGt*PdGb{3N^a;((i;QwJ@(Qza%uL>)K=4Ip;DSVhev**M7{@s|NXjWEOP-32%lb%zUmzdootG=?lGxIPc$z|GcPPOH%`wjGA>BX z%1Nv$haQ!HGTvxzZfYGF=MVOCW^dR1CUX1Q^0Dzt^2l$Dwc+5?aR zy^5zO&8Q%;+}Jd&qRO6uvtDJ7)^CYi;? zNfk-v73pS$CRt_~*%cY(nTdtjNoBb?WvR(n^IUPdaeh^jd3j!CW>#8av3W{bNl|KL zA~?3tW8KKY$k4*jf~ZC66^S_}`9PPb)3Ytg19h$}mbw%}O&WswgT*%uG+mHf3pMZfRy{ zZfZtUOQ^CaD?8VixMe`NV!E;@E445OYzE~qZ35bxX^FP{3N5D7(lT>RbMo^utI7(p ziwXC)H3GFGbd*5*|IMrnE%c1>A6Jr+o1UAVZ&Z+y zlVn<6Zd8Q2{P*j?pR+X2V zl98NTRg_VlQc{$kVv>@F^DgOZ;~cY`B+${$7_(1_Rrv*`6-fnIX@yA@#f9lPRaxmq z1<3`)Nu}ill_|M}rIneL#)+w!iA9xF6=r5R=Ehi4W~F(05@_fqr7SHmF+VlAJh>#h zDkT%^4sR4z< zY6Yl#2al8{XXd6W6c;5U4d)gc=@yoz=IW*ABxUBNgGTZ6!*x?#joi$W0?p#XBg#DT z^1brCDx5t{Lh=l3^)vN~jgb|X8R|QRhIqs$8KfDcT9_Cam?vABSR^J{Bw3i7nZVU1_dD za$;U`YEB|(1RZixJm}huynKb+)XaPZP~RglH#09IUq>Mkq_iZzC{ZCVzo;Zt7w#?4 zN!g~R5L=2<6%q@IQj1gbN)i?FQbET66sJ~{BM*8JJobn&=rC;%$zkWELiy z>LQ^0LZ}%=0TV z@)J$+OwDrBK}Yr$CMV~VCMBB`C`ijOO)JPtDycFqOw2DXFfT1DFETT&Do?H| z&CkbL|CgsHW@V+Al%^S{8KoK*3fT6LX^aAm!%SNqL~1 z`oxaWVJ3^xa*Qv$BnnN=(u!Q!_L3&5DeYvT`z$i?MENFf+C=G$V3p zSY>iXS!Eu0y$f*#7qsPpl$=W|lQZ(oAZ8FMxS+c66kH}&M#g%k=6FWvN)mJPL1*A7 zBp+U=P>_!?M3kPGl$vHzQCMrvk$a#>ZL z63U)~b|s+ilofEqC}%I%z3@^O!JhY@}iVfBeUG{!qn2DJoC!p6^YqK$t7iG=4lxzS-BOa=>=6u z*a}fI@V+zy6XJ4tRe5%46>+0`xcZ;vRpn`wMPM@snH%8U4xeF? zTVhmDmR)6*kyo5uk(zI0YEo2Lnv|WGoL-QdSYA?5Rat0WkXKcdZdzDSUX-3`grk}- zE;g^qElw*dEjKMMuSiWRt1?Z?Nl(Ig3RG&ASyDk_F1C?|3}Z86qs)o|qpaN05@X|J zqau^EeDi#x;FLEL#P#QJWm1#eGLy85 z-11~_yAV%gqZxoFzRf`E2~BZrE2uIz$~4bNsm#mGHY+kRsxmUqFe@o5h0fqYkAcak zGAcDHDX%I@&M{5Ss?5kMt1`|`GR^=EKUSKS6qICUn3k4iCS@2`<>VEo7H1`;;Ybb@ zNyeF#X8A>CNrmYqRTafmrm4AAMdrBL_qm1Ti79F6$wr`xSqFBUjlV*$Q-}^|Dg|^Q z4(OVz)LhWjTcDm&szOO>PHI_Z@!`do#e7nEUUp7eMpkZdVs=VKvTBrQ{ z)J)IV6z`7Z6r;?lyxc^iB(vfYv$BG8lPvSB0@JGe%uMt0LenJk+^RfN5p zv)tr@M6)tb^9f7MP?cYmUu<5Qnx0mkky@CNn_5|wnNwC!fUEtNQ(9D3m6&T}j#)FL zmKdj{mXw#}XBB3qWtx}dXXfP=CFZAOmzN|Kl@{jYmX>5Bm1LwAmY5l3B&VblWu{}T z8O*aXvU0Ohvoop+aAQG2a}#1QF>12Q0T{Sr{9d7#P%| zuIkE41g+|V&J;dhv*yXJp2y9NkDHc1Zf<_uwEl7P+{aBz9yf1(+}!cFaoyvFo+mrj zzHHte?D&39$CDl1kDI#Q@9BKp(EGS?`Qyf!j~kkv_I5q(XnWb%_ZWocKW<#{xN-91 zrryVm-5}YX>5m(CJe@ER#BFSQzM}haW7pFqE1zwc{|S!s4oc5yn6o++st z*GtoqN=i-hvF(6KNioYTDoHFX%uNCvdTC~oR*;`&lvG-pWRjU|Y;K&Bl$n`cnrBpM znvtEJUY=N*lZ|yPTBS*8X=-U!MNxTCX0CZvYF<@lNlFfQYz;l8%`J>A42>*^Iy9!t zxU@XEq}(VSJpONDWnidhj{ml=lr*E9{EQOg!V>e$Y~vE6lH8oK(n?cfo$6T3lY5 zmY186US3*W25M$t)b{yF#wCSCNySMiWv1y#6=vC)>4iB-S=h$^&5aEV3{8wo361|7 z=YY!Yq`dMZGw^8>#N{Vw367kfjFR&5GRnbb5Xw(b-FWhoDd=nx6UfbTpu!*DjX$Xs zkO6xo9VIx^s#+-}wK%ybv!En1KTpX@DL6APJttM+&~}CJP&a=ce|P_25XZ>C$QVQ$ zDir7EL5`uzD9ue$BK9_*q%7mijO4W3^2C(<(#)!=4D+PS@>H{|LX%YEQnQ?5(D`v` zRYn!XiAClW$%SRc$)F8nI!gE}6H{YT3scit9i@`Y67Vf+jtXI!C6I+CVX27Z3R~fv zUyz!oS6q-^l9^Tso*Bu^E6LF(o$?=6D6dyR$5Vpi)11P@9OEpbytIn4oQlHa(u|TEW1J@jW@aU)WG5CFnSo;r zWf&yAsKmUW%)BH&E5kI+EZf95FSpb*$;8~qG&RF4w=%IZ-Q1)!+t@s_*fcLAuQ1mv zu@LLTg0Wd?aivLdSxQnyYDSV-u4zGqQ9)%{8J4{$<|am#hDPQVkX2cr%lz>4Lyhvv zjq@wObu9ViN?ut(1;h+OaZOCQVqyflSAg8KkG$_8I5n{-(LKMkC^xkP$JPhX1@=fO z*&Mv@fuziaY$0e~FEcMFGaoVs2%4G1VXuh^%8m;o6X;$BP%&o}tyE9~YMX+06M#xP z9D50pi;60Y(z2@ZbIbBGsw$I>iV8BTGV^mY^2>8eGjp>`lgz8kOe?Z-%_>aF3zL#c zjY~m|J1o5^)8w@D)ZEmng4F!X#GEX%;^cg@)bu2rM|dP>n3<&|few?e1)pq-mK{^7 zGL3SR3NrJIvQ3N2vdhyFQ?iPaQ}XgN%hFQJQp&PRk`jxuONvWODzXYQimFnw({W7m zlR7M9lvrYBR+)-zDQ>z+NpZe$W>#jQaYa^nSxS0ZX1ZBrp1Dy~UX^jNnOQ-4ab8Kj ziBWo)Nn*N5QgK=$_Pf+G^D0bJGqXy{$}+3+jEgfW(h3X9k}9yBHeqgVZfX=IXNmR4?*S&^PsVVam&l%H5$keid0QJI`;Y-*O1RaBOloNbnG zlwn$spOcrGl8v>Oos^nfZkAPAWt>r#n2}_boSA8wn3j_TK2sfirI)d>nW2T51u?Cx z(u%aaf|6q7p>qObD$srhO3jy+msJVc#t+&pl$>9Ly zmeG=olT%W%D${b(^2*Z_(~L|iDvUFeQ&Np`Dyve=$`Y$`^3u)C3UZ87%uK2>lS-1Z zGI7*-E1Tpe-SW7O3||p?M(1s4OWzyD-@}xi~2?D<#vs z5HwMokz!&58oo_0&oVVGDXK6rOH4@tT|1VSW||EeJ;PFj6(^e-=ci?trKc9BB$}C+ zmlvd#nUxjdthh@n3XL*N)4_Mtqjd<eqZ%wK&OENafNG&rmF*m6yE3Yss&nqa&$69fl zf=(eaH6!M{urg!Qoa9VWPS=1|y+}zFWQJJ|#0<(y3RBQcRR)&eCK~pVBCs?Oyvki6 zATcQ?U!k}(DYFQ)iUhQ(9lW!&pa`-GB@uk%M5;njW|2Z!Vx~e#QEF}~bd(uspHNa( zo^eugZb3<=SxH8IYI$l-ZboTIVy?MyPGMSUrAcK)d1gjNW_nIhnMqk%X;N`M_MUM@ zQE5qDnt5_|aj8j4X{k|GQE^q7nF-c?LKdc$hK3fP5iVHyjPvNA(#neL3X^Oj@NO@n z)@(t`1EeSdnNgTj3^s#M6hU?4DNRg3$B7Xb14%6`FgL5rF)t}jD$mQ!E=Wu(F)7MR z%Sq3#Dk&>W&o#=aGD^y<$V^YqEw9W^O3pLOgO7or=j-B}wCu9V?5eWLER*Dlw2}<- z!p!uFx+!qSw=+^mAid}EV>lFYJTwqk5o0(#qo0Xkq zRBmjPZCqBIo}X-1Zk(Qyo0y!AwWD8JX=Y+nR#lmwlWCG#nNen1oSa@(QU>w>#&C+U znUSHnks)ydy&2gBl_|tkL1^cLdZBr)S-w$Ha#eDvxlyulSy4f8SyFMLd8KJ~L0)-wMrlb&YOYyD z1&%I2c7}P0X@zNdWr0a~QI&aCYH4~&rLkEut_hXQ%#6IsqHM^7Nn?Y(F_lCq6&3BFChvG&89nBRi!yyDTr=sI)2#_vWGE^2)NJszRe2ES-3h zqSWG|j5O0S)1*ApG-Ko9tc5|2RZe0`RaUB5wn?5zVNw!kb_*lbnp#*I8XFl9)oiUaEduSD)IoYZ9KNHOY}3-IAXBp;R;Wt6An z7K2S8R0hNK<4Mn^R>o#}#wK`|!K5Z8m8X`Qr>C1GmKhtT8mAW(=clGrn4}k(rRh1`}mm6o9WhCY&!W&LXC?!|ASx#nUVM$qbZcc%zS#d^zd2(`@Svt;T zFr`V^*@>p6*bXvCGE2-cH7`yp&8;j*G%Ez{J1fY{&nikUE+{rDuShL0$}P%DH!>|D~yV9^j8ZD3Nn+DDk@Wrii!)(((-am zN;0!j(r_N1R$7pkl?+miS#o8Vl^UBAXIAAKflk{>E;7kXu1ZO%Fi*`+FD@x9%u7l$ zO*GA}G&3zMFHfliZSXC?n*S@)OH(Q`lFZ6YE6R#8)3cLHlgmnSQ$a;9X8s4AlxJvx zSiue&?9?@|)HN~;0j&l$1+6ABw8VcpSz)C~NmW6qX|idac|}rUQDI(UQf^tkNm`O| zc1dwkeqL%}WmR6OXmSw@n1W_D&e)^RE`6HCy2z{C`i zrIjhUrX^M2!-EMHl4xywl$?{2Q(}??HU>{AiDm%a%wue%XKIQ%*aPn*RAp6`XBJf@ zCz&NBXPRXvCY$GicK8)zbQ01F(#n!5vdWVx%d#s?vr3XoGP8=x)2d2Jt4y;CD?uBt zbIeLhP12I9DpJcbOLMc#uqNfSEYq}YGrwHc6Me_et}t@X?|%{u4!^%iLq&NZc1`RSx$;kPEJ8$VgcwZ zkQ6f$^E6ZATxjT*XXat<=uS0BOfSeb&&V*!Pckz%D@dzK&(2IYO)AX9n}H`AA?wDIhs;2CG+LN~w@_gp zMoY{DZ`J}WV9Zy@O)Se)$c6#T$TT(0Oau)}BxV#ARF#yKmlkDbXBp+2o0eAQ7G|VZ zC8npFWR&L>XO)#yWaXw77GuA0C^@Gn1vHS8lU-O?Ri08@TAY!YlT(OwL%oTak)f#t zXmSe{qPU01DvAm#Gb@pIyO1&9UQ}3`12Kb8@Do%28<^^uo8j$mCzoU;8<*v#r>2`^ z6{c0D8Wm?$nimwAS7n!H6zAup7x8=F@c zrDYp|4T5xCNRH|GdQb0W=EKyyaSCE=lmX@ArZeEp}Vq{vB zTy9!clvRnNb&^|JQIb<`nvZP=GtIctI3+VJ8?<~XFQuxq$|Nx>ImJ9B-84VD$RsPr z)FeMU#XQfXvLZj-xG=ZCG|LR@+P|`F6LV8@6O+WE#Hv)&iq!nveDjq2JaFd{txPqu zG&2L8l0npPL{(C;X-NgRJcN}1_`7h>Y>C`7sY)uU1g+i&9ebHd=`5NFXjQVQDc($- zR#;Y=o1dCuY@Cx)S(KZen~`m7Vw_h}m|SR{ZI*6gQe0S&mS&QiRal%{T$W{;1Ui}k z>%bGq7se$Q8)cc5VcV~gT9#~7l$}y+o?V%lYm{1^kyvJuRZwVVQkj@kP?VEgo|RdZ zmY8c)m{ysRTwIY|!*8EGb_<>pyw zCZ**`rC3{(d1=N4mAM&N<#|a}dHHEsxw!=ups55bt8gp~3{4G<&CQ5u&zG5`6`Ld` zL&LEERFHy(22i$aC1)g-=p}(pJWfn5$uH7NNlng4EJ_3qiRi$VUV)Cc#F}I9>rTwc z$uIZJOU?&vL07U;O3cYoLO)bDxx^^7Fe^8?)U>!FKfTx}wJ^V^w4lNy+r+H2+{i33 zFC*WyAUUJNG$*m7syrvHCZDa$hgA4`V)Tv7A1=h_Gp@{sx(&2o6LN@!b7@gYrUI8TczKv#K77#@M!HW+DacRH zNUACG`nc>_5US^!1Se|Esb(W_xwbV2# zFU7pPDk;k-Dc8u@xGX0pE2lEU*u1Q`vZ^GnvcN1YDK)7wDW#$;BfF|32b}J!L49l` zD-dZ_kdtg>o|9LeY*dwLY+g~AQ<|1umQ|UNY*dJ?vNba@wlp*{Hnv0*T6p?OCZNGm z@OS~im_{2lKyF!8n53H(rh$#Y6WM46;3?7rRNtW7v-81rRJMwmX=neXXR#BC$l2kB&{qt zrP8dxI6ohIOdBU=XI7OK6@jMYF)FIWZ1eP_irm67lgzB#3^OTYd^NP&0B9l~5MTMSWvyF2y3(Cz+%!~69Gct?{jWhFdlC!ZM zQ;!#FcNEw4N)F()xAtE?!`%%sF5+a#x|vN$)Zv@$=%EUBWT zpbE6+rL-usM9E4CZ34i=%EUy^$P91NN;5SsN;a>oGS163Eze0WO)kjHD6c3o%1KN$ zPpv33F09BZG&M=ANJ%y;G%n0bF3v3mO-5j87!>ALRhAc3W@M+O7#rm!nx_~Q8y8n4 z8)HveCb`AMX`oxP!ABgRcNmJ3%StngjdSzM(-KQ_N>Z!L(#$K9(#sN4va?I7ib~Rw ztBR7-OcP5h3Mw*6DnSSHVoO@N<;i8KMrOul8C6+{r5S03CAqoDCHYu~EiDa<3@wa^ zYVDSp7#md;LT=e6SQj8C4y2?7GNYu-m`az0m{}Pc=ouN}uk|yG%(9Y<%0V~&7G&iZ zr5I=DV zGAqt2&n+&_OEN0SFRMt-Da2aq7ntQ&Wv7>ASEVFXW>pzg=4E6TrQ~5f&BM&V(9+P@ z+!8u&2I_DngT}%VlaupHA=d>YCYNNEr7BsK6qTmxfLazv?F&3M6{i*|Ss7XyTN)V} zm>7f1&&&n&CegB`v6Ydbo-t@%4$GC}g*o}f1*KJ0C1oikCAno~6{h7zRk_LFaXa); zAyXqWLn9L-$XEgD`5%d8W@Sbx8Rb>xsU|sPCB~H|#>J`G$r)v76~<+GWkvRk=kOl__~ym079f#z`iLxkWiesV2raj{ztu%g9g4HiPb^ z#6E`Z=;`UH@L~RpwdfeL{a=xmoKakmn3R8gmX(Hmk!edUL`YbO)KC zF>yy=8y8h(8d12>*0`uL*{lq_35fDa#>~plQqK_o+Mf*5#H1>-io*1g!lVpS^ThO= ze6!s0%#!@F!jiQ7B-8RNbJLWna+9pef`Y8XQj-!BTqlDW8JU+Ql_#3ynB*GgB~@l+ z6{lC`l;z_*0w6Qls4N*=DPuGz(#ujSD+@BpQ!275({jslORCaJOU;VY5;KyGjZ&)e z(~EMmit@^nQ<5{wbBdFVjEgd`mIV3Pg#~$)#w8h<$;O#kW`#+q#mUB*Mp!S1vNSa| zG9_|3aaCeQL3SeY+8Vi|FV zuHdRQjY>?)($c_Y5DG!U4KhtGsg(LAB)^hmQi|1a&`&W3~I!*v8kScA$T7Z&X~?MDk(_ID9JUdOfk(e zuPQ4qDmAG{tSZf_N~_4MtVm5yDoroRG)c`Z2OVIPQC19II*O$wOzMirqLQSnv^+DT zbj)={X-TD~Nk*C3<(Y{Ui76Gur`KhKQI2NxNC#9$7n5Pt)7=f0Zp*6zON(z#*Q*(3k(@aZ^ zO>;8R3Mxv{lT)%yOOuSMax=;kD~<9~QjLm|a!t$gip)(?(knna`!V+9XXcof8WmO) znq?N|Cgta5r&m_w73G(KatV4OGcz_eHZ(RPcqT7sPkw1dT6(TI=;Q;$=nc*xXSDGY zq;@#Sn9O8|F?iA$ngMuM+U1FA5mStX* zR#lpmm5tHxNl8g8D9FvKG)^(kF3c;+si-hdD^JWQNij3ZD@--7D9<)BORvf=&o56h zEi2B<&CkrnI*vlZ&B*4429}0q#zw>x=A~5ypzCG8C#n1rQemliVM$eqQA%==k!fCWN=0Q^mPuu0MoxxVerZy9VrE)SLAj}MUPeJq zY9`iLFfGi_M%l$?sU?M}xtT?|CPtNIg=RTspx&WbNm@x+MSf0t znR!`a5~#+v6EAmX=0F1{TDeNMB}bW}awPY6M{Rf&nIX5ii!dhuvsXlh|- zY(!M?SY}d^Sz($5-iJ&ugMwRI@RisIm*;{^DNHT`n}VlwMAeTcdzyjTtY${2OBB%) zYFSlTT1G}@PMJw+W^txjd0AdgRZ(G*Nmh1Z2536I$~Yq{-^?T>r7AO}$ke3LtQY?NGLo|%|cmS0#|l97ljp%$hlW@V=oU^}2D&A1>ZHL=t< zqcpKJyC|zXHKoY3Br)5pD9a=_&nVL@BO^H{r=Y5=psXU_I5)8{yPyi|6n0K(nTc_p zS#Dm5k%>uOa&l@$eok&?BG&z=MxdPuCdBPrH!{N*{WAqGWitc!58#CYya+5YFU~DZ z%q&byEh(=^Nvt$8F3n0$&n&^Hh%%Cj(~a^?j4QL#QnFI>%5zMzOia_wlJY9eO4IXG z(^HI-(^4~%jf(QJ(zA1_jFZYzu{HzBDw7LJ%2G^?(kqgat5S;-Q;khBOpUS5Je!yq z8yZ^@cYRhxQBiVIiV-*jAz3sTXEOktPNDq*q*Pv7Q3P5U0XBnBDu?RElg3S~3{3UR zEb*rC^s1EPB-7l&%(A44#L8roiqibl%;NH_tZbvIid?hO{EV#9g6z_&!rV#|le`q8 zirfqwGtc>ng@u`gW|>(gc_~$8xfx}~`Bgc^skkm_$|=ptGcGeX!nWr+sWLseBsI;f zFe$%0t=POIH6tT8*{q_dAg3a)u(Bd0E3K@cG{e{&RGb+nrxll{nq%!hnpEberePRu~tjo9E>x78K@G6jv1G8|S7~RZ>A-Q4yAeWoBk!W@uz#M%0SA$_!)U^fDu`IhEk`KLm;p=->=;5mK39RF;zl zHiJ;Yg6hUwgczCYSz3SxCa7@=Pe@T>S!!`H_*PwPw-1BHs*#T1F|wfS3?ARayv#(G z)Uf>I!>e!{zGDbF-44l0L-5&uGmZ013iES8Ep7DHo|%P-rJ=EzAu$cr z(#mpZ{x?oFF3rl%GA~Kb&#E#?$tcb?HcvLsEGR5XFUv7aEK4;tugu6V$jwhGOE*c% zFflCxb^S3q?NvFZMg=+Lm8B(lg-N+-*(E9E*(O;j;4FeR4PkC%Vrgh%U_wl{vCPP% zu-q&IJbqU~R91u5w@6tHWJYlk#0)}N4XPVYRx`CSGSM@!1TXl9x029Wq#2;QbF)ng zDhji5Q*(;Tb4?11%_@o$%PLFDs&dT>D>4exD>GA5(ks)_Qp`$I^HY${phMYPNa_(U z#rc_~DY>Q4$rO}@7OBQ5#aT%vDFsHw<@t%3=4ScjSvg51DP_jR$%Sd4{aWQE1;!>- zIYsG-WhrT8MalVCyQVpn>B+g7*;&S!+39)d#>Q#sS!r3tX`m7iJw=+ESy~zySQ2%a zSDA4}VQD6$rX@S3jWY@hb0B6=9@F5h9p(fs$0*A)%}P$sD#%SQH%duMOe-%*OEk$! zNy^E{%*;1QPAbW$OixZNFfUI_E6q1e0?pB3-~XOnnp##^l$}wWS6G^ql4M+!XWyU#`S!oqU;6VwZ@+Q_Eqj64UYE}W*6hd(Z*N-QE zf=|FQGQ&G^n`~5+oK|3BTA6BER+Uy^T5Ot|n37eQ47%dMG&QxXD7m1@JSnZ*I5{h) zC?%`J*t8h5jtEN|E5)=lH!m&OJl`mL_yK26L` zDK0InEYGYgNi(Y|FwHDX$t+CC%F8z@sWizqOD|4Ot}4$cN;5GrOUcd7PfRJwz*=J% z<)D+%l7t zO!K4yurYX|8_fVb@of&895**Z-LwktwVD)~CFW(DC6%P5R~RK0B_&p76qr?3WMTAL z6Z6Uv%Z*YC^7BjzjPuizlCsi@jnYevGcwAPbJI&p3v$fO%T06g^76`(@=7vuii?b~ zZhXruD^II5D>p7EE-%fg03Du}TT)i zT4h!m*o>l7&}M}6?3}#P^vv8m)7B6-D`#S*Cf(8Of!2 z1?i>P`KI~VnMI&A9~i}mrIERzxuvNgF~x|9X@+rCnvpT6RZgfFK}#3NwY-UGdYM@s z*chrLEHf)(Lp@`2)LT2@3Cjp{m`PP;Np4w1Ze@~ra$;hMQAv(D<`Fs>$>vFUW@*M{ zsb&?WnK@aBRfSdgi6sSDxrIh$m5Jsl*{LO_1xaODNjZt8*%?MdA=Ea#N z<;JOn#yKUah34sHg~d6b+W|2`-`K*^(9+l#vC$h(C)p_5)GQ?pe4jSi3Ck$kq%xxn zYzCg7NA6#n%#z%Eh|0q&6$%coEC$Ud zDWobCAKt7`k_x%KJ3Te81a!t^u0lp)Nvc9#YFWo=KXqNm@xxZdqk{Io5VZW=djZ zQf_vZd2y<7RhD_8k#SC0a#jIo{sX;$GP5u+GPE=>CvLE%*fb}}9Na!6w}2|GEH=$Z zg_uEk0cB!kY^i5vhFUS8^#D`SQZiHWi*n18s)|jDj4D&gvNQ4$i!-W{Qp+6{IF*W~NkRWM?IVPZUBMw@A+@D>hHdFRx5bD@irYEJ)8zO-m}N zs?0J^FDuD5$}a~k$v4Y1%1^DxEi@@FD>cc&I&V>4l9ZU9m6?-SWRh8Ko?2K~Y@TmY zl7lrhm|Gecnwk)|1HH1aGO^s4xXr3)SqZ7JP+D17X|EKr;YOUNiwM%D1q< zkr&I%GIO#`vMMr4^YgRvs{PV$l~%c{zgD$>);vP-JWi^`Jn)ACZ% z!0T(!OF{!9b3mMyICMG7DRVJpJ<(gNR6d7fJ?)fXIFwU+_Hm@kn$}ur7$jK;9GR;XXGEFm2O3o?D zFfB^N-bXj8sLHA;tt?8fGE2$Lsx+!d$;e48PtUD3Zb4dRZgIJBQ3~jwBGV*uvx35sBvY)VZdPiBQF3aI znR$9;VN$tiZca*No_S&tcmf(Nrp-((O%06=Oc4DI=oMgzSs7(!W%-Hel?5p!Y3aGS zMy4sINma%s1=%Gnz04>H*W^-eLB4rbN^T0YAi_SD>sqRinwOlPS5gE%j{|cg0l3gq$Jal5|bjM+_c<`oWd;A(!3M{45g_^E{JGvq~cz119PDMTO>>#<|6rMQQ03MHR_r+1WXHIXI7e zE;KDqE-y95HeixeRG4p8Xp(J~W0Gl}W1N?il2nugI^!ulIVr8E+|<-G*T}3YE2+fD zI3?3OxiCL32Wu9}$*QW%GD<8;EiX<^&dn(_I`b2Y8d3Lrb*bK^J+QiDxM9&1an8IAo)Ch06nVe&uXl|CBTv?Qy zmztfJY?5SZRG5;OT2zpfTa=TSo>*E@l%8KunNggPmXw~8nO108ioM({EHTT@EK1BP zG&8Tt%_`2UOvx@Ut;oT(>oX^(vLe3%di5<@`y@57G^;W*Gc_X}w7M~+w9v#Hbfs2G zR(f%1acZ(zo~dz0MP*`kNqTmPnMq21MWrd$_DNYmQASx#iK$s~ab|95VR=D@d1`tx zc&ie6q?=oSR?!$B?f<~joi;8n%`Gc80?%EO8R=!l<)s-_`Cv0Bk90FDBhX4#{Cm2} ziqZ>nOmj+1OEYtdN-7Fd%QMWgQ_a$|j0!W;vvSG`^FY%VnNgNenXyrJrcqi>QE^6Mnpp*o8Zsx%q_oPYxWp&}v$dLDn3rWrS7>ZvZfIzR6w{fY25V+YYFCiu&N)LgUlobvRHf<#l(^6cbf z^V~cm)BK#uqST_C%G}Zv)BK8T)1=B$)9l>x!nDMsqMRfgD-v>wO3RYWOY+JpO3QMx zjZ4jvjH`+(ii&Zhj%@Rsth|C;=y}Dc`<0TjvJ+Fw@{B91%yY9$(kly73X)1P(^5*) z3i7k_603^xijpf+$_mVq&5SC`j7!YRv0v0vTxObHl3tpUnv{~1Q*4x$Zd6#6S(t%! z-qO_A1hm+gsI@0Xpv`Fo;L%9pTFTH#exy!HnNdZGNeXCz1mSJNP~CWXD5h4Xpo_GP zL3vNH0_GtA6#)6|ytd<))@&ChC=>=H(ZGuNNyQsPHNe^3yG-$ai#fbad8F zDN0SuO-?LH&PXlNMdc*sX6mJ9rrBp?*czIe$CMfv7@3uq+Zut7^+Q?^k(#Fz3%^!3 zr?LoS7zn2pft$|xc_me4kX=qh)l=}fHlvA$E`>+y z%_Zd*=B4CklvNld7v|(ws5DM4ElJJI%S%crOHIteQ5xlz86}mN=7L(b7<CDk#cN z%gh0V95lXho>hcXV`5oiNn#PG(o}&Kfk`>}pyiuIscB$Q@SS&7N=b=n*+r?D8HJf? zmAS@gMp=oeX_Yy7M(L^9MHTrOX(ojwCMMY`r6k9+)F`dQ$fP9GJlQNYH8D9SCpRxCBP+SIq$n$~Bqi0v z1l0FWPc%16&MnExOCq9xF)}wXGcW_4)l`(4Sdxl-1&@WUk!6UXrImphXn~0_WJm+` z%Ee6eoE(MX)KrDa{8EL?JcY!({F02+B88mHv{Z%ijMO}Z@>GSyqEv;X{E`fXBsHxdxhko&Dy68h%q-nJBR{7w%Oo?kv^XczI3u;f z7{};`d3sTPMW$(5dYVa5Ri$}(rnz}#Zdn1&eGa7==EjL>*+y6nwof!OHYq4gOEfPv zDK@W2&Pz4UF)lAJGOMg8FG_jG zD~pRWi@|0Psyv8^X=6)0Q^*QPT+6aj%t}&BOU=tGtEx&0iVDh2b1L&IbMp!_($h<_ z&2lpG%2JZEP4ZI;vWkq-^0RXi3o<|jAf)O>S(a6jmu*&1S!QCAnU`*sWSX0tm|>P` zmWJz0tK5=Q^CF|dbR$#DZcbWZWs*r!R(fGhc2aJYky%zoaRKPGg{)+=?3|3!vZQ?D zf*hlqG?S9#g0!l%;~tQ;Z7=Q_M5cD@(9c9cCs5riRAF zsbx5;x{{nCqmoQhqY_N_CgogA<`!nghNk8w z#7q~J8CRqfC#4#J7dMev27%1TO)CYPL3zpqcfif@mq8iXrj^-Q=0$16xrvD-iRGDz z#py=LX6cow$tGEaDWz4J`6b55S!rek<&{RoRmoY!#yGlIq#jY5QBa;$Zkl0~hZ)nU zdHEIPMI}W>Sthyp`DrO-g&8KLCPw9XxkiOK<2nWk7{ zIyX75$|yG}(KMqZCAYBHyr>v-C`}o71Qe}NV`gY+Xk=t&ju@iGJ=9WKUQwJvX>C(p zQEW=J+QtYp)M8$#^fU zgzLxCL^1)bpEEYWAKQ6}#)a9H*%f6O#+6xF`4#zD8Rf>Q#)awW$t9+#W`!n6Rc2+W zWw|-|$;qWfY2{{VIGRXVCFVvIRVCRurN${n$r)uiIYt?oRYhjF`dld$WhEJAIap`V z5{pYr(sGMUt165OG7EByOY+Q$Q%qBh%96~}3eC!@Dl&4?(m;FpDocy2s?v?K%dw6R znx|!z=4MnGC6=0}8s%1Im}lpjW~3EhJL%Ef5VYOgg1FeOD#|Y~PXq6SB)f@JRg|9! z>bl|I?Tp;O!V}Y`R)%JJ28OUPf9Uw1iMgJo3Eml^JE`mFJ~nJN+>!tE9BBFfGHpBqJp=uc|a9sl>D_J=M50GdsW1Jh!wY zx1_Kjt1{iBFr_rjD8H~Gvk>cItMb%bBV*I#%&N?a!rY|f@(j>zX`n;OQ;I8!igGGTl8p<^)6=VR(u;~sOLJ3-(kpS~bMu^ZqufH%w8FBI z;)3LqQlrA`qO8&!oHONF<%!0{rRhc$n5BAhK}AYIaz;){R&H8hc~x;~l5u`{QK?aK zig9^WK}u?Jrg>6wfpI}@YL#hnMq#3926#FdV~0wmS$=9-c4|p!W?7keR!U-eaao#C zT46bs2C}&Y=x{ev1ELOyG_I;l%t!{G#!6gFL&s{7%Jeehs!Ee=h#7=p8mb#lJ~y*6 zvd}ZL0Iz_@KBaDCppcPoU}S=H_m45C?P6$&WAm?Rc}cR7ab|W(3TVG_N};()S$d{< z1?J}8q%$(#)T)AvjP&y2N^{f1 z)bv!Wg>rI|Q9(*-c5X^)RfV}}aamGXiCKD95@_@YqY5@KGcg1$AtGX}acOx@aaDdU zc-I}l$U%;eIPoRVzQw4AIoqr#-B ze6!ptv!b%Xw5lwuqXwzv1^HP;M&NrWD|3pCjZ$*6iz`e)EjaYhH?=f1H?**@ByQ9o zDL19C1TxM#p>J$uV6JCsh&t>H5B+pwW25vG)8dlsiqeYI zRCCZlv-v5-$x5JMyiBC#XG)=2SwT@wcD}iBd0w)yX<}(snMrP0PHAaYUUFtpS#nVh zXoAipBR8wqxU{^YG9?jf=$jdvR+wd(=B1bAWF{Mzm?oJe8Jkz-S70lsOhH=+OpS3>wp~NG>$XFG)!^ zGDGq5lRv zCVWcE^9l>nvcSjc5iA>!n_MVKGp{f!s|sueo}`Ja8&9cV3@XTt@ZW=xSX7djQJ9xf zQks`pno(L-k)2UoV4PW*QdVx7lapAUnqHP^RGw0hT4SQB_7& zs##%Ca(b0%im6d+~D!a-kJFzU^$fP*4*tj^u zyfP&_**q%=Yb{`5XbIZdG9D8CmIOgl=pmGN#R}OfB`y zEm4<`qK{LXBv+K@n&hSCXXlm`Wu;`48s#RLRG22EmKA0gm82JBmKBzyXO`zARTP2l z-7`-IpTLJDrVDd(%rdhe`+t**iYkoKva0fP%knd-DwBZSicuQ))DYqd7-(A$sel2Q zVP<9uHiJ+B1J#YEfH4DAd`8CL-XHdsVo0F7LPlmDXtiZ>W`15`jzUVJf^&Y(;VrpI zpwj`+R%a!pq^ISVn3bg@7N-~)m6=zXn--ar7iVPW8|RivewC1#eGm1ZSX zrYBYvR$|>_oLF30R9R}2S!ixvo|J1=k(iNMU{+Cv^-NB43j;$#V`E}gWt3JHRHf#Z z7=hdTgc=~oSr;h=KxUYgq=3yJ6a&Qc%8bqRjLq=&%91KG$|@=gi}K7%%9FE=3rlkH zb8@RHD)W+4j4M-9bIOd;vU9SEO-;;F($b1D%~H}*K{I(+GJbYNMNUywxlw6)QigE> zWHl1lo1mnkZ=3@qztr2uvSM2>%|}$85J2O6{bcN8K!x8#hFQF~uf7|2B;o_Rr;`51*sT28TP zj(N7RaanOmc3MtJc9~hSnQ2i~YFTPwR+*V`d5KwNW^tBDRYqy5aZz?kVs19pg@#p$ zrpZNx#ii!iiRG!;xn>1r#yM%}N!U&rGqp4aEyXt@YA~@pCDXJh-v~TRMO0FT4+bJ9 z=JJ$`igGisDTER;TtA)y#l*_MM9t=o0^rBQdp6jQc_TymX}kWQJ8B~n3`Fbnq_QUXp&`=U6zuUf_1K=AU`*) zFh8#(sjAqtG&QL#xvV&^C_M|?E=@B_BMUaVGc8O^1RwrRT-5_kQ2Xuvml8#bhaY1TwNf79AGmsG(shQ~+B}!IC2B3R4$}>|+GL)yP%UR9NGMpdR zG&ip(%gX~Ta>~UzHd9bhRZwPHT$)^3Ra%*qQkk5domG^c0cww;kIk4Hnwc0Hni&uk z(^aMUDHSH*O-01TG_=5kr%;pJ(yG#YbE8tQ8Ab5S3@e9FV;ZU(YfKx27=rE!v@$i+ NGcW*;mlJ#98UX)HUhe<^ diff --git a/jetstream/.gitignore b/jetstream/.gitignore deleted file mode 100644 index a526342..0000000 --- a/jetstream/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -target/ -pkg/ -**/*.rs.bk -dist/ -traces/ -*.DS_Store -.cargo/ -.env diff --git a/jetstream/Cargo.toml b/jetstream/Cargo.toml deleted file mode 100644 index 96bd42d..0000000 --- a/jetstream/Cargo.toml +++ /dev/null @@ -1,38 +0,0 @@ -[package] -authors = ["videah ", "phil "] -name = "jetstream" -version = "0.1.1" -edition = "2021" -license = "MIT" -description = "Library for easily interacting with and consuming the Bluesky Jetstream service." -repository = "https://github.com/at-microcosm/links" -readme = "README.md" - -[dependencies] -async-trait = "0.1.83" -atrium-api = { version = "0.25.4", default-features = false, features = [ - "namespace-appbsky", -] } -tokio = { version = "1.44.2", features = ["full", "sync", "time"] } -tokio-tungstenite = { version = "0.26.2", features = [ - "connect", - "native-tls-vendored", - "url", -] } -futures-util = "0.3.31" -metrics = { version = "0.24.2", optional = true } -url = "2.5.4" -serde = { version = "1.0.215", features = ["derive"] } -serde_json = { version = "1.0.140", features = ["raw_value"] } -chrono = "0.4.38" -zstd = "0.13.2" -thiserror = "2.0.3" -log = "0.4.22" - -[dev-dependencies] -anyhow = "1.0.93" -clap = { version = "4.5.20", features = ["derive"] } - -[features] -default = [] -metrics = ["dep:metrics"] diff --git a/jetstream/LICENSE b/jetstream/LICENSE deleted file mode 100644 index 89cf4bc..0000000 --- a/jetstream/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2024 videah - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file diff --git a/jetstream/README.md b/jetstream/README.md deleted file mode 100644 index cd96072..0000000 --- a/jetstream/README.md +++ /dev/null @@ -1,68 +0,0 @@ -# fork of the awesome jetstream-oxide - -fork note: this readme is likely a bit out of date! i've been messing around with some apis. - -A typed Rust library for easily interacting with and consuming the -Bluesky [Jetstream](https://github.com/bluesky-social/jetstream) -service. - -```rust -let config = JetstreamConfig { - endpoint: DefaultJetstreamEndpoints::USEastOne.into(), - compression: JetstreamCompression::Zstd, - ..Default::default() -}; - -let jetstream = JetstreamConnector::new(config).unwrap(); -let receiver = jetstream.connect().await?; - -while let Ok(event) = receiver.recv_async().await { - if let Commit(commit) = event { - match commit { - CommitEvent::Create { info, commit } => { - println!("Received create event: {:#?}", info); - } - CommitEvent::Update { info, commit } => { - println!("Received update event: {:#?}", info); - } - CommitEvent::Delete { info, commit } => { - println!("Received delete event: {:#?}", info); - } - } - } -} -``` - -## Example - -A small example CLI utility to show how to use this crate can be found in the `examples` directory. To run it, use the -following command: - -```sh -cargo run --example basic -- --nsid "app.bsky.feed.post" -``` - -This will display a real-time feed of every single post that is being made or deleted in the entire Bluesky network, -right in your terminal! - -You can filter it down to just specific accounts like this: - -```sh -cargo run --example basic -- \ ---nsid "app.bsky.feed.post" \ ---did "did:plc:inze6wrmsm7pjl7yta3oig77" -``` - -This listens for posts that *I personally make*. You can substitute your own DID and make a few test posts yourself if -you'd -like of course! - - -## Running `rustfmt` (requires nightly) - -```bash -# get nightly set up -rustup toolchain install nightly --allow-downgrade -c rustfmt -# run the nightly version of fmt -cargo +nightly fmt -``` diff --git a/jetstream/examples/arbitrary_record.rs b/jetstream/examples/arbitrary_record.rs deleted file mode 100644 index fd162d1..0000000 --- a/jetstream/examples/arbitrary_record.rs +++ /dev/null @@ -1,74 +0,0 @@ -//! An example of how to listen for create/delete events on a specific DID and potentialy unknown -//! NSID - -use atrium_api::types::string; -use clap::Parser; -use jetstream::{ - events::{ - CommitOp, - EventKind, - JetstreamEvent, - }, - DefaultJetstreamEndpoints, - JetstreamCompression, - JetstreamConfig, - JetstreamConnector, -}; - -#[derive(Parser, Debug)] -#[command(version, about, long_about = None)] -struct Args { - /// The DIDs to listen for events on, if not provided we will listen for all DIDs. - #[arg(short, long)] - did: Option>, - /// The NSID for the collection to listen for (e.g. `blue.flashes.feed.post`). - #[arg(short, long)] - nsid: string::Nsid, -} - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - let args = Args::parse(); - - let dids = args.did.unwrap_or_default(); - let config: JetstreamConfig = JetstreamConfig { - endpoint: DefaultJetstreamEndpoints::USEastOne.into(), - wanted_collections: vec![args.nsid.clone()], - wanted_dids: dids.clone(), - compression: JetstreamCompression::Zstd, - ..Default::default() - }; - - let jetstream = JetstreamConnector::new(config)?; - let mut receiver = jetstream.connect().await?; - - println!( - "Listening for new and updated '{}' events on DIDs: {:?}", - args.nsid.as_str(), - dids - ); - - while let Some(event) = receiver.recv().await { - if let JetstreamEvent { - kind: EventKind::Commit, - commit: Some(commit), - .. - } = event - { - if commit.collection != args.nsid { - continue; - } - if !(commit.operation == CommitOp::Create || commit.operation == CommitOp::Update) { - continue; - } - let Some(rec) = commit.record else { continue }; - println!( - "New or updated record! ({})\n{:?}\n", - commit.rkey.as_str(), - rec.get() - ); - } - } - - Ok(()) -} diff --git a/jetstream/examples/basic.rs b/jetstream/examples/basic.rs deleted file mode 100644 index adb7282..0000000 --- a/jetstream/examples/basic.rs +++ /dev/null @@ -1,67 +0,0 @@ -//! A very basic example of how to listen for create/delete events on a specific DID and NSID. - -use atrium_api::{ - record::KnownRecord::AppBskyFeedPost, - types::string, -}; -use clap::Parser; -use jetstream::{ - events::{ - CommitEvent, - CommitOp, - EventKind, - JetstreamEvent, - }, - DefaultJetstreamEndpoints, - JetstreamCompression, - JetstreamConfig, - JetstreamConnector, -}; - -#[derive(Parser, Debug)] -#[command(version, about, long_about = None)] -struct Args { - /// The DIDs to listen for events on, if not provided we will listen for all DIDs. - #[arg(short, long)] - did: Option>, -} - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - let args = Args::parse(); - - let dids = args.did.unwrap_or_default(); - let config = JetstreamConfig { - endpoint: DefaultJetstreamEndpoints::USEastOne.into(), - wanted_collections: vec![string::Nsid::new("app.bsky.feed.post".to_string()).unwrap()], - wanted_dids: dids.clone(), - compression: JetstreamCompression::Zstd, - ..Default::default() - }; - - let jetstream = JetstreamConnector::new(config)?; - let mut receiver = jetstream.connect().await?; - - println!("Listening for 'app.bsky.feed.post' events on DIDs: {dids:?}"); - - while let Some(event) = receiver.recv().await { - if let JetstreamEvent { - kind: EventKind::Commit, - commit: - Some(CommitEvent { - operation: CommitOp::Create, - rkey, - record: Some(record), - .. - }), - .. - } = event - { - if let Ok(AppBskyFeedPost(rec)) = serde_json::from_str(record.get()) { - println!("New post created! ({})\n{:?}\n", rkey.as_str(), rec.text); - } - } - } - - Ok(()) -} diff --git a/jetstream/rustfmt.toml b/jetstream/rustfmt.toml deleted file mode 100644 index 57478ce..0000000 --- a/jetstream/rustfmt.toml +++ /dev/null @@ -1,6 +0,0 @@ -group_imports = "StdExternalCrate" -comment_width = 100 -format_code_in_doc_comments = true -imports_granularity = "Crate" -imports_layout = "Vertical" -wrap_comments = true \ No newline at end of file diff --git a/jetstream/src/error.rs b/jetstream/src/error.rs deleted file mode 100644 index 8fdfc39..0000000 --- a/jetstream/src/error.rs +++ /dev/null @@ -1,47 +0,0 @@ -//! Various error types. -use std::io; - -use thiserror::Error; - -/// Possible errors that can occur when a [JetstreamConfig](crate::JetstreamConfig) that is passed -/// to a [JetstreamConnector](crate::JetstreamConnector) is invalid. -#[derive(Error, Debug)] -pub enum ConfigValidationError { - #[error("too many wanted collections: {0} > 100")] - TooManyWantedCollections(usize), - #[error("too many wanted DIDs: {0} > 10,000")] - TooManyDids(usize), - #[error("invalid endpoint: {0}")] - InvalidEndpoint(#[from] url::ParseError), -} - -/// Possible errors that can occur in the process of connecting to a Jetstream instance over -/// WebSockets. -/// -/// See [JetstreamConnector::connect](crate::JetstreamConnector::connect). -#[derive(Error, Debug)] -pub enum ConnectionError { - #[error("invalid endpoint: {0}")] - InvalidEndpoint(#[from] url::ParseError), - #[error("failed to construct url: {0}")] - InvalidEndpointUri(#[from] tokio_tungstenite::tungstenite::http::uri::InvalidUri), - #[error("failed to connect to Jetstream instance: {0}")] - WebSocketFailure(#[from] tokio_tungstenite::tungstenite::Error), - #[error("the Jetstream config is invalid (this really should not happen here): {0}")] - InvalidConfig(#[from] ConfigValidationError), -} - -/// Possible errors that can occur when receiving events from a Jetstream instance over WebSockets. -/// -/// See [websocket_task](crate::websocket_task). -#[derive(Error, Debug)] -pub enum JetstreamEventError { - #[error("failed to load built-in zstd dictionary for decoding: {0}")] - CompressionDictionaryError(io::Error), - #[error("failed to send ping or pong: {0}")] - PingPongError(#[from] tokio_tungstenite::tungstenite::Error), - #[error("no messages received within ttl")] - NoMessagesReceived, - #[error("jetstream event receiver closed")] - ReceiverClosedError, -} diff --git a/jetstream/src/events.rs b/jetstream/src/events.rs deleted file mode 100644 index 78c3227..0000000 --- a/jetstream/src/events.rs +++ /dev/null @@ -1,217 +0,0 @@ -use std::time::{ - Duration, - SystemTime, - SystemTimeError, - UNIX_EPOCH, -}; - -use chrono::Utc; -use serde::{ - Deserialize, - Serialize, -}; -use serde_json::value::RawValue; - -use crate::exports; - -/// Opaque wrapper for the time_us cursor used by jetstream -#[derive(Debug, Serialize, Deserialize, Copy, Clone, PartialEq, PartialOrd)] -pub struct Cursor(u64); - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "snake_case")] -pub struct JetstreamEvent { - #[serde(rename = "time_us")] - pub cursor: Cursor, - pub did: exports::Did, - pub kind: EventKind, - pub commit: Option, - pub identity: Option, - pub account: Option, -} - -#[derive(Debug, Deserialize, PartialEq)] -#[serde(rename_all = "snake_case")] -pub enum EventKind { - Commit, - Identity, - Account, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "snake_case")] -pub struct CommitEvent { - pub collection: exports::Nsid, - pub rkey: exports::RecordKey, - pub rev: String, - pub operation: CommitOp, - pub record: Option>, - pub cid: Option, -} - -#[derive(Debug, Deserialize, PartialEq)] -#[serde(rename_all = "snake_case")] -pub enum CommitOp { - Create, - Update, - Delete, -} - -#[derive(Debug, Deserialize, PartialEq)] -pub struct IdentityEvent { - pub did: exports::Did, - pub handle: Option, - pub seq: u64, - pub time: chrono::DateTime, -} - -#[derive(Debug, Deserialize, PartialEq)] -pub struct AccountEvent { - pub active: bool, - pub did: exports::Did, - pub seq: u64, - pub time: chrono::DateTime, - pub status: Option, -} - -impl Cursor { - /// Get a cursor that will consume all available jetstream replay - /// - /// This sets the cursor to zero. - /// - /// Jetstream instances typically only have a few days of replay. - pub fn from_start() -> Self { - Self(0) - } - /// Get a cursor for a specific time - /// - /// Panics: if t is older than the unix epoch: Jan 1, 1970. - /// - /// If you want to receive all available jetstream replay (typically a few days), use - /// .from_start() - /// - /// Warning: this exploits the internal implementation detail of jetstream cursors - /// being ~microsecond timestamps. - pub fn at(t: impl Into) -> Self { - let unix_dt = t - .into() - .duration_since(UNIX_EPOCH) - .expect("cannot set jetstream cursor earlier than unix epoch"); - Self(unix_dt.as_micros() as u64) - } - /// Get a cursor rewound from now by this amount - /// - /// Panics: if d is greater than the time since the unix epoch: Jan 1, 1970. - /// - /// Jetstream instances typically only have a few days of replay. - /// - /// Warning: this exploits the internal implementation detail of jetstream cursors - /// being ~microsecond timestamps. - pub fn back_by(d: Duration) -> Self { - Self::at(SystemTime::now() - d) - } - /// Get a Cursor from a raw u64 - /// - /// For example, from a jetstream event's `time_us` field. - pub fn from_raw_u64(time_us: u64) -> Self { - Self(time_us) - } - /// Get the raw u64 value from this cursor. - pub fn to_raw_u64(&self) -> u64 { - self.0 - } - /// Format the cursor value for use in a jetstream connection url querystring - pub fn to_jetstream(&self) -> String { - self.0.to_string() - } - /// Compute the time span since an earlier cursor or [SystemTime] - /// - /// Warning: this exploits the internal implementation detail of jetstream cursors - /// being ~microsecond timestamps. - pub fn duration_since( - &self, - earlier: impl Into, - ) -> Result { - let t: SystemTime = self.into(); - t.duration_since(earlier.into()) - } - /// Compute the age of the cursor vs the local clock - /// - /// Warning: this exploits the internal implementation detail of jetstream cursors - pub fn elapsed(&self) -> Result { - let t: SystemTime = self.into(); - t.elapsed() - } - /// Compute the age of the cursor vs the local clock - /// - /// Converts the resulting duration into an f64, which can be negative! - /// - /// Warning: this exploits the internal implementation detail of jetstream cursors - pub fn elapsed_micros_f64(&self) -> f64 { - match self.elapsed() { - Ok(d) => d.as_micros() as f64, - Err(e) => -(e.duration().as_micros() as f64), - } - } - /// Get the immediate next cursor value - /// - /// This is possible for the implementation of jetstream cursors - pub fn next(&self) -> Cursor { - Self(self.0 + 1) - } -} - -impl From<&Cursor> for SystemTime { - /// Convert a cursor directly to a [SystemTime] - /// - /// Warning: this exploits the internal implementation detail of jetstream cursors - /// being ~microsecond timestamps. - fn from(c: &Cursor) -> Self { - UNIX_EPOCH + Duration::from_micros(c.0) - } -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_parse_commit_event() -> anyhow::Result<()> { - let json = r#"{ - "rev":"3llrdsginou2i", - "operation":"create", - "collection":"app.bsky.feed.post", - "rkey":"3llrdsglqdc2s", - "cid": "bafyreidofvwoqvd2cnzbun6dkzgfucxh57tirf3ohhde7lsvh4fu3jehgy", - "record": {"$type":"app.bsky.feed.post","createdAt":"2025-04-01T16:58:06.154Z","langs":["en"],"text":"I wish apirl 1st would stop existing lol"} - }"#; - let commit: CommitEvent = serde_json::from_str(json)?; - assert_eq!( - commit.cid.unwrap(), - "bafyreidofvwoqvd2cnzbun6dkzgfucxh57tirf3ohhde7lsvh4fu3jehgy".parse()? - ); - assert_eq!( - commit.record.unwrap().get(), - r#"{"$type":"app.bsky.feed.post","createdAt":"2025-04-01T16:58:06.154Z","langs":["en"],"text":"I wish apirl 1st would stop existing lol"}"# - ); - Ok(()) - } - - #[test] - fn test_parse_whole_event() -> anyhow::Result<()> { - let json = r#"{"did":"did:plc:ai3dzf35cth7s3st7n7jsd7r","time_us":1743526687419798,"kind":"commit","commit":{"rev":"3llrdsginou2i","operation":"create","collection":"app.bsky.feed.post","rkey":"3llrdsglqdc2s","record":{"$type":"app.bsky.feed.post","createdAt":"2025-04-01T16:58:06.154Z","langs":["en"],"text":"I wish apirl 1st would stop existing lol"},"cid":"bafyreidofvwoqvd2cnzbun6dkzgfucxh57tirf3ohhde7lsvh4fu3jehgy"}}"#; - let event: JetstreamEvent = serde_json::from_str(json)?; - assert_eq!(event.kind, EventKind::Commit); - assert!(event.commit.is_some()); - let commit = event.commit.unwrap(); - assert_eq!( - commit.cid.unwrap(), - "bafyreidofvwoqvd2cnzbun6dkzgfucxh57tirf3ohhde7lsvh4fu3jehgy".parse()? - ); - assert_eq!( - commit.record.unwrap().get(), - r#"{"$type":"app.bsky.feed.post","createdAt":"2025-04-01T16:58:06.154Z","langs":["en"],"text":"I wish apirl 1st would stop existing lol"}"# - ); - Ok(()) - } -} diff --git a/jetstream/src/exports.rs b/jetstream/src/exports.rs deleted file mode 100644 index ae303e8..0000000 --- a/jetstream/src/exports.rs +++ /dev/null @@ -1,9 +0,0 @@ -//! Useful exports for third-party crates used by this project. - -pub use atrium_api::types::string::{ - Cid, - Did, - Handle, - Nsid, - RecordKey, -}; diff --git a/jetstream/src/lib.rs b/jetstream/src/lib.rs deleted file mode 100644 index 29a2c15..0000000 --- a/jetstream/src/lib.rs +++ /dev/null @@ -1,646 +0,0 @@ -pub mod error; -pub mod events; -pub mod exports; - -use std::{ - io::Cursor as IoCursor, - time::{ - Duration, - Instant, - }, -}; - -use futures_util::{ - stream::StreamExt, - SinkExt, -}; -#[cfg(feature = "metrics")] -use metrics::{ - counter, - describe_counter, - Unit, -}; -use tokio::{ - net::TcpStream, - sync::mpsc::{ - channel, - Receiver, - Sender, - }, - time::timeout, -}; -use tokio_tungstenite::{ - connect_async, - tungstenite::{ - client::{ - ClientRequestBuilder, - IntoClientRequest, - }, - handshake::client::Request, - Message, - }, - MaybeTlsStream, - WebSocketStream, -}; -use url::Url; -use zstd::dict::DecoderDictionary; - -use crate::{ - error::{ - ConfigValidationError, - ConnectionError, - JetstreamEventError, - }, - events::{ - Cursor, - JetstreamEvent, - }, -}; - -/// The Jetstream endpoints officially provided by Bluesky themselves. -/// -/// There are no guarantees that these endpoints will always be available, but you are free -/// to run your own Jetstream instance in any case. -pub enum DefaultJetstreamEndpoints { - /// `jetstream1.us-east.bsky.network` - USEastOne, - /// `jetstream2.us-east.bsky.network` - USEastTwo, - /// `jetstream1.us-west.bsky.network` - USWestOne, - /// `jetstream2.us-west.bsky.network` - USWestTwo, -} - -impl DefaultJetstreamEndpoints { - /// Helper to reference official jetstream instances by shortcut - /// - /// This function will pass through a jetstream endpoint URL unless it matches a shortcut, - /// in which case it will be rewritten to the corresponding bluesky-operated jetstream endpoint - /// URL. - /// - /// The shortcuts available are - /// - 'us-east-1' - /// - 'us-east-2' - /// - 'us-west-1' - /// - 'us-west-2' - pub fn endpoint_or_shortcut(s: &str) -> String { - match s { - "us-east-1" => DefaultJetstreamEndpoints::USEastOne.into(), - "us-east-2" => DefaultJetstreamEndpoints::USEastTwo.into(), - "us-west-1" => DefaultJetstreamEndpoints::USWestOne.into(), - "us-west-2" => DefaultJetstreamEndpoints::USWestTwo.into(), - custom => custom.into(), - } - } -} - -impl From for String { - fn from(endpoint: DefaultJetstreamEndpoints) -> Self { - match endpoint { - DefaultJetstreamEndpoints::USEastOne => { - "wss://jetstream1.us-east.bsky.network/subscribe".to_owned() - } - DefaultJetstreamEndpoints::USEastTwo => { - "wss://jetstream2.us-east.bsky.network/subscribe".to_owned() - } - DefaultJetstreamEndpoints::USWestOne => { - "wss://jetstream1.us-west.bsky.network/subscribe".to_owned() - } - DefaultJetstreamEndpoints::USWestTwo => { - "wss://jetstream2.us-west.bsky.network/subscribe".to_owned() - } - } - } -} - -/// The maximum number of wanted collections that can be requested on a single Jetstream connection. -const MAX_WANTED_COLLECTIONS: usize = 100; -/// The maximum number of wanted DIDs that can be requested on a single Jetstream connection. -const MAX_WANTED_DIDS: usize = 10_000; - -/// The custom `zstd` dictionary used for decoding compressed Jetstream messages. -/// -/// Sourced from the [official Bluesky Jetstream repo.](https://github.com/bluesky-social/jetstream/tree/main/pkg/models) -const JETSTREAM_ZSTD_DICTIONARY: &[u8] = include_bytes!("../zstd/dictionary"); - -/// A receiver channel for consuming Jetstream events. -pub type JetstreamReceiver = Receiver; - -/// An internal sender channel for sending Jetstream events to [JetstreamReceiver]'s. -type JetstreamSender = Sender; - -/// A wrapper connector type for working with a WebSocket connection to a Jetstream instance to -/// receive and consume events. See [JetstreamConnector::connect] for more info. -pub struct JetstreamConnector { - /// The configuration for the Jetstream connection. - config: JetstreamConfig, -} - -pub enum JetstreamCompression { - /// No compression, just raw plaintext JSON. - None, - /// Use the `zstd` compression algorithm, which can result in a ~56% smaller messages on - /// average. See [here](https://github.com/bluesky-social/jetstream?tab=readme-ov-file#compression) for more info. - Zstd, -} - -impl From for bool { - fn from(compression: JetstreamCompression) -> Self { - match compression { - JetstreamCompression::None => false, - JetstreamCompression::Zstd => true, - } - } -} - -impl From for JetstreamCompression { - fn from(compress: bool) -> Self { - if compress { - JetstreamCompression::Zstd - } else { - JetstreamCompression::None - } - } -} - -pub struct JetstreamConfig { - /// A Jetstream endpoint to connect to with a WebSocket Scheme i.e. - /// `wss://jetstream1.us-east.bsky.network/subscribe`. - pub endpoint: String, - /// A list of collection [NSIDs](https://atproto.com/specs/nsid) to filter events for. - /// - /// An empty list will receive events for *all* collections. - /// - /// Regardless of desired collections, all subscribers receive - /// [AccountEvent](events::account::AccountEvent) and - /// [IdentityEvent](events::identity::Identity) events. - pub wanted_collections: Vec, - /// A list of repo [DIDs](https://atproto.com/specs/did) to filter events for. - /// - /// An empty list will receive events for *all* repos, which is a lot of events! - pub wanted_dids: Vec, - /// The compression algorithm to request and use for the WebSocket connection (if any). - pub compression: JetstreamCompression, - /// User agent string to include with the jetstream connection request - pub user_agent: Option, - /// Do not append jetstream client info to user agent string - pub omit_user_agent_jetstream_info: bool, - /// Enable automatic cursor for auto-reconnect - /// - /// By default, reconnects will never set a cursor for the connection, so a small number of - /// events will always be dropped. - /// - /// If you want gapless playback across reconnects, set this to `true`. If you always want - /// the latest available events and can tolerate missing some: `false`. - pub replay_on_reconnect: bool, - /// Maximum size of send channel for jetstream events. - /// - /// If your consuming task can't keep up with every new jetstream event in real-time, - /// you might get disconnected from the server as a "slow consumer". Increasing channel_size - /// can help prevent that if your consumer sometimes pauses, at a cost of higher memory - /// usage while events are buffered. - pub channel_size: usize, - /// How long since the last jetstream message before we consider the connection dead - /// - /// Default: 15s - pub liveliness_ttl: Duration, -} - -impl Default for JetstreamConfig { - fn default() -> Self { - JetstreamConfig { - endpoint: DefaultJetstreamEndpoints::USEastOne.into(), - wanted_collections: Vec::new(), - wanted_dids: Vec::new(), - compression: JetstreamCompression::None, - user_agent: None, - omit_user_agent_jetstream_info: false, - replay_on_reconnect: false, - channel_size: 4096, // a few seconds of firehose buffer - liveliness_ttl: Duration::from_secs(15), - } - } -} - -impl JetstreamConfig { - /// Constructs a new endpoint URL with the given [JetstreamConfig] applied. - pub fn get_request_builder( - &self, - ) -> impl Fn(Option) -> Result { - let did_search_query = self - .wanted_dids - .iter() - .map(|s| ("wantedDids", s.to_string())); - - let collection_search_query = self - .wanted_collections - .iter() - .map(|s| ("wantedCollections", s.to_string())); - - let compression = ( - "compress", - match self.compression { - JetstreamCompression::None => "false".to_owned(), - JetstreamCompression::Zstd => "true".to_owned(), - }, - ); - - let base_params = did_search_query - .chain(collection_search_query) - .chain(std::iter::once(compression)) - .collect::>(); - - let ua_info: Option = if self.omit_user_agent_jetstream_info { - None - } else { - Some(format!( - "v{} via jetstream-oxide (microcosm/links fork)", - env!("CARGO_PKG_VERSION") - )) - }; - let maybe_ua = match (&self.user_agent, ua_info) { - (Some(ua), Some(info)) => Some(format!("{ua} {info}")), - (Some(ua), None) => Some(ua.clone()), - (None, Some(info)) => Some(info.clone()), - (None, None) => None, - }; - - let endpoint = self.endpoint.clone(); - move |maybe_cursor: Option| { - let mut params = base_params.clone(); - if let Some(ref cursor) = maybe_cursor { - params.push(("cursor", cursor.to_jetstream())); - } - let url = Url::parse_with_params(&endpoint, params)?; - - let mut req = ClientRequestBuilder::new(url.as_str().parse()?); - if let Some(ua) = &maybe_ua { - req = req.with_header("user-agent", ua) - }; - Ok(req.into_client_request()?) - } - } - - /// Validates the configuration to make sure it is within the limits of the Jetstream API. - /// - /// # Constants - /// The following constants are used to validate the configuration and should only be changed - /// if the Jetstream API has itself changed. - /// - [MAX_WANTED_COLLECTIONS] - /// - [MAX_WANTED_DIDS] - /// - /// # Endpoint - /// - /// The provided `endpoint` is attempted to be parsed so that any errors occur early. - pub fn validate(&self) -> Result<(), ConfigValidationError> { - let collections = self.wanted_collections.len(); - let dids = self.wanted_dids.len(); - - if collections > MAX_WANTED_COLLECTIONS { - return Err(ConfigValidationError::TooManyWantedCollections(collections)); - } - - if dids > MAX_WANTED_DIDS { - return Err(ConfigValidationError::TooManyDids(dids)); - } - - let _ = self.endpoint.parse::()?; - - Ok(()) - } -} - -#[cfg(feature = "metrics")] -fn describe_metrics() { - describe_counter!( - "jetstream_connects", - Unit::Count, - "how many times we've tried to connect" - ); - describe_counter!( - "jetstream_disconnects", - Unit::Count, - "how many times we've been disconnected" - ); - describe_counter!( - "jetstream_total_events_received", - Unit::Count, - "total number of events received" - ); - describe_counter!( - "jetstream_total_bytes_received", - Unit::Count, - "total uncompressed bytes received, not including websocket overhead" - ); - describe_counter!( - "jetstream_total_event_errors", - Unit::Count, - "total errors when handling events" - ); - describe_counter!( - "jetstream_total_events_sent", - Unit::Count, - "total events sent to the consumer" - ); -} - -impl JetstreamConnector { - /// Create a Jetstream connector with a valid [JetstreamConfig]. - /// - /// After creation, you can call [connect] to connect to the provided Jetstream instance. - pub fn new(config: JetstreamConfig) -> Result { - #[cfg(feature = "metrics")] - describe_metrics(); - - // We validate the configuration here so any issues are caught early. - config.validate()?; - Ok(JetstreamConnector { config }) - } - - /// Connects to a Jetstream instance as defined in the [JetstreamConfig]. - /// - /// A [JetstreamReceiver] is returned which can be used to respond to events. When all instances - /// of this receiver are dropped, the connection and task are automatically closed. - pub async fn connect(&self) -> Result { - self.connect_cursor(None).await - } - - /// Connects to a Jetstream instance as defined in the [JetstreamConfig] with playback from a - /// cursor - /// - /// A cursor from the future will result in live-tail operation. - /// - /// The cursor is only used for first successfull connection -- on auto-reconnect it will - /// live-tail by default. Set `replay_on_reconnect: true` in the config if you need to - /// receive every event, which will keep track of the last-seen cursor and reconnect from - /// there. - pub async fn connect_cursor( - &self, - cursor: Option, - ) -> Result { - // We validate the config again for good measure. Probably not necessary but it can't hurt. - self.config - .validate() - .map_err(ConnectionError::InvalidConfig)?; - - let (send_channel, receive_channel) = channel(self.config.channel_size); - let replay_on_reconnect = self.config.replay_on_reconnect; - let liveliness_ttl = self.config.liveliness_ttl; - let build_request = self.config.get_request_builder(); - - tokio::task::spawn(async move { - // TODO: maybe return the task handle so we can surface any errors - let max_retries = 300; - let base_delay_ms = 1_000; // 1 second - let max_delay_ms = 30_000; // 30 seconds - let success_threshold_s = 15; // 15 seconds, retry count is reset if we were connected at least this long - - let mut retry_attempt = 0; - let mut connect_cursor = cursor; - loop { - let dict = DecoderDictionary::copy(JETSTREAM_ZSTD_DICTIONARY); - - let req = match build_request(connect_cursor) { - Ok(req) => req, - Err(e) => { - log::error!("Could not build jetstream websocket request: {e:?}"); - break; // this is always fatal? no retry. - } - }; - - #[cfg(feature = "metrics")] - if let Some(host) = req.uri().host() { - let retry = if retry_attempt > 0 { "yes" } else { "no" }; - counter!("jetstream_connects", "host" => host.to_string(), "retry" => retry) - .increment(1); - } - - let mut last_cursor = connect_cursor; - retry_attempt += 1; - if let Ok((ws_stream, _)) = connect_async(req).await { - let t_connected = Instant::now(); - log::info!("jetstream connected. starting websocket task..."); - if let Err(e) = websocket_task( - dict, - ws_stream, - send_channel.clone(), - &mut last_cursor, - liveliness_ttl, - ) - .await - { - match e { - JetstreamEventError::ReceiverClosedError => { - #[cfg(feature="metrics")] - counter!("jetstream_disconnects", "reason" => "channel", "fatal" => "yes").increment(1); - log::error!("Jetstream receiver channel closed. Exiting consumer."); - return; - } - JetstreamEventError::CompressionDictionaryError(_) => { - #[cfg(feature="metrics")] - counter!("jetstream_disconnects", "reason" => "zstd", "fatal" => "no").increment(1); - } - JetstreamEventError::NoMessagesReceived => { - #[cfg(feature="metrics")] - counter!("jetstream_disconnects", "reason" => "ttl", "fatal" => "no").increment(1); - } - JetstreamEventError::PingPongError(_) => { - #[cfg(feature="metrics")] - counter!("jetstream_disconnects", "reason" => "pingpong", "fatal" => "no").increment(1); - } - } - log::warn!("Jetstream closed after encountering error: {e:?}"); - } else { - #[cfg(feature = "metrics")] - counter!("jetstream_disconnects", "reason" => "close", "fatal" => "no") - .increment(1); - log::warn!("Jetstream connection closed cleanly"); - } - if t_connected.elapsed() > Duration::from_secs(success_threshold_s) { - log::warn!("Jetstream: more than {success_threshold_s}s since last reconnect, reconnecting immediately."); - retry_attempt = 0; - } - } - - if retry_attempt >= max_retries { - log::error!("jetstream: hit max retries, bye"); - break; - } - - connect_cursor = if replay_on_reconnect { - last_cursor - } else { - None - }; - - if retry_attempt > 0 { - // Exponential backoff - let delay = - (base_delay_ms * (2_u64.saturating_pow(retry_attempt))).min(max_delay_ms); - log::error!("Connection failed, retry #{retry_attempt} in {delay}ms..."); - tokio::time::sleep(Duration::from_millis(delay)).await; - log::info!("Attempting to reconnect..."); - } - } - log::error!("Connection retries exhausted. Jetstream is disconnected."); - }); - - Ok(receive_channel) - } -} - -/// The main task that handles the WebSocket connection and sends [JetstreamEvent]'s to any -/// receivers that are listening for them. -async fn websocket_task( - dictionary: DecoderDictionary<'_>, - ws: WebSocketStream>, - send_channel: JetstreamSender, - last_cursor: &mut Option, - liveliness_ttl: Duration, -) -> Result<(), JetstreamEventError> { - // TODO: Use the write half to allow the user to change configuration settings on the fly. - let (mut socket_write, mut socket_read) = ws.split(); - - let mut closing_connection = false; - loop { - let next = match timeout(liveliness_ttl, socket_read.next()).await { - Ok(n) => n, - Err(_) => { - log::warn!("jetstream no events for {liveliness_ttl:?}, closing"); - _ = socket_write.close().await; - return Err(JetstreamEventError::NoMessagesReceived); - } - }; - match next { - Some(Ok(message)) => match message { - Message::Text(json) => { - #[cfg(feature = "metrics")] - { - counter!("jetstream_total_events_received", "compressed" => "false") - .increment(1); - counter!("jetstream_total_bytes_received", "compressed" => "false") - .increment(json.len() as u64); - } - let event: JetstreamEvent = match serde_json::from_str(&json) { - Ok(ev) => ev, - Err(e) => { - #[cfg(feature = "metrics")] - counter!("jetstream_total_event_errors", "reason" => "deserialize") - .increment(1); - log::warn!( - "failed to parse json: {e:?} (from {})", - json.get(..24).unwrap_or(&json) - ); - continue; - } - }; - let event_cursor = event.cursor; - - if let Some(last) = last_cursor { - if event_cursor <= *last { - #[cfg(feature = "metrics")] - counter!("jetstream_total_event_errors", "reason" => "old") - .increment(1); - log::warn!("event cursor {event_cursor:?} was not newer than the last one: {last:?}. dropping event."); - continue; - } - } - - if send_channel.send(event).await.is_err() { - log::warn!( - "All receivers for the Jetstream connection have been dropped, closing connection." - ); - socket_write.close().await?; - return Err(JetstreamEventError::ReceiverClosedError); - } else if let Some(last) = last_cursor.as_mut() { - *last = event_cursor; - } - #[cfg(feature = "metrics")] - counter!("jetstream_total_events_sent").increment(1); - } - Message::Binary(zstd_json) => { - #[cfg(feature = "metrics")] - { - counter!("jetstream_total_events_received", "compressed" => "true") - .increment(1); - counter!("jetstream_total_bytes_received", "compressed" => "true") - .increment(zstd_json.len() as u64); - } - let mut cursor = IoCursor::new(zstd_json); - let decoder = - zstd::stream::Decoder::with_prepared_dictionary(&mut cursor, &dictionary) - .map_err(JetstreamEventError::CompressionDictionaryError)?; - - let event: JetstreamEvent = match serde_json::from_reader(decoder) { - Ok(ev) => ev, - Err(e) => { - #[cfg(feature = "metrics")] - counter!("jetstream_total_event_errors", "reason" => "deserialize") - .increment(1); - log::warn!("failed to parse json: {e:?}"); - continue; - } - }; - let event_cursor = event.cursor; - - if let Some(last) = last_cursor { - if event_cursor <= *last { - #[cfg(feature = "metrics")] - counter!("jetstream_total_event_errors", "reason" => "old") - .increment(1); - log::warn!("event cursor {event_cursor:?} was not newer than the last one: {last:?}. dropping event."); - continue; - } - } - - if send_channel.send(event).await.is_err() { - log::warn!( - "All receivers for the Jetstream connection have been dropped, closing connection." - ); - socket_write.close().await?; - return Err(JetstreamEventError::ReceiverClosedError); - } else if let Some(last) = last_cursor.as_mut() { - *last = event_cursor; - } - #[cfg(feature = "metrics")] - counter!("jetstream_total_events_sent").increment(1); - } - Message::Ping(vec) => { - log::trace!("Ping recieved, responding"); - socket_write - .send(Message::Pong(vec)) - .await - .map_err(JetstreamEventError::PingPongError)?; - } - Message::Close(close_frame) => { - log::trace!("Close recieved. I guess we just log here?"); - if let Some(close_frame) = close_frame { - let reason = close_frame.reason; - let code = close_frame.code; - log::trace!("Connection closed. Reason: {reason}, Code: {code}"); - } - } - Message::Pong(pong) => { - let pong_payload = - String::from_utf8(pong.to_vec()).unwrap_or("Invalid payload".to_string()); - log::trace!("Pong recieved. Payload: {pong_payload}"); - } - Message::Frame(_) => (), - }, - Some(Err(error)) => { - log::error!("Web socket error: {error}"); - closing_connection = true; - } - None => { - log::error!("No web socket result"); - closing_connection = true; - } - } - if closing_connection { - log::trace!("closing connection"); - _ = socket_write.close().await; - return Ok(()); - } - } -} diff --git a/jetstream/zstd/dictionary b/jetstream/zstd/dictionary deleted file mode 100644 index 106847e7007b4ac409fd58912fed85d69aedb352..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 112640 zcmXqCV(=!{O*BDG-~!hqzW*T5Jk#gUiDfQbJ}pxwiOl@Lf7aZ)EO^t-8w}jijK-`C z3=9m14kQ~GOgYX}%d)_sjh&^vm5G_9nW=j^D_hS54t5TnCWinKu@<(48xjmzN_bu} zSu!v%q?}e{Jji&MJ)=p2g^_{5Q9?mVpn;izfq{{Mfq{jAfq{d8fk7`VF*&tFIX5-0 zBr`uxNk=IqGeyY?L|PT(BwH1iR~eb7mZv2qrwtN|C5c5PN>)Y&MztW7If;4c#Y$GuN~w8Du{ug6sTCzk zR!WZ>`kYGh6nyf_GV{_OH}rFbXC#&=WF!_8q~;YXlxL*oDI_YCXXNLk7U$=tDr6+) zr0FIVrKYB&7AfSWrj`^dlxLP?C?qNr<>w~mm1HI>B$ejbD}<-!2DrDrBC?w{kC={2cCYDqx6sJ}wWELx=q^4!&WtODoR4U}>DU@WSDkNv5CTHhl z7MJMNaXFWkq$)U-=2M3WNJDvMGxGb%E&GEJ&V64O!>lZsN!jdF776El)aO>%Nglxl0aszFhL6fv0<1?eVcl?7Ef#@VH*iRFpq z#$^>56{$)(N+p@Osqv-7N>+yEMrIZk2Igj_7KRqaCgwUyC6xuKN>)nA`MF9uATl?z z1QdTosbxx5O2#=Rxn;&FMOn!vMj0S;po+mtb8=FX!70Q_DY2jc;>EPo)D%5X;40}T z6=kPZBAJnxSqe6zC^b31C4P(%r`G9F-tWzO0Ot3H?J%% zO--uGPs&U)Hcv@S%c)8$D>pI8F;UV{DlN)XvQkPcvC`L1$xH#K_o__e>=NV3UQqI^X~MMZmih161ol2nDn+PW=Z)4rDo-+#(7Ce#b$}cMJbv2<>tnP1r?PgW=1B(sihf}nW<(OmDs$W zl#*R(Txp(KXlhcKQdF6kXqH@7VpNia&HJVX=7tt#mZn5`zbrkcDAC9aWDdCr-KZ=* zy|fZ+2IUFe)XLOI&%g+8LeD5kFDx@EGd0f3GRa6XF*hnM$jr;hHLomAE6Xr9D$Fn` zEXgV_EzU|WDJsd$HA^%p0p)Zo3B97I$TZ6+KReGnwcMzxEV;b6qO{mJJqcGrFHOnF z%goL(D#J|Z$>wH>rr9QCM!ES(xke^t<&~wyImJ1p#VLjP$(81&8O9k&MwMpS1?f3? zrDf?AMLAhm6S`SNu}NxHWrmS)VPC#Hm zTr<-gqe_rDWX5!9rD;x8QVQ4%%46EZ%FtZTz`zn;k(OFemYMnr~K+lu~JK zQdVG`pIVZbTTx(MY*w0AnVFW8S7~HYQIVceU}BV!3@ZDv#dKw^NkxuHNlH#yWm#r= zWs*sONs@U{9wer%3{0$y%=L_ojqtf9xiGgpJ3l!mtF)>hH?=%5r?@mVxg@`=FsURf zs~|Z!!>BMVC#BRhuQI7LIV;nstSY?_djXqTUTI!sRG3m>T9#T_k!@UMR8*c;T2%&d z4XA1b*DCM=wzML(ptL9zTe?d(E;lbUs;VeW$E zWyTdrMtN0b*%f(}mBq&9=4Pgqr5Wa_n79(Aw8A94+z4DJ5h`KQixLYm^wRQka`MYj+fAjFrR5db`Cw!4 zL^hfMc$!WoRwl-J2F9p03aA|kZt7$e<(8D^ftpyRR>lT;1_pRbfDDs@q{Ni+#LA@X z^xUfClBB}a4AZhoqmtB|f+F+c+(MI-qQugyq_V=4f~>;a(p=-*Ozb5q zrU(NAJqrsHd}V#Qd4X|KQC@y-X-0*KsY#AWWnxycNt#i*SwUq%L2+?ea#ngtMqz4x zd1jhvL2-JPX$Fq6-l!-mCAloas4A}@C!@SPKQk*eC!?YQXJb0GINQ9cFxSW!Go~~0 zvkD3e(+di6vyDsBlk(Dwvr^O3%9C=-@{CF6w|D;*06z?4*L6{N!v<2M=Ed%*a5`z!0_8fR`dkrlv;brR61A zspSRPr5TxNDaA#_1^E>i{eh(Hg2c?Eg4Ck&;;Ouo{H$!FjN&TOiRv?8snDm@cy z2BA2BmbQ4}z!=mgGqW@V#R0CthorP*6VsHEybQC-^c=JDs)BMee)8edr^GwsS>=GlRG~=wyBJ=$6^1?LZ)SRr0(!{(%vr>}+6XR5@Jxr6_tn!M2 zyo~(R#O%C^!o;NVj5MP>Gpwb*si}#fk);7~Ln&2d#!1;mplBtSZqb@=$m3>4Riz~v z5M%HpT{Hvmq+U}iQ)4|tW7OInUUHXK7-yTMHg7*}RxBo~_$mggqsCYfQB8Y%fn zS!Mb8CMITP1^J2PsYWL0899Yn$vIhh8TpkJ#+gNlNg2t;nUxt8xmm@zrbVgXHatcv z%fvLLFfp+(BPTsSGdr)`+$1ljJU1f~dq2z45ImGcM9E!dT$quak!(~9Dt*XoWr55{ z%_#<(fhXvRY-O2Q8CvLBni$|KHBw45igMC3l1)-f3QbbXD~*$KOfpPOaw<)X)6DZy zbMsP+Q;jN3ij&ffOU$whs_W53;)?8Sqf&4_Gc7PpG|$h?F-^)#Og1V^%1+5PGs!AT&&VstE6z{OOEXT&Gb_Yp zb$)tLPI^&peo|hkc}0?0VNPyRs!3)g&MaO~TAoo_W>f&S8g&Rax3HipH`y$?EXT~W z&@|t;B+1CABGo7@Ik7k=CC$9b*rYPiD61;RC^OYG%dD^<(HLt}-aIwMthgXAwJ5)` zq#~ubyx26|v@kOTJj{nyHJO{48yFgy5H*NYW}I4PUQS^H&p5TrsH&7o4Lma|12a8y zBfOoq^i1QT(!7$AvdW6WjNHuPjQq+x)579(qly%heB+XmDl?PZ^wO%Nywb|7yt0zw z%=BbXS%@W(6_uu#ROV#l=BJjJC8lR47i8uZW)!EV;*9AW)BGf}B%@R;F<*Qf!i0T9907QdVLLnm{NmLY?lbEY2=0$;?VgF)J~y%uLV9Of@#oPfh_3VWH(7 zGh;IoLo-8DVk*eeipV^dtYC$BuY(!8h)+mJ_kNm7+*L20RJeo|?Ea#lfE zR&hpt0jS}em2X~@X_Q$|oNa7ml2uY#lA2SJnwwQn1j>ULxhFr>s3Il3vY;R1roPYy zJyQ~k%TtO|le5k9^E1=)a!gH2OA@n;3r$KgN|Ot6i_*$Ws`8A}it|&AO$w`WlGD?& zunu~rniiC185~}WGRrnCNK4HsEjGF#2OGeIZ2fn1;)u4rsZZu*%|qz1^H!Z>E+<5ZuBH? zY+z()Vs1=Kg;83OoKawwVq^-+I%Kv#L1tv6Ld>8%$s1dl8t56A;H@yy%uPzn3yl&@ za>_GO3oCB_+L$;rmK1x2MeW4gSeA}I&dl&VFaGEd2`%t|WEGfyqeN;R)4DoZyt zt4K9X%t*{kOv}ix$TKY|Pb$mKugoyZH!3$xEz8d=#M%hVt1>puFe)fW%&*KgEz3*J zHcKo^F-}hc#SmIdo0}P18W|Xy5;HzgW}H+}oNsDW05XSAOrv$vkZLlJF*zk^U}Nw^ zHktu=`gvwn2F7}3W~ieS@P1x-Nq%l>eqM@Ek*RTFYGz_`Mqy@om1&j|IP}rhKP0A< z=2s+^m6_z_B&U|-o0^oQml~&K72%A`ssiJbBqO6N%xs&HR#;h{TAWi-R#Z@0k(*(f zZJt_TmYCnpTpOZC;p|ZB~(;T4o zxu`O?AhoIpYdg=v#K6$d+?=?`EHln2Dl$qTCNgofNy?1Nj5A6hX5fiQWZih9(a2cO z&qLxYRhcxV*?T$tXR)GOHrD$TX`UsjQ?tBRL~I zu{^B`)W61(Z3`>&vx`#`D^rcqb4zp5GfmRXtBNX8DsaSfdbvqnib+-pwA+Z@a zHZe6eDabM|DL2k4GtSPeN-In+$OU--V`$d^6w?;QM72rEvof;sbB#dbRAjeF%Cj<( zOG?3JP#)99Rz^m8mX;Ry1_&}z%F@bms!|J+i%ZfHOH#^ms|rj~E0c|r%8N|Pa?GpJ zO)8R8%yW}7&60Aea*fh6^C2yIj5STg1tulQ<%LyA##LFBrs+A^rG=U48EGltmb=sY)#= z$;r>nt1!tar~-vImV{SSnx0mbmz$E2oLp6uQJzv#l%Hafl819#FuyP*zpx52ri9)^ z$}3OFEGe%r&8Nlhs!GdDHLG%+D{_j=&C&}p!Hrn7`ohA%#MIEp$dsrR04AAvnMR35-~>#l zzCbP{kShz5%)BJ?Ot2Z0Cpm{(AiW|W#y zRFPkrXUSZfQkgW_C(uiBWQ9 zO0sEsVnvBrftjgsKK4Na5=MYcEltb}jg3u+ng1&-Ps}VU0IgC%^pSART%c9#NRbUP zrl6$M2(%ypPh_JRfTv0^wlXx*Gc&_60$g5Zl0{%Ss7Zc$URi|^sAM6x=qxkIPtP>U z0GolwQ^@HbkB`hiZCgVNyko10g~e$m#wBJ+#)-xzd8wJHDOp7+=|<^Q#sx;EiKf}c z>7}_D#>Ewh#yN$VY3aE|=82#ziX{bCRvMY46qn~$nWSfBnHQ(0rIqER7iK}%PGMQC zRGOYrVGe2`Vzh0OGLy24OY<|$Gc)t^5=%;ovI=u6v-6D$6LX3RsMUFlHeuZ-~hDPS*#MDZqRVgNwWm!g`fGDX{|Yz3dpDuu%<6>>{6OY#*`a|`lQQi~E(5_J>`@{0;fQxy^w z(o^$FQsGiCl{twz3VEr;#i>B4G7=TiGIKJ(UP;c+OU^0HEY(xs zQgAFT&MYoTwBqu~FG|fzRPf9#D9SHMRLIL$2+B{+R>;g#2+9PF?IsmvrleQ;7iFer z<|XEY7Ud{eDP@$D6ck(O>ldY#fea}uP0iIy&&f~9*GtaN)lV%;%`3?-)-TFW&eqM$ z(=E!(*9T>;rl~n*rdj#PC23iv#aI^tWM||V7n>KA8E2Oy zrIZw=n;8|R6zAuIS^?-qtGT(Mk)eTwB{4&RWyYB$=~ao~c^|TyOU9WcNv4HhGbpb_ z&8!S8^h}Mwi->SlqKTEpRcS@#DT!Id85K#!+1VLM<&}v^rKXh`W%=nv6&dDLMyZu$ z$>lkw=DDUO>6PZV#uQSrD$^3P)68;BD+(%&i;QyiqPf`xi3QoEMc77< z60>v6GAfcva?-PkOv}nM^Nr0-GR$(aOEL;GQ!6skk_!s+(oCyLvT~~O(hJJ+3kym> z6WSP+XjMg8es*PHMpc=yQGr=nN=jy7UU^~`_Ca5BOG8sj;+Bh5WEGX9BpFp8noFR8 zq-3m1mC)Mj@CLteQfYp9QDS)^c+?xi=|xGE#TgZOrfKPgDXHnnB_?I2d1i%K;1z0U zy;3tXBXdI&L!wrzlvXBXWMvqGhjU9RGc(IUD{gd@it_U@H&&$-XD1h!m*ksdb6?=hC#>ld{8@~RG}cXD7`eVBwt4XRH^5JiVRRY zq%5^aq4@C1qRdo#dwY9@ki!c>1yylka$+8&jt1AUpbbwsCEx-TqzSb93ACTEI5!`o zyh$o5Pc6wSsH#XU&o$3XNzE=a%E(VC&QD9s%PlHSG)gJUFUl}AN;63*PfE*4HBZS+ z!I}z7vr7wd3X{r8jnk?!D=N)W%X10}l1#BirIER*p^>qv38E*7bGKe;g?VY3Sq^by zn7C4*NiN73vy>9BF?fm`Gy`zex`tK;##ScgdWIILBbo34>8ixk%<_sVQ?r~xGt=Cv ziq!0s(hRfWWX#Y{PR&j&HA_iPEy*j&uBu2WFDfmmFf-3AC`(N=POD0-Fv`g*F{`LD zD>o@AGO9FBHNiT(lUP|^Qc_T!S6otBT9%S&npu&WpIlIkb;+oqF{smTZbVGzmlx+I zmZln|Bgz2Kh!JUJfJts?d2z0Br3u&=DuuqWv7V_BxXlAE^WmYNUSOVIY*ta8SZZ#P zl%J|poLQBsWMye)X>M)-%4wM;IiN9t;KK`(5>xUON>YmsF9dA_R7lM$DN0qyD=hz1SzL3p~w`8m2^7wAGxrM7MJGe8tNIE8k-uMLl%-_Z`1fI_~jSnrxur_>VUdEsl^4Upe?%aEK{tdI+8{)3vR#X1TFMTa+6WP;lL`K1bp`3gm; zDfvYTDfz_;#s-IWIb)>fl$6ZWr0nFZ%oOv?yyX0nx)Qht7AD)vFFtRmyW^2CzDBIC-+^s<7?%*+gvoT40Ttrs%`V4D$vmN(*u`Gcq$u)AN(cb5b&^@^ULO%}q@*L8DoaB#N^4H#s>a&nUM% zrL3qdy{f`Iqqv~TxUAHy0$1zBtT4y4GOx_21as~yrJyRiDy^)nyr9H5rM$?vAhSFz zB_%)AEZMB8ILFv1BR#LQEUzFdx!fqZBD1O_IXMsODh<=rg5pARlaxFoqqH)!oWwli zN|TbrbZmn?<|Zb_h9(x4#LS4485NiqnU{c?0F|H)Q9)u+Y940opOKrDmQ_|#m6wu} zSy7czURaotX_}N;S(cKWn4FrPoS#!^Y*tX7o^GC(S!`Thl4+U_8dSg%v}R_>=4F|Q zX_aY7na1Xod4(Cdg(Zb0<~V{j+bG>At+=4V2wW#%Z<*w!W|o)c8Yd-{XQw0;m{t{* z7bX^#msFKimYJC6q*Rs`mYWslRuq}#n5UZ?C1vFr;~1qjDoV>t&&W(pD=#U|GBr=k zO3MSSmBO`%F{?P=tje^i2)xD>(K0Cn_3%@Za}tZ7!zn3=3I#>^1(}Ho0d5M#`AJ2o zR$Sn5q{M;({eq(Ww9K4T{gTS!3_U$PjFKX$z{ohaup~Xryf7stts*b0sv<4DusAoT zs46kNB*Q$@G`G|^JKZcRKc}G37&JkYjkV@WNl!}8s>;qSDNC=)FG@}{O{*|Ut1t(J z1bRQm)Y8)2(9q0?xRN3*)hNdtI)nr{Qa}lGpa3Zg^~=*zN(&0WW>8*I7=xGGLi(`q zIuC7?Pr7kRc3ELYqG@_oN>WC8hFPVNk$Gi#N=ja8L271}aambzu2EJ&K}uO=Syf?K zQfWmBs7S?C`D>oNm;&WQDu2)zL7~z zSx!k(W`QD$0IO1g=$QBiVcQI<)0VyUTVa!FcB9*#1!qN>8&xGX8JxWc3& zu{0??(=-IMQ%z-VtG}nQD#X=W_DIuSwW?7No7S+ zZenJ6W>reQsZphAdR|#(QF@taMm}hU4a-RxMdbx0Ny#NeNmUspMJ1*w6=_vwCOJuk zpjI1tH_Fh+%+So3sPzM-{SxF_C zm3c`fDV3?_DHX|;B}K*Nd1hq=MI|M9MLEeP>8Yi;#(6nOm64}lVyl@jiDKDcEAnk7Kp8U`}MG$R>o2BDaS>c-Q*GP5!^*0Zz(FYwBN z9j$}1HSEdmu4mJizTY$X+03TLjjLa5-Tt_-_i^L0$Bok;H|~4fxcG77{Kt*mkDH+9 zn3N_aW|ifES_Bx=8ObSS$t7j!Da9sLd1*POC1uG)1!g6QMR}#=X~q>fC8ZUaCPnE< z>1oDQrFjKKIeEp{V<*4J*gVZR+oU9;GS55(w7;psq&zJX+enPLp^2%XiKRJl6IaD0 z$)=D^cVy?os^a3xv<&d(KSHrXxYcTEWoW8rXl#yebw^5Cc4lfpRgPIkVNzjsa%ECl zYFb5#X@zM;VR1>NNqJdmR#sVYd46RX*19UdvjB~F`bm1o@H8IUYTf= zQe0+gYFd$1W?q$ATwGpWVU%iCmYQXpTA5pxo}QRhnV(Zqnw^tdf}?ItO*YOh&deyz zDmO|?&nPmkDk~_-NG`(JKPjz9H#bkoFsi_e>7>NM!ko%t(=-zk6FnmXa6y8z*_4`N zmSmi4o^EPdnpBWknO2^iQ<$G;YF1X9ZE94STvl38Rh(00mXlamo|9yhmy=t7quErR zTVS47nqHQkkyM(Qo}ZPOo>fwinT2zBxU@8>C@aGR+cNmn{KT~Ml>EwyETe+Na`W=! zLSwVk!kmh-#DWy_g3?SQv*gOOGE=jnvXo-uWYdg_#3HPV3rq4+a!o7qO;e1_E6t6~ zOAF1*@{E#;KzR*4shU|>T7cS3#4Ii>tujtd%%HFA6|OC1r^w6-L>ZNi{t$Gpne| zsMIJoEu*A7)7&I2Eu+%BJj*yKqr%8Iv%t7Azbq>;F}J9!%&e%e(kM5t9P5;iscE5! zSxHf*SxQDmR#{bYdWm_qafUH?EiYQ*z|6qV(9qb@h^Qgh^7MiN(Y5mu*{6_uwLm1JiYW>@6rWt3#4<|O5rSEhj)A?S(J+}PC6(9G0~sCJD} zdbTO@QEMik)j}4Qc(+O7np;##bQY#CMIQtS(%w>C7D&mMxbUvc}i(nNixo^LTRy)NoINu zc&-F}9iVxVX+e34QDtU9MV4_^PF`MWRc?Atg_%iqj&WjTQ7UNDLwcflPGM0=S$>Ln zzEKI*wKX}WX=%oVd8TP)X=R0_C7DH~rG?39*`NqT4`?$JOLIeWGvWr;DoU%W3d_KY zN1E(HaCdsCzsYTgY zMU_cLX&HIuSd)2%QI=^^N?B5>X>LiHS%y)5ijhf%aSFDw*xbOv*wD<#z?_KYdudf} zMOHRsxjn(q2RGlLO+5HKok=dpj8y2#e>_2ttQ$}B-4wK5-_Qi_ux3hSc6o_$Qd(k_ zNn$}-MMZj6T1l2^zIk50agtGDv2k90MS5ygPI^j3Nm7wfsYzvyF{nq2C7%_faZH%ltaEHFtkGA*qzFUZJA%B#$* z!qqk{EhsN3swl;F08El`MOHykN=j;Big{XkT1j42MrL|WWmZaxsZnJ~PI9?%PI7Xx zX=ZUrR;F21YJsU)2G$YP?9|+(!mMoboD|cXJo8kuBGVM(l!{!eeMB>JOG6U_6QV{` zjnj%tjVr->I*3bT&|V%=A}a%J=1R;2n?Wd%L3QIvWZ?b51}5Oq0C*n}WmQLZVo`pM zLON*jG~@8flGLJndyKrDUTU0VT54=+oRU>#R#lo`S)QJlQIThsW0ad!l~`JmZ=PRR zWn@~JR+?p8lv`SsYnqC+OJG!4WL%h?kzQ(&o1I%+SX@<=nw^zb1Zvx$m#pT-CI*J) zrpClVi%4Acc{1oFVlcI`}43q2%lj4f>6CNnEAPERi=GJ+ggnpjzoRFPqlRGOWilU7ht zl%JHGnwe8sm0?y=Vw8(@ca?E!L5@*crb&KbdI{)&5R=@*s@(iU@cuXSmXxWnrJ<35 zIWcirT2WbLY*vcglEPg}BA2vCFBw_8yXXcw0mFAk3q*j#YW||Zh z7?)OMC6^^9=b0xJCl?fz7UUHcnUxt;nI`2X=9;D%m*s&rpkk~*Nlr2{(S(cGsVxE~&R*;r$ zng|*x%d5!AG)^wgC^I%LPRh(Pt1!+oHOr_n%Q8zWG{u_FjB*PzD$PqPQ$by?^hDFV zJmbnl({gO5(peZ<7#W&am=V)sDKja{EioS7=(2SW;k= zV_aBPo>5qpo0M*xl$e@kY;Ka8fweuJU6Gobot$o*R+4IFo>gIgOHD$hzQ%gQj#$SN#0DmP0uH8(O(Nh{AU&Pgi*1r(N& zrZO!huPQCEG%F=9vnbE3*rY1o*wnZ@6<3X6USSG4YS<_pOYgkgEIBW;ASbOP$K2dJ zKhrcVu_U=7Kcz6KATJ}&xU4L*A}uL3E7LR|l%|Ij3V<~)3S7lS< zPQEH}1QATl6RaK=?zDZh;asZm*BQhr`#sZnYYXowd*VVj$nni`rL5jE^o zW>lD7o|10_T5UyK5r(bVTbN#$X$CffP_l;V#*?T`txQbyj4Z&NT6lj0a@|8wYC#UZ zbtXv_xrtRp=D8IaX?c~V##yGxNv0XeRhjvPr3K~L=9%d^IVQ!G>1HNL#ipt0<)!H< zxca)e<>h(CMW#79RmEwE#l`vMMX3cvg{3%894)OVOEE7k08KyRNc)LZ6{g8KW%-$D zl^G_f#+gNBxygAMDds5|$=O+%`T3O@$*K9d1xeY)1xCq5Mv2B{DLBSNNp)LERdRA! zWv-DCa@u#yQwYsV&Ph!yQV34YNKGltNma;CQ}E5tOHWNLPE{z+FU?6&NJ>>GE=>mA zjFFU>lapGckds-GT9jB+sgRdnmYSnb0$$_?-v1B2^`W%VJg+>z5`2UO!3WNw;SU~HO{ zYhqrJmtI+xmRXdBqogq}%_%L*$V@iLPb)|%ODd~M$*M>*FDbxL(&Uy_q!g#8lz`3| z$B`0JlalfZ@=6O#P4lyp64OodD$0_pj4JaB%2V@9jq}R$vx-vlOH)kp(@IUsj0*E} z^7C-izg4A0W=2I7MpeeSrI`g4MwzLZ$!Uc+_nPJAR+X4nm6aQrqof2!IKXIuW>ln? z7Z;gimzEWnres$YmK3I%7L;UUnP!(3RaK-FXOH{UACN? zlx1FAlwFXUn39{4V_cS;ZJLo}l83DYYG!O`X=q|X)M>e;6{*IBW@W^+ztEcRDAjVR zac&aC7(A&K%>X<_m@#;c#1hA{<=pI~;*84dl%&$+in8+bsd!PugpkJ zO3KPKNlMKvD@rY_$j>Xt$garDFDNZfF32dzEjCU|E6gv?$Sox=BVZ*2RCuhGxd524+OHevMMh^HZz9$Iz8jLiYcg80i^8 zGA#HyAkbAk)k!6{Ri`&AVVW#lzV~<4GmH634#1loB&RtNa!6^0Po2?sOEAQZus>ixd(S zlJb)i6;i+(40BWSQZm!?6+o-`K^yFnK|2b;3wet{%?0qLzf^_f#L{B${=37A@`^JP z6^g-I^zxI6GSd^m`;kC92$WIWk2ckinx3AVm}XQ7y0{=CB`>Gkv?M7fH@zq|v9hSD zDmT?UC%vjNC#kTg%qTe}KfT81Hq`30#t>8WPLnU%%KxQhLflGLoCiejT;aM6mg{wys$ zDKEP)BegKgJg=~*$~@P&FfT1VyRyRAG$lJDr?k>6H94a+-MF&Q*tFcVs=%le`=PT{ zrWyHpg%ugbMuqu_iK*$CnT3UBS(#WzoDB^u3{4FUh*=F$T2+uzo>K(wlt5d6pm{3n zlS<&Yf_F_2WoTL&`uV>Z8Ac^Z#pXt-RjKJ|skz4Km8N+a=0)a(6@@uvDMh)t#wN+8 z>ACskDdi#2bl%JlSS!$ALT2_)( zlxB-yQxJwvopH_*oa6ElqylZ+Eh za&t`*Q;W)zO-jnjD{~8Ts!G#~lGC!2Q&LhYOskC2j0&OtK|a8fRsc zrxd0ql@ui>n-`jtWM`KdC*#a#MTwau<|*J^f3-SFkow=yM9;_q?>Kk5QE5q9MOCFy ziFtlSWlCjwWl~ypa*=7ac}ZeMT6uQ4QCfa-R(YCfVMSTDDK7Vtcq>FsNm*H9 zS(=eiK4uY}l$vK!U}kKRpJ$q!n4Ml(k(5@HlW%Hfnwy!ERFPerT9sawpIBsUYMy0Y zlw(wAY+{BrWf~P%6dETQ6%>`3Rwm_D6;!3<78#j<&v-y+jhwPMDRaTyE23clIdCD{e-8^M#g4)7GOPQ&uWmy#&X8B1eiDh|Jrm3a*Ddpv5 zX^9yX1*T<1C28gvW(65V<+=H3g_(s(Ch27*xRwZK7ABXOB$|~MCRLOfr{$QImX?FOrj29h3m&#CK?&&nOmaHETE;ljJ&eMtOC;t@pn9z|6wRT%)RTqcrp6V^Wz_oSl7%)(g+&Ey)-E&w>&>1G2PT8H?gcNFA@8V zQh7yHRmo;mIhiT>iIwF=plj~3%}sM~oj{sXoReglpOp;ST?QTlLpoq0J>T5K$h@jB zqarcM%q-P7vC=#@%hWilqOveKF~1@?)ubRNy&$o;+^9G$DJQKmE2|Rgdk3aJ&kPEJk= zsS25SX*s2-dC8fHDWI+QDVgb+C5btZvqn-fK_~l^q!xj9%9drOB!bS#QAkWmEh+&W z8&m-rG)~k39isy}-liC2T5^7FehTym5YP^M$YCfcsR{+DIjMRI!HJnkpp#xQixo-| zlk)R(5)}eV!5#L(QqcJ*`FWsjI(Rpo0_f0?g8br4$mvBX`3i}}#gH=rKt}_mRutr9 zCMQ}c#3;FzDkLUnKqdnqr{#d%nss;~Xu=>VH5ck+Jq6cNh4lR567az#DXEziiA4$} zr4W-db3x+{xe7U;Gg%Z0iV_t-yN?xeOY>6l^%Me%KnL;UD&&F28V;{iC;`!FhZiPi z=73IfNi2fw08~g+NKDQH9ftxwHYv4KA+IzqIZ>e`vn*9l!4q_{Pg;JaLTRo-UTXT` z&B>V%M<*ud=ar-;gOr1;0iE{)@_`QMCRUIyN>V|`uoxLbj(bT-RmcSGq}G8Pc2KNP zkXl@vpIEGrmtT~dm;;_JQOL|q1fBJh3m#a>1X)~`nyZkPsF0Wfx^6l7@G4N8>nVWd zT0jPY?s-iuR!B@K%_&I)MNV$g;gz`xpp&3-!JDN)X6VHzDTF104tz>f@X0So1zD+3 zTmrgvIRzAvARADQ9#RNOOiN77QE)CTPAtjS4=hd11;tk$B={g9QIKDnoUf3QnRa+B zC^Qq3iZY8}=T3n_1QgPc<7LV+ixP_<$LAzMCU-!AlbD=eTvD2wnh8m?kn^9wu?jxa z33`GRIO0&{z{vqD4o(x`(9VUY;e5!166nC2!&?-RN{f@h$ut){vy@l_3W}iA#GK43 zuy9FHVhMPnDj$6KO};`R=%Aj&ybP#d?%|awnaTM&;4`p5v7HExnB3A7(9Uv&#GFh} zv_W^4>gp;KrIuuZcD#dAYH=y(2JDi|Jkar3$r<^{`8kQ8UF`+=MGC1EsksF?kOQ(H zsS^}s;Pa!B^K+9j^HQqPO*8ULN-Gk}3o?s~3QH=g%F`2#vU76F3-gn6t8&bZjkEI0 zO>9_Vhctdf$dq|}m(+*D(e(xS}5?2HPtl+^U3id-`zcUR<= zSLGDtnCB$u0KCMO#wX6NLXY`u zYGj_AnQKyDR$`WFUTj)Xl$~ajmsyfwUS(`*oMe_;g>_9tNk&eQNkwL|c~NOrj%iU! zxp_{RsaYQQ&MdTEw7Hq3k)g2#ahq$5GeGlK;06$)%~4B}t~H=4P2$W;r?LRh8KpMpY?BRTajG#udid#>R=s z>4^n7#d+Wc7nac~6VtR3lfr)N@PAsw$IA3b9>flaga% zVp3_A2wLToS(=n?nwyi7ZJr!$Amlb3s z85NnDWaj1;q!m}D8YP=o<`$V^TQdpTvubE!LEKrT6LfJgqX$G0%mVL3zS5 zu`;pLGcv*36-lo!E>BEJGc7MN%S<&lGA%LBPEJh9Oe#q$NKH&J%}>osG|4kbF3G9N zG)YRzN=wc#!7&VKTAX82n3|E4Wn7q)YFw3*Us03^%C-79Zj>+0H8x4fsl>KdJSjQR z#I(4oA~Q87Gd07!Dl^F>udtvr*}Sqm$GkKr&p5~2q&Uf}B+sn0q_iqEIlC$gYva_| zyf`(@#5^x4tuWENAU(@G%P6h1stRi-*uuyVbe0NnYoJR~jY+!h6yB9Uit94tl9X~H zFFb|o$I}%twK6f%voOTl6-hJ8Eyyj%GcGDJOEXF=HZ#h~OD`)c$}%ggtSHP*FUw6y zHO))TsZ6RgF-y0u)N~*{$C^RuQE=tVFD5y%# zNUh3DF3c;-$SK0Qh_kBFxFk6wIsVNq31 zMR8tgT2WGBig8s*d4)+)zFC!dR+ec}R-#dQd17`^a&k7-VV}&rO0yiJ?6e%y-0aN! zyrQIX^8&M^Vr=JcnHyS|8k$)UxwWJ`zbq*eI+aSyIT-L1iQX!@gD&64v z@uqk~3q2#q?r@wHXNGyHk!eX`V@LB2^@Nq!l$ftr(*nwpeYQI%JaZS-An z#>vU4X~j7e#i?0lW=0vtMMWt^X33SQr9~B~<;EFV`5C5VMunwm#<}@9$%VR@d9HC{Zf;(pxlx*NVtIaLR%xDbS%pbbSz>0gQC?YchPhEia!yWq zT6%J2VpdXV73dsT%mozXIaw81Cg$ZSg=V?=xkd#=#px-f#wFkzxY7GT=4Qr*7G?&f z#Ej${=M|S_WEg>#ZV?yTn9bEPg8)cYe6_lr!rezdlB_>srW)v0`rxuwNl%<+y=VzFh!*Az6X>+8N z<`k8crFLI1#rdg;g_)^FS*BH) z*?GAYMk!_mNvS!xRr%(o>4lj|#>quVDaKXVIr)X<8A+vB8v#`rNks)!#m1@GCTZ!J z<)&HY`9)bpkm4LI#h96ySr{4_84xx5UzwJiY?=X{w;?yhlvbuCn>!U)?+W$3UUcqBSEwFJCQ5PS@O zK6vO4G>)EGlvu1#T$-d{WT4=U4xr5|B}D(KG_R^4w;;dJq$ndlHKRN^EhoRADy=Fp zyD+)1Dl5GJbVf;~d4^e;S!RZLalT1rX)dJsj4V7GB-*rDJn|L&CbSp zNtU^Rk%6J5IguB4rkAG`C4z6jCN6TIH5YQiu1YUYEUg5aK`3&dy75Ggsg;4Dp1HXx zzEUi?EH5W5C9|jkbX}Oac~)kjNlI~9fk|GaX`WG1R&h#pK~Y&=eqKgVqG^tCRW>Ma zvCknEmY8LOhSUqq%&T&=a&WFy$jD4FFDj~puE|EJL^HC}QnSi3l1t0V zj1n`=t4d3fjg5`+Q%W+E%uLdAa#Jcwt1|M-%`-C!O|$be63eP`v93x<%`P#{E6+-; zDlIoHN-N9D%E>RQtgOH`(rad5W?^JtO5DQ1@?5jR+zi;l!Q^stW0UOEO5^;j@?x_} zvs`m*<3MJXhL(n=mP9RvD6KM0&M?acFQy_tDVZiGr5J(Dpgbv=fG!QPG{d_F6?80K zc5ZHFYEp?=d45uXd6s!rzG-4Xm1&lFNk(C@SwU_|mYKOpYJOr~R+@21W{w$-#&T96 z=pMW5l8WSf(~Nx6DigEv#G;f8Toqz&X;x{Od3rInq?DRdT9{;JTwGBcnx0*dla-Q}m{M9;l44qAT##y1oRXWCY@Th7eGDi+IXgMqG^?_rIL$1%urM>( z+_bzZu>k8vUeKlJ1}4O9udB?=E+~WCXij!aS7v5sSLK4oP^b~pMwWUe_!l0e6r~g! znHLn7r>5m(l;)(Q6q=i56{VF`W)+tul^SK5R1}sMR;8EZ=4P9km1b3#7USA_PwG{6 zr8xzKd8WBWDVW`&q>RGsDs%Is(h}nmljQR3oNRN`;`|EZ+@$Q{s`7%2inP+~;i_83L57C)x4?6`9;|8Oo22{ax#lcphn;=%E1>5f)^u!bb!|MmssiRr(~vB z733sa6`K`j7N!^Fl;u^H<>#B26y}s>nHp!7BSuXTw&@$^7#mlar01uX8XJN7_V7Lt zyv386o|#ctmY7^-T9sLzlx>uqSeRW=o@|0qC#U756&IUU6`5zJr6r}LXPV`trW7Qa zl%|!WCuLNcn59)D<(E~M7-eRqm>A_IXD63pA9*k}E>Fub$;dP}%}cM!$YMu#rJ`I|^KxH%bm93BvO3Tm5$p_UCND*3Em70~A zmjhnojVvvbQT zjEq3$5NaxeTWin;0g}7RjMK9-D+uTO^hl{%PR_sQj<%~ zaj6{BC{m(%Djq{)I_61@BzSy z#ir)QMoC5`g+{4aMFrVKrlooLW~oMHxk<&T<>l!W#TCgWMd`%_#buQ_W~NDr`M4(A z3kwT#jB^u9O|wi=i}ETmGt;U{l8Z|+5m^}=+n~0&zHyF8ZfRwrsd+^<_y#nzalxdt zEYkwhoZM3L%Ivh1tn5;=%#z$(6SI`OwDQctyp)onH1nj~yj-J<(v)KJid>^ybF9-{ zNvQ=T*~WRsm1*T=nTdsJs!K(bT}w(8S!3m^GiJ73C)7Wd-1_2sC$+ zzV5uD+@#P5a=#+wN!-NB&`{5Sz-f;8##KhrshW3MJ8!gRVHRe zCMEf0{N5}bkJeP7^{2J&5SDwO3li%Gczly%1lfOQYs5fQwq!~OR`Lj(kgPz%&Ia?@^eeG zjnndy@-ovhP0X=YeicbMrD?f^Nm)fX`FTYt#RVBfrP*b<*arU14Ghc-EzC@b+iH`R zlbsLQ5Jh%OSEc2omZX4N-IT|)sg;3&p0O#o;|j0;&_;hVa*KX5%`=PBlG2k)EAmpaKo=)rWWB1S((GK*^1O=Tf~>^6+ziw5iln5p z0<%)%)XKt)ioC>}!jzri=mI%x40xBw-yZZ9oQPb*GMH3FG|C+NX-1+-;= zC)^FKOilF+%q{WN?TJN26&dAaRVhivB`MieMa3zJx#ksSMQQ0q70KC2d1Xa8g-K~f zhuhgbo2b|vUe-DhWM1Rb%n)U$+aR6^US4;}vkU67TL z4;tt?yiy@wp*SPITp=Y|867$TAl&q@CoT~Jq{6h1z;NaOp}bN+(h%djH=YUirhTo(n3(19CK>N%&amwFRwB!4|Ix6rCE7Og;`-{2KXFN z^mJ=(VPI)!WJc8ZW0`TWX>M65xCo}8)BqWiXr2o;22au@zSJ-SpO;{S+A@Wg8io0} zMoA_WrK#mrnW@F)r8!jv#f8bqMoQ>|=V>WbY39jgDJ6y3IVF|JrYR|91$k*^rIlso zmHAoaNrl-NWl7m4>7`{Ux#q@INo6M3mz`CZCYP00WfT-8l@_I!rDYW+SEUx28iN+R zV1&N0k&&Ufr4cdBr81+G+(hG2BT(mrVCaMMJ*bTc>VsydRw`M6RgO3A56&IOx+ zC+LxNqm~+mX1az3x<-~Eh89-F#-QVFp?8Bqle@1cXmNF3esOW4LViw0BIF+2w9+Eb zvh$Qwg(T35^pt!B-$YOgBUQoAQNhL4%{AB&BQ`P;Qwz%ybBZcU(@HbVt8$H!vy0R6 zD)Y0=)AGxc@-wo`a?*0Dva-wzQ%X$BjEYQiDzWD5%Jkyo+|tVA9FxMr%IvbT)Re5s zoWgW$Iora}%+k=%%#xVH_R363GmMQ=jgl%V(oKys(lgCW%#%y<^UF=mt1@t%sZo|@ zTAWp0Qe>Q(Wl~g-WtN+2RGv{#o`j>tms^}tnwy&pIU}<~ioc zrb$Kxg~fSE$%%=@nZ~$Uqq+Iz*}3MKu%kwy(O;Yix;?cRwAMNmvRWFxv=_P^0JO|q zp&+riII&0}DX}Co1+-?n6qHu-!J7i0#}4GADx@kDXXcfFW(%_NA?xTtyA8nG7!EIn ztlLg40$op-S`50pI3-a5bmwcTLP};)YGz5I4!UaaN_6n5@#K-inVVsqlon)4jR+(p>kzJKvQk+`}x<<7uzbGXq zFVU#5veF2=Oao)ApgglOCojLExU@X6ETbSly(rh%JkvNA>kSU3CPs#47RJO(tCbm5 z7UkxpgC{?UD;c0I0_3p*qspR;A~UcVgh~deZq$+iCa5>)p*62lOEXP! z(~67=^Rmlw%~DIt(lX1-vvLb6Q?imWO!6u#3kx%|5|hm`bJHtQb1M>yQgPkjkeFDU zoS9-$WDdGxryx5sBiksmq9iQ~SGFpxN~x$a%>`ZCiBShsnWh<+R+bo>=A@)%7Z+q_ z87JnO6jW7JRpnI_mgJjfC7YM!7MJB`CKl(Wq?sFm7Wfn^Sw&YXCFYbUSt*6)DuAx! z%}mQo2HpNwmRVF%nwSIKv;fK_`3f;g&iT2ix%rU&2Zja;xtTc`hqtB{gA#LTap~cO zMVXLWhn;dtQ;V}Jbrh7%^o-05^h}J+bQGKt^AhtC6?_#QHLP+-0&{Zpz!zhI_aEgv zYFMMAkeQcW1lqMy3c8xl$O2??VtT$Fhz8v*9iybAqm&4`v9Ba3u_QAebcb0+YG!&y ziISC}nSp_hQh8=dNrsY@k%56ht&S3Cw@WHW5$qI`q?~-v*mqHC8dx+ZGcOx5QNZo?Mhxkeh6hol|UHkeHjElAlqO zUI;4xb2D>OLm=&8u!Z_r1*xDzQ6M)g8XK5cSb$DBsnt=!-qr%`wSngPBJg};N@i7R zkpg(j6FB$hCKl^KfP%jQG;kFX!FQs9$|>m4X9~EYL`NYNv|}i(G*clTDM}O+QGmjS z`IA~d%Mr)U-U&Jg*?J zB)2@PEX6dns@yc!I1TFnReEAos##ICNltQpN>X}JRfTzCdYWk#))hTQ=0=95#zc+N zlo^>-q?lEJr`yRdUd$@Yk_*6QP@{M;GSjm(N9`t}6)!18$>l~xB`Nupx#<;2rllsy z#Gz!cOpGRD7qI4wOdH>n~!B`v3@A|)rc zEVVc#J3Y71+_*R=x3nS&bVsgPZcar?xp`h$2I!Zasyz&EW;KJP0)W{Gt(GDND$OMhjXQrg)m1LHH z&%H|luj_`*cHl8PBQY-peEn)-UP*plW`0&_US@t#b{cX4sidP+oLZ=4WoT(^X=H9{ zU<}$OoS6%{v=t*3O-=O7L3e&&S>$14W?EX3o{?%?R$87|Rb`rPWLA`AWRBH`7NAYB zMnvr{DFYp7m7hk@s3^#cv|{2Gc@VD5%|NT6EDZ7PBT6bwF*mQOGB3?3D66Q*D9X=I zu1ZQSO0G0YGc!%hPb$vJC`d2OEGy1WHcL-3H_k4sz>(WZQ!Dezs!Gx-(^AXQ%S;N> z@^cEaG7E8?aG6`0np9aX}KjCnIDa@|%qYh=ue_?TEU&~QF|V>Lv!b9lzpM&08idgSHa9jjG&C}Y4yGtsDJ6rK$tCCK z=4O_F+7?BrWw6exaaK`PE@jJ~%8av$N(&*mi%>lT?LOma=9+=#3rrv}otm4JihZIN zETdPJnUb21r`2TuI<^$r>M}JqtJP70=&-6*BC64qk(iaAnv|25ZEBL2kyMs#lv($w;@!t^B5{JiXn zqN=Ql^7Qn~q^v^I-0YO3T;udyT$ewYj6;$PyV_iy>o|KeaZkk$>3);zCP?%I!oLrD!mWBQ3DKj%eQ&TgdCWgw(vXat3 zg*UwJ$2|pzHXw-9c`Yq3%QDF_qv{RtW_rdJs7vSIBZ|33`Q?S?g(ZcVl}R}nSt%8{ zX{G6&aE^_OEfZ1 z&aSM=&rT^fN-{Mvt1K?T8v14CS&1p-l{scSQl%v?|%g_7km z$})2^jPsKV(sPVUQVNsuDvC|3jEhYQ3$u)K&6A7MlgrK0ib^swjM6IeKo{4W;W~@a zC?&BdE#KIzBBj_QzbdaRv%tKvs_)=JSdyF8^R4RohkN=`vua$bH~ zVqsox38=_KF9*zwEsYH=3@t%@8$zL9T9IRJlvQ8^T8>FD^ucoj&`tw#5nPdDR+3)< zHUm%4BkRUf0vKDFTIiV>qm}?@9sIPMlDyoMoYc}Jv)syZ^VICjyt0Cnoc#QZ?2M$M zlI)7gvb2H%^U4Bqqr$3;O4E|0Qc%MNOA|WFI4iHHyeOkQDbXk|y`ng$qOda8Bn9{Q zVOd^QR$@Nr(DPayNN?Z3LeJa?Zx1QGvOFy=->AYk zF*&2CJgcHKD?L9mttj8jJPUMY5Dw>Nq-IoP8W$EDXC)@5mn5eYXC$SiCYj>C#xyrM zH^(FubUy+5aA#6^Wo5B(MQOfyNlHO_Qhs?wT1H`6l~HNH$PDMeHiCJ=a zdP;U_m2q}{64r#7QfOYDnUR`QWtx&yW}KT{mS~)uQw3faiIFgkj0_FU4Gf8D9#j|= z6_)2gZ(1Q{doZ?yX;hSLTn09S8VS?HP|wm7@6Mo9qwK1zk| zjC7;Y(ws`OqSV~7#GJg$eB-SA{ESTFoTB3V%p4psZCsIAT#{T|mR*!mWLi<0oo||& zm6e%{^Jc1ivy$v2lQe86J*JgrSC*P2mFHyUW>h7mCncp7=b5CI6(lF+=b0O4mSm-x znO782s?0U5gp}N57eQ5(IRz<@N`&&5HU&*88=2r=5|Nx(1R-hviEKw(*(Tc0o z%;enoL!pe<{1TGWAG#lGz0KdPo`F&^Jy(nr?=qE z0+U?hl&pNy6!Q|J^4#p?yu?%!Qxnt542)($Vs2q+idlI|N<~#+Zf;p#hH+6rdSz~z zSxI)bNkN%ua$;^#Mpa39WnxKkQK4}`X_6V%2C7MBd3kP1PI6+INv3&5X|`!fN@9VL zNfK!3FxptFnTe4FXh#!Kd9oruBPSWUzZ2SqP|{JtzIqy5u|r2pkb=InB0nQNJqx@? z6i?72>&6rACRPSUdZv*3LLO~!%N;(*Q@ z%PdP(aP#zY^zn7|3vpFQOioP7F9jWmR+Okv20GJTA+0newFt6yAvd)s1$0KfLQ;Nl z37C3JEZnW<@M73Qg#rN)^#iB%>g=Ek6* zFDyAZEhQL~J=ZEhRfE)2Of_E5oGJ zG%ckpJ+G=LztkkjtimL}#Hc*QtUR&GG{-2_D5tD2KO@VyDy;--O|u=^RmM2}@00UQudhoC-FBP+EfO z#*>zet&A=7Eby1v>A7WDl{uNIg%y=XW~nBb$!5uU<>rM|RawQx+38iKNf~*@re+yM z#YRS@X=W+O85Nm0YV!1)Ec4W=ax)Y2{8Z!Yq*RmiN)w~hRGejYX=z@SSuSYlK4|O> zrGJ^4RZ*3alvG)knpA3*Q3NX7vW$z<($Xr@i}Q_(%}R>%D$G(6P4jZiD~hwtKurv+ zF>RKSm|K`vY?hapTbP<|l%JTLUr=g>^(=RD&{^(AMn*)H8dZ6fpeCd#D8-N)(^Ywu zIaQF7hw_*<1>MqYXpXmb24%ctW6-EfdP;6#c2Q<}ZiQ)1K~{2^X>MM+S#Dx>l396v zYG#>vj#-XzX>xHvI`lXfJuHP@WubXhZYq`Tg)*@+HrKN-1~){|3Oz!%0VO3R8WkCv zo2DC8RuttGl;s-drx&HBmlmbugN~7}NUth1OD#<+G_A@=O-#(qGcv}tB`+l{KQ+-j zJ3G<1pe(g2tIFIsEv?inI|Emnqtv`4Gug}tbOa!d;|Y^e3M+EV)6=T5iVKShN(+p$ z@=MB$%JR}vDl_wqtFjY~l9Tf?64SDi%0Wld<(ieG;mBJhCE3YFSrxfO8KpVd$>pZG znOQ}N1?9Ni20EI%4BCBD0$l#S3TadUH1149#2qP9SR zdSiL1kj0jiG@*=hvoo?xsJCX*+{)Nc&%_d(sxeo37iXoF=b4oyXBFj{Bo(LTR3zsY znIz?#VuXHbYO+zTk#SyHQd*vIW?p%@v3a>^Vqs2URc>-oT4`~Lxmj^$dRlgNp|PoP zK}u?Jg&EeuFu5o@r!29+sJJTC$gHZkw7l5JI6Xfd`yC^u7KSFKum&ua$$v8|BO^Uy zBe35{&9OxpnaSy8sYxa|sm1B0rg>$VN##lD=DCUKrKx4bm6^sFNkv7;sm3OGl}4p` zrIlr6IC=?HIi+c-W+@r@$rY&?DMh6vd8TG&#o0Iq4@)vi3NnpyjM6}1T$G;=o_s^8 zc#@33-7C}N@+7k~Q}eQ-;@raY@}ktV#EdFq(~Oclfe`8RTLV zm!?#hWaZ>k6eNR2D65iFOf$=iO)83t^Q!U^D~ruatMZF0%W^9cbF#{d%t4oLR+V8* z73pcl85w3Kx#^WA<=LjmX6cz_rB%jhpfyS8!y4uWre=m_mgYo_w^d~)XBMP_w@nbt zli;Q?v^j&^u&&BZPA!I*L6s`fM9;_)?|55ENoitwiBWo1L9UTea)ohfVp>{3QA&kT zZfR+Gfk~Q4QkhwLZfTj3d2W7TX?A95X(EmSyRbB~EH|~hG%dp@CpobosjM(PxhyFq z9cN`yTv%mVRA`iqxm+bVqsTlfBiYQjFefJ|)uf`ZsKU71B+aY}G*)h2oRwFVo>-n{ zQf6*kQecu&SyE7*m5A+7h|1*BjEvIUg5rXLl+4W3?Bp^d^QuG>tfjAsp^>4XrI`_A z$_jo91GYHI&o@d-A*vk!)s3gY3U0=jSmGUe$}mk!GOa95G%Cm{O3JB9&oeSJ&Pp~l zs;tVWs>~}bHA^$jNjEc2FRL&sPdCX)PtGiWHe=A21E-{zSDBj>R%H~V85flq<(8DB zniQrO;ogQ=o|I&6Vv6kMPiCss| zVQNWvVPQd8c4<{{VwGuP9_UUUEK94hvW!a;vkQxIjf!(hj8fAIimQyOGO{bMwS|l= zjSbB$iCaKol%1JgOymL*w9z1>qPNT_t1LMwAG|`VD76H<3pXXrJgcf0YoMeRnidyV z6_llzl~yH}6jv2zn&f0BWf~c!CK;C+mzo%rXB8Gy80Y4K4s0sPOi9d50Z8e4O0MsLaGDH>VI{ z2Ia}$40P+3sS)1eg%gv~3d)o6Q_Ks~Qc}}X(~>fhQWH&*5{pvHsxm67s`5%rD>L&7 zQcFw=(<}1Iv-1mbakNE9yiF;$)Fe02ytn|=vq7ItNz2VF$TTT0E-3?zuVk9#Cnn{k zGR`q8H8;sJttc%q zP0z^Bz;>j71!yyxxv>Q?1znj*WnN|p?k%jH>%3X!Pe|ZF*PbqDz7L@%P)qn|4AxO zFECCn%*iOsFiI;dD=DkWHcCx5$w@9OsVX)}DmE@lE66EHEKg0%FeyvPD>tc31La*T z!-6EA2wPlWR+M3eZT(Mra*Ao9X=YANR%vdzxp75aaehH%Wl?HjNk(a2nsHL5d0MJz zae01aT1q)+omzfX71ktKQdD77XjYn0Xq=l@S&?N_S!SM_m0W~vv8$Phskx!0i5W3_ z+Da>mk_wE|i7R|?m&hPvvdzuF#^6bnXa?YEYnfOXnCclB;~3yiEGa1}E&;Wj3QLp9 zl2Xg6(vp)iGNA{;r6|FZL`rF5R#}=^MGok&-O@61v*hf8{KTBxG}D}tlFCA};*`9? ztiqzKtaM}JwB$;oLgOT?t1b!(l1eIz3zD)E%gb`Ba`H_Ii&G1$azMalz?}Tp!5PW3eq#p^URVeOR~#S@-vM~vy#j+^An3IlhVs` z^ApR9jH=3uOe@SQN{o{Wl1hxylCrT@ys5c~#U_=h*@b4N1)xnVIY!B8DLIwUH817) z<{4!<6`8526~-xL=EW&hnPxaQ%^4+So0e9X8&zTUWKxSu^OLet%L@w3%*_j{vNBBb zinFqeazXCQ%*{17GEFHf$S+PbOE1bSOis+ot0>0mT4S^1QqW-%Nd>6|S?NXDITd+j z>BZ)t+7F|dU|?x%XlOv(p7pA{(u#si60YLIdWdgTUTINDCD;@^85P{hhU>?ZQowy8 zLo>WB<&3nPto)+%oTSRUqO{z+qVh~Lld^Kts@(KKqmra@!SQsWe(Zh?9Ae{;u37*Ru+bamWJlwi=$z+KklX0CIy8iCMlF&FkWU-kXBd$ zS-*iNx=AS+Obzu+OmLJ8$(hNO#)&CL>E;zx#YJgZY5Cb@xrG@?7~?~!Wu=8hMp-3Q zW#*+OMODVh$r;8;CZ_q7iG@XZMagM}#Yx49IVq``x#_va+46l?=z7KSFqMy5t)M9uq~6dRcp7J+LNXj%oWb;UmOjFx(kDyHOKrk!eM49=4NJ%uUP;4UJ4JiE_G8VP$q< zA-E|5bvkHy9(JdL%Q@)C50bmfj0!7L%M!t6;K??~y788g#uj>}MyPEWw8be&SxM&R z#^!})={cp9=85S>xn{X$=E()6>4l(6ZBom!lTu77&5F#D)67c?Qp~b(cBe8;b2HO2 zvx?12Q}PO{Qj0T$&2g<=&oEBP%F0Lu9SMw4MyBT_nH85MCYP3GmQ-YBC+Aiq zrIn_D&Ra{aGO0+)HZRI4Gc7C2NK7)zEHNr9G0DZgEi%)ru)++K)02y`(u_d+3QDt* z3b2-uW}p(%#KeTSiB40KlyVcqA$(=Y=4mGRiD@b66~&nu*(S+Fl|~f>RXDH7Dy_`Q zOij+hc6CW=RbFv!YDR8FVPQ#OK}Jz(Mnz6hW?52hUJ7{P&Mc=WKP#muvohbT+$hmF zC8HnYlPVK43)0PV%gRhmD@}6?iz})!ii$FG%Tx2R&62Zll*DF9Wk#my<~c=KSs96G zg~eG$l~v|uNjMjb=9`wKrj>v)G)5v%D@#nvHA~7%&dRJxt}-brG|4O}sLHM?H7`gi zFs-UA%TG%#O*PG|%E>4wPAf4s%`C>+5GXgVFe%RlE!Qf@D$7eWGcHX^PRd9FZ5Tyw z_L*52ni(1!5O?`hWl2U}sj(4w4>6%G0=PheH`|a3w9?9wjP$A$uqlM%8m=Etk!E6L zVyI_eig*8SMpa=7_+Y<^^wP4j@{-i70;8(LqLPBD@;vkG?7aNMRI{{{q)Zdj?DCw7 zl+4l`@Wo(Q>fNmL)UpEe#EkNkbmOw@+%)rw#ER0g6k|xU57FC8%}FaZEy&9*C@MC} z%PLAWFE2AGtx76SF32}7C^bziHciS+E-6h;F-y+O$STP%$jJd^L@YK|WmFX#6;+jF znUh+Xmu_BCo|ToDW|o&% z3c7l^1k_u^64Qm|Nv1g_d3mO#Mi~`3CFaSdCRurji8(l9I^WE=DmfW6ON&1Gm{^f& zR+*KYmu!@oUzKB?m{gT&W|EU*oRgeioK~EXmYGpZBN>JEKlCsK5 z)6C+sl$6wxY)P=2C9Mp0^-ZH!VzNl8JmmA-y%VsS}+N`9VR zeqK&yUaEdcVtTQDZfZ$lT26jZVvLePa!z7#v2BbJ=oFpI{5*w>#Nv#S#Pk>?g`(6P z+ZZJ<8)Ahrm`W}#NX!GXQW55WoRqAOkO9l1xmCXaEDEY5l8Q1@(kuOoGC{Z6=7bjI zC|Tiju|sl3W>I2dy1t>Iv7xziSf=d#Mz=xk0*VgJFw%k(5N#OHQbd(?k04yPZ<695p!m9lI zL_N^%C>)Li%{3!C(%7sPq%tQlDK)1Ur}FQY@Sw_)0QVUX&Q_YLh%ZiIDj1qIq%Cd_pE0c}V@)A?i3QJQn(yJ=dQ;f>YObgPh z3W}4;D$~*sZbRGuZ)|01u4jaHh$UKwE2FG9wJg0TE!WH}HOHvDtfDeGGsU#9vNSWR zusl1{Bq`C@EF-VXv?|fGsKVSR!!#S$LCNVkIaS8xWhPm9M%gB1$!S$)+3DsbnYbqE zb8<~f%99h5jj}L@AX7>ci!*Zaij%7f(({b-%#+LtGEK6Q3yiaqtBew}b4v@$K_?2P z=O*Tro0pfQn-!H|tw9Pa&C7C<^NkD5DwFcfG7ItxO_EEpOtIA<<|gJAhDMea#C5HU zGs;r)jldVG5^9Tr6SW?;)4a=!N{fxtE5V%?Jk13(1Mrk;rdB4FdY0y>gOKpy>eT#- zbkpprib9jj#Io$7s)Dq%l0@_LB1r9lG+B^Zo|;&kn`By$)ADT{GlMO8szWm>XvL26lQc1ebrd0B2^ z3Dy~+m@*Ov}v9H>yOsNB~~qLwX*inPye_#@J@V z6VsB5jY@MXi?UNqQi>9DN-|6e(kjanqyLIo>yL!59*qu_iW8UlV28QrbMOZs+|11d{Cbpp7?Q( zF@ik}k1a%O8&#&47Nk@ZgWKa6LyTEfrAeu#6{Q8mrTO`#`GpxJxoHJC>EP)$^d-HP zW`>5w7DV;Q%S;mUO4E#p8=MEb9NHE}xH}hQMovyK*bF?`5Lq{#641=b*hJ6L0zAZs zy(OKLS(R8+l&=74`6}d==B4B-B!iB{1u+tniW0$x<>looBqbIpfKGGFOjIaTC@D?J zS4hh*E>2Y_0PX83C`!dFH8aYKi!$(?rb^9w+|WQ@4Vs7$NKE6B`FDN3(2HOfoO%gi%3E-nK#Xwg%dsimQr zp^2d>WC0-T_>Yw2WV7^i(>#;RbmP?Gk^$>rI_pks)NEAots zGt*PdGb{3N^a;((i;QwJ@(Qza%uL>)K=4Ip;DSVhev**M7{@s|NXjWEOP-32%lb%zUmzdootG=?lGxIPc$z|GcPPOH%`wjGA>BX z%1Nv$haQ!HGTvxzZfYGF=MVOCW^dR1CUX1Q^0Dzt^2l$Dwc+5?aR zy^5zO&8Q%;+}Jd&qRO6uvtDJ7)^CYi;? zNfk-v73pS$CRt_~*%cY(nTdtjNoBb?WvR(n^IUPdaeh^jd3j!CW>#8av3W{bNl|KL zA~?3tW8KKY$k4*jf~ZC66^S_}`9PPb)3Ytg19h$}mbw%}O&WswgT*%uG+mHf3pMZfRy{ zZfZtUOQ^CaD?8VixMe`NV!E;@E445OYzE~qZ35bxX^FP{3N5D7(lT>RbMo^utI7(p ziwXC)H3GFGbd*5*|IMrnE%c1>A6Jr+o1UAVZ&Z+y zlVn<6Zd8Q2{P*j?pR+X2V zl98NTRg_VlQc{$kVv>@F^DgOZ;~cY`B+${$7_(1_Rrv*`6-fnIX@yA@#f9lPRaxmq z1<3`)Nu}ill_|M}rIneL#)+w!iA9xF6=r5R=Ehi4W~F(05@_fqr7SHmF+VlAJh>#h zDkT%^4sR4z< zY6Yl#2al8{XXd6W6c;5U4d)gc=@yoz=IW*ABxUBNgGTZ6!*x?#joi$W0?p#XBg#DT z^1brCDx5t{Lh=l3^)vN~jgb|X8R|QRhIqs$8KfDcT9_Cam?vABSR^J{Bw3i7nZVU1_dD za$;U`YEB|(1RZixJm}huynKb+)XaPZP~RglH#09IUq>Mkq_iZzC{ZCVzo;Zt7w#?4 zN!g~R5L=2<6%q@IQj1gbN)i?FQbET66sJ~{BM*8JJobn&=rC;%$zkWELiy z>LQ^0LZ}%=0TV z@)J$+OwDrBK}Yr$CMV~VCMBB`C`ijOO)JPtDycFqOw2DXFfT1DFETT&Do?H| z&CkbL|CgsHW@V+Al%^S{8KoK*3fT6LX^aAm!%SNqL~1 z`oxaWVJ3^xa*Qv$BnnN=(u!Q!_L3&5DeYvT`z$i?MENFf+C=G$V3p zSY>iXS!Eu0y$f*#7qsPpl$=W|lQZ(oAZ8FMxS+c66kH}&M#g%k=6FWvN)mJPL1*A7 zBp+U=P>_!?M3kPGl$vHzQCMrvk$a#>ZL z63U)~b|s+ilofEqC}%I%z3@^O!JhY@}iVfBeUG{!qn2DJoC!p6^YqK$t7iG=4lxzS-BOa=>=6u z*a}fI@V+zy6XJ4tRe5%46>+0`xcZ;vRpn`wMPM@snH%8U4xeF? zTVhmDmR)6*kyo5uk(zI0YEo2Lnv|WGoL-QdSYA?5Rat0WkXKcdZdzDSUX-3`grk}- zE;g^qElw*dEjKMMuSiWRt1?Z?Nl(Ig3RG&ASyDk_F1C?|3}Z86qs)o|qpaN05@X|J zqau^EeDi#x;FLEL#P#QJWm1#eGLy85 z-11~_yAV%gqZxoFzRf`E2~BZrE2uIz$~4bNsm#mGHY+kRsxmUqFe@o5h0fqYkAcak zGAcDHDX%I@&M{5Ss?5kMt1`|`GR^=EKUSKS6qICUn3k4iCS@2`<>VEo7H1`;;Ybb@ zNyeF#X8A>CNrmYqRTafmrm4AAMdrBL_qm1Ti79F6$wr`xSqFBUjlV*$Q-}^|Dg|^Q z4(OVz)LhWjTcDm&szOO>PHI_Z@!`do#e7nEUUp7eMpkZdVs=VKvTBrQ{ z)J)IV6z`7Z6r;?lyxc^iB(vfYv$BG8lPvSB0@JGe%uMt0LenJk+^RfN5p zv)tr@M6)tb^9f7MP?cYmUu<5Qnx0mkky@CNn_5|wnNwC!fUEtNQ(9D3m6&T}j#)FL zmKdj{mXw#}XBB3qWtx}dXXfP=CFZAOmzN|Kl@{jYmX>5Bm1LwAmY5l3B&VblWu{}T z8O*aXvU0Ohvoop+aAQG2a}#1QF>12Q0T{Sr{9d7#P%| zuIkE41g+|V&J;dhv*yXJp2y9NkDHc1Zf<_uwEl7P+{aBz9yf1(+}!cFaoyvFo+mrj zzHHte?D&39$CDl1kDI#Q@9BKp(EGS?`Qyf!j~kkv_I5q(XnWb%_ZWocKW<#{xN-91 zrryVm-5}YX>5m(CJe@ER#BFSQzM}haW7pFqE1zwc{|S!s4oc5yn6o++st z*GtoqN=i-hvF(6KNioYTDoHFX%uNCvdTC~oR*;`&lvG-pWRjU|Y;K&Bl$n`cnrBpM znvtEJUY=N*lZ|yPTBS*8X=-U!MNxTCX0CZvYF<@lNlFfQYz;l8%`J>A42>*^Iy9!t zxU@XEq}(VSJpONDWnidhj{ml=lr*E9{EQOg!V>e$Y~vE6lH8oK(n?cfo$6T3lY5 zmY186US3*W25M$t)b{yF#wCSCNySMiWv1y#6=vC)>4iB-S=h$^&5aEV3{8wo361|7 z=YY!Yq`dMZGw^8>#N{Vw367kfjFR&5GRnbb5Xw(b-FWhoDd=nx6UfbTpu!*DjX$Xs zkO6xo9VIx^s#+-}wK%ybv!En1KTpX@DL6APJttM+&~}CJP&a=ce|P_25XZ>C$QVQ$ zDir7EL5`uzD9ue$BK9_*q%7mijO4W3^2C(<(#)!=4D+PS@>H{|LX%YEQnQ?5(D`v` zRYn!XiAClW$%SRc$)F8nI!gE}6H{YT3scit9i@`Y67Vf+jtXI!C6I+CVX27Z3R~fv zUyz!oS6q-^l9^Tso*Bu^E6LF(o$?=6D6dyR$5Vpi)11P@9OEpbytIn4oQlHa(u|TEW1J@jW@aU)WG5CFnSo;r zWf&yAsKmUW%)BH&E5kI+EZf95FSpb*$;8~qG&RF4w=%IZ-Q1)!+t@s_*fcLAuQ1mv zu@LLTg0Wd?aivLdSxQnyYDSV-u4zGqQ9)%{8J4{$<|am#hDPQVkX2cr%lz>4Lyhvv zjq@wObu9ViN?ut(1;h+OaZOCQVqyflSAg8KkG$_8I5n{-(LKMkC^xkP$JPhX1@=fO z*&Mv@fuziaY$0e~FEcMFGaoVs2%4G1VXuh^%8m;o6X;$BP%&o}tyE9~YMX+06M#xP z9D50pi;60Y(z2@ZbIbBGsw$I>iV8BTGV^mY^2>8eGjp>`lgz8kOe?Z-%_>aF3zL#c zjY~m|J1o5^)8w@D)ZEmng4F!X#GEX%;^cg@)bu2rM|dP>n3<&|few?e1)pq-mK{^7 zGL3SR3NrJIvQ3N2vdhyFQ?iPaQ}XgN%hFQJQp&PRk`jxuONvWODzXYQimFnw({W7m zlR7M9lvrYBR+)-zDQ>z+NpZe$W>#jQaYa^nSxS0ZX1ZBrp1Dy~UX^jNnOQ-4ab8Kj ziBWo)Nn*N5QgK=$_Pf+G^D0bJGqXy{$}+3+jEgfW(h3X9k}9yBHeqgVZfX=IXNmR4?*S&^PsVVam&l%H5$keid0QJI`;Y-*O1RaBOloNbnG zlwn$spOcrGl8v>Oos^nfZkAPAWt>r#n2}_boSA8wn3j_TK2sfirI)d>nW2T51u?Cx z(u%aaf|6q7p>qObD$srhO3jy+msJVc#t+&pl$>9Ly zmeG=olT%W%D${b(^2*Z_(~L|iDvUFeQ&Np`Dyve=$`Y$`^3u)C3UZ87%uK2>lS-1Z zGI7*-E1Tpe-SW7O3||p?M(1s4OWzyD-@}xi~2?D<#vs z5HwMokz!&58oo_0&oVVGDXK6rOH4@tT|1VSW||EeJ;PFj6(^e-=ci?trKc9BB$}C+ zmlvd#nUxjdthh@n3XL*N)4_Mtqjd<eqZ%wK&OENafNG&rmF*m6yE3Yss&nqa&$69fl zf=(eaH6!M{urg!Qoa9VWPS=1|y+}zFWQJJ|#0<(y3RBQcRR)&eCK~pVBCs?Oyvki6 zATcQ?U!k}(DYFQ)iUhQ(9lW!&pa`-GB@uk%M5;njW|2Z!Vx~e#QEF}~bd(uspHNa( zo^eugZb3<=SxH8IYI$l-ZboTIVy?MyPGMSUrAcK)d1gjNW_nIhnMqk%X;N`M_MUM@ zQE5qDnt5_|aj8j4X{k|GQE^q7nF-c?LKdc$hK3fP5iVHyjPvNA(#neL3X^Oj@NO@n z)@(t`1EeSdnNgTj3^s#M6hU?4DNRg3$B7Xb14%6`FgL5rF)t}jD$mQ!E=Wu(F)7MR z%Sq3#Dk&>W&o#=aGD^y<$V^YqEw9W^O3pLOgO7or=j-B}wCu9V?5eWLER*Dlw2}<- z!p!uFx+!qSw=+^mAid}EV>lFYJTwqk5o0(#qo0Xkq zRBmjPZCqBIo}X-1Zk(Qyo0y!AwWD8JX=Y+nR#lmwlWCG#nNen1oSa@(QU>w>#&C+U znUSHnks)ydy&2gBl_|tkL1^cLdZBr)S-w$Ha#eDvxlyulSy4f8SyFMLd8KJ~L0)-wMrlb&YOYyD z1&%I2c7}P0X@zNdWr0a~QI&aCYH4~&rLkEut_hXQ%#6IsqHM^7Nn?Y(F_lCq6&3BFChvG&89nBRi!yyDTr=sI)2#_vWGE^2)NJszRe2ES-3h zqSWG|j5O0S)1*ApG-Ko9tc5|2RZe0`RaUB5wn?5zVNw!kb_*lbnp#*I8XFl9)oiUaEduSD)IoYZ9KNHOY}3-IAXBp;R;Wt6An z7K2S8R0hNK<4Mn^R>o#}#wK`|!K5Z8m8X`Qr>C1GmKhtT8mAW(=clGrn4}k(rRh1`}mm6o9WhCY&!W&LXC?!|ASx#nUVM$qbZcc%zS#d^zd2(`@Svt;T zFr`V^*@>p6*bXvCGE2-cH7`yp&8;j*G%Ez{J1fY{&nikUE+{rDuShL0$}P%DH!>|D~yV9^j8ZD3Nn+DDk@Wrii!)(((-am zN;0!j(r_N1R$7pkl?+miS#o8Vl^UBAXIAAKflk{>E;7kXu1ZO%Fi*`+FD@x9%u7l$ zO*GA}G&3zMFHfliZSXC?n*S@)OH(Q`lFZ6YE6R#8)3cLHlgmnSQ$a;9X8s4AlxJvx zSiue&?9?@|)HN~;0j&l$1+6ABw8VcpSz)C~NmW6qX|idac|}rUQDI(UQf^tkNm`O| zc1dwkeqL%}WmR6OXmSw@n1W_D&e)^RE`6HCy2z{C`i zrIjhUrX^M2!-EMHl4xywl$?{2Q(}??HU>{AiDm%a%wue%XKIQ%*aPn*RAp6`XBJf@ zCz&NBXPRXvCY$GicK8)zbQ01F(#n!5vdWVx%d#s?vr3XoGP8=x)2d2Jt4y;CD?uBt zbIeLhP12I9DpJcbOLMc#uqNfSEYq}YGrwHc6Me_et}t@X?|%{u4!^%iLq&NZc1`RSx$;kPEJ8$VgcwZ zkQ6f$^E6ZATxjT*XXat<=uS0BOfSeb&&V*!Pckz%D@dzK&(2IYO)AX9n}H`AA?wDIhs;2CG+LN~w@_gp zMoY{DZ`J}WV9Zy@O)Se)$c6#T$TT(0Oau)}BxV#ARF#yKmlkDbXBp+2o0eAQ7G|VZ zC8npFWR&L>XO)#yWaXw77GuA0C^@Gn1vHS8lU-O?Ri08@TAY!YlT(OwL%oTak)f#t zXmSe{qPU01DvAm#Gb@pIyO1&9UQ}3`12Kb8@Do%28<^^uo8j$mCzoU;8<*v#r>2`^ z6{c0D8Wm?$nimwAS7n!H6zAup7x8=F@c zrDYp|4T5xCNRH|GdQb0W=EKyyaSCE=lmX@ArZeEp}Vq{vB zTy9!clvRnNb&^|JQIb<`nvZP=GtIctI3+VJ8?<~XFQuxq$|Nx>ImJ9B-84VD$RsPr z)FeMU#XQfXvLZj-xG=ZCG|LR@+P|`F6LV8@6O+WE#Hv)&iq!nveDjq2JaFd{txPqu zG&2L8l0npPL{(C;X-NgRJcN}1_`7h>Y>C`7sY)uU1g+i&9ebHd=`5NFXjQVQDc($- zR#;Y=o1dCuY@Cx)S(KZen~`m7Vw_h}m|SR{ZI*6gQe0S&mS&QiRal%{T$W{;1Ui}k z>%bGq7se$Q8)cc5VcV~gT9#~7l$}y+o?V%lYm{1^kyvJuRZwVVQkj@kP?VEgo|RdZ zmY8c)m{ysRTwIY|!*8EGb_<>pyw zCZ**`rC3{(d1=N4mAM&N<#|a}dHHEsxw!=ups55bt8gp~3{4G<&CQ5u&zG5`6`Ld` zL&LEERFHy(22i$aC1)g-=p}(pJWfn5$uH7NNlng4EJ_3qiRi$VUV)Cc#F}I9>rTwc z$uIZJOU?&vL07U;O3cYoLO)bDxx^^7Fe^8?)U>!FKfTx}wJ^V^w4lNy+r+H2+{i33 zFC*WyAUUJNG$*m7syrvHCZDa$hgA4`V)Tv7A1=h_Gp@{sx(&2o6LN@!b7@gYrUI8TczKv#K77#@M!HW+DacRH zNUACG`nc>_5US^!1Se|Esb(W_xwbV2# zFU7pPDk;k-Dc8u@xGX0pE2lEU*u1Q`vZ^GnvcN1YDK)7wDW#$;BfF|32b}J!L49l` zD-dZ_kdtg>o|9LeY*dwLY+g~AQ<|1umQ|UNY*dJ?vNba@wlp*{Hnv0*T6p?OCZNGm z@OS~im_{2lKyF!8n53H(rh$#Y6WM46;3?7rRNtW7v-81rRJMwmX=neXXR#BC$l2kB&{qt zrP8dxI6ohIOdBU=XI7OK6@jMYF)FIWZ1eP_irm67lgzB#3^OTYd^NP&0B9l~5MTMSWvyF2y3(Cz+%!~69Gct?{jWhFdlC!ZM zQ;!#FcNEw4N)F()xAtE?!`%%sF5+a#x|vN$)Zv@$=%EUBWT zpbE6+rL-usM9E4CZ34i=%EUy^$P91NN;5SsN;a>oGS163Eze0WO)kjHD6c3o%1KN$ zPpv33F09BZG&M=ANJ%y;G%n0bF3v3mO-5j87!>ALRhAc3W@M+O7#rm!nx_~Q8y8n4 z8)HveCb`AMX`oxP!ABgRcNmJ3%StngjdSzM(-KQ_N>Z!L(#$K9(#sN4va?I7ib~Rw ztBR7-OcP5h3Mw*6DnSSHVoO@N<;i8KMrOul8C6+{r5S03CAqoDCHYu~EiDa<3@wa^ zYVDSp7#md;LT=e6SQj8C4y2?7GNYu-m`az0m{}Pc=ouN}uk|yG%(9Y<%0V~&7G&iZ zr5I=DV zGAqt2&n+&_OEN0SFRMt-Da2aq7ntQ&Wv7>ASEVFXW>pzg=4E6TrQ~5f&BM&V(9+P@ z+!8u&2I_DngT}%VlaupHA=d>YCYNNEr7BsK6qTmxfLazv?F&3M6{i*|Ss7XyTN)V} zm>7f1&&&n&CegB`v6Ydbo-t@%4$GC}g*o}f1*KJ0C1oikCAno~6{h7zRk_LFaXa); zAyXqWLn9L-$XEgD`5%d8W@Sbx8Rb>xsU|sPCB~H|#>J`G$r)v76~<+GWkvRk=kOl__~ym079f#z`iLxkWiesV2raj{ztu%g9g4HiPb^ z#6E`Z=;`UH@L~RpwdfeL{a=xmoKakmn3R8gmX(Hmk!edUL`YbO)KC zF>yy=8y8h(8d12>*0`uL*{lq_35fDa#>~plQqK_o+Mf*5#H1>-io*1g!lVpS^ThO= ze6!s0%#!@F!jiQ7B-8RNbJLWna+9pef`Y8XQj-!BTqlDW8JU+Ql_#3ynB*GgB~@l+ z6{lC`l;z_*0w6Qls4N*=DPuGz(#ujSD+@BpQ!275({jslORCaJOU;VY5;KyGjZ&)e z(~EMmit@^nQ<5{wbBdFVjEgd`mIV3Pg#~$)#w8h<$;O#kW`#+q#mUB*Mp!S1vNSa| zG9_|3aaCeQL3SeY+8Vi|FV zuHdRQjY>?)($c_Y5DG!U4KhtGsg(LAB)^hmQi|1a&`&W3~I!*v8kScA$T7Z&X~?MDk(_ID9JUdOfk(e zuPQ4qDmAG{tSZf_N~_4MtVm5yDoroRG)c`Z2OVIPQC19II*O$wOzMirqLQSnv^+DT zbj)={X-TD~Nk*C3<(Y{Ui76Gur`KhKQI2NxNC#9$7n5Pt)7=f0Zp*6zON(z#*Q*(3k(@aZ^ zO>;8R3Mxv{lT)%yOOuSMax=;kD~<9~QjLm|a!t$gip)(?(knna`!V+9XXcof8WmO) znq?N|Cgta5r&m_w73G(KatV4OGcz_eHZ(RPcqT7sPkw1dT6(TI=;Q;$=nc*xXSDGY zq;@#Sn9O8|F?iA$ngMuM+U1FA5mStX* zR#lpmm5tHxNl8g8D9FvKG)^(kF3c;+si-hdD^JWQNij3ZD@--7D9<)BORvf=&o56h zEi2B<&CkrnI*vlZ&B*4429}0q#zw>x=A~5ypzCG8C#n1rQemliVM$eqQA%==k!fCWN=0Q^mPuu0MoxxVerZy9VrE)SLAj}MUPeJq zY9`iLFfGi_M%l$?sU?M}xtT?|CPtNIg=RTspx&WbNm@x+MSf0t znR!`a5~#+v6EAmX=0F1{TDeNMB}bW}awPY6M{Rf&nIX5ii!dhuvsXlh|- zY(!M?SY}d^Sz($5-iJ&ugMwRI@RisIm*;{^DNHT`n}VlwMAeTcdzyjTtY${2OBB%) zYFSlTT1G}@PMJw+W^txjd0AdgRZ(G*Nmh1Z2536I$~Yq{-^?T>r7AO}$ke3LtQY?NGLo|%|cmS0#|l97ljp%$hlW@V=oU^}2D&A1>ZHL=t< zqcpKJyC|zXHKoY3Br)5pD9a=_&nVL@BO^H{r=Y5=psXU_I5)8{yPyi|6n0K(nTc_p zS#Dm5k%>uOa&l@$eok&?BG&z=MxdPuCdBPrH!{N*{WAqGWitc!58#CYya+5YFU~DZ z%q&byEh(=^Nvt$8F3n0$&n&^Hh%%Cj(~a^?j4QL#QnFI>%5zMzOia_wlJY9eO4IXG z(^HI-(^4~%jf(QJ(zA1_jFZYzu{HzBDw7LJ%2G^?(kqgat5S;-Q;khBOpUS5Je!yq z8yZ^@cYRhxQBiVIiV-*jAz3sTXEOktPNDq*q*Pv7Q3P5U0XBnBDu?RElg3S~3{3UR zEb*rC^s1EPB-7l&%(A44#L8roiqibl%;NH_tZbvIid?hO{EV#9g6z_&!rV#|le`q8 zirfqwGtc>ng@u`gW|>(gc_~$8xfx}~`Bgc^skkm_$|=ptGcGeX!nWr+sWLseBsI;f zFe$%0t=POIH6tT8*{q_dAg3a)u(Bd0E3K@cG{e{&RGb+nrxll{nq%!hnpEberePRu~tjo9E>x78K@G6jv1G8|S7~RZ>A-Q4yAeWoBk!W@uz#M%0SA$_!)U^fDu`IhEk`KLm;p=->=;5mK39RF;zl zHiJ;Yg6hUwgczCYSz3SxCa7@=Pe@T>S!!`H_*PwPw-1BHs*#T1F|wfS3?ARayv#(G z)Uf>I!>e!{zGDbF-44l0L-5&uGmZ013iES8Ep7DHo|%P-rJ=EzAu$cr z(#mpZ{x?oFF3rl%GA~Kb&#E#?$tcb?HcvLsEGR5XFUv7aEK4;tugu6V$jwhGOE*c% zFflCxb^S3q?NvFZMg=+Lm8B(lg-N+-*(E9E*(O;j;4FeR4PkC%Vrgh%U_wl{vCPP% zu-q&IJbqU~R91u5w@6tHWJYlk#0)}N4XPVYRx`CSGSM@!1TXl9x029Wq#2;QbF)ng zDhji5Q*(;Tb4?11%_@o$%PLFDs&dT>D>4exD>GA5(ks)_Qp`$I^HY${phMYPNa_(U z#rc_~DY>Q4$rO}@7OBQ5#aT%vDFsHw<@t%3=4ScjSvg51DP_jR$%Sd4{aWQE1;!>- zIYsG-WhrT8MalVCyQVpn>B+g7*;&S!+39)d#>Q#sS!r3tX`m7iJw=+ESy~zySQ2%a zSDA4}VQD6$rX@S3jWY@hb0B6=9@F5h9p(fs$0*A)%}P$sD#%SQH%duMOe-%*OEk$! zNy^E{%*;1QPAbW$OixZNFfUI_E6q1e0?pB3-~XOnnp##^l$}wWS6G^ql4M+!XWyU#`S!oqU;6VwZ@+Q_Eqj64UYE}W*6hd(Z*N-QE zf=|FQGQ&G^n`~5+oK|3BTA6BER+Uy^T5Ot|n37eQ47%dMG&QxXD7m1@JSnZ*I5{h) zC?%`J*t8h5jtEN|E5)=lH!m&OJl`mL_yK26L` zDK0InEYGYgNi(Y|FwHDX$t+CC%F8z@sWizqOD|4Ot}4$cN;5GrOUcd7PfRJwz*=J% z<)D+%l7t zO!K4yurYX|8_fVb@of&895**Z-LwktwVD)~CFW(DC6%P5R~RK0B_&p76qr?3WMTAL z6Z6Uv%Z*YC^7BjzjPuizlCsi@jnYevGcwAPbJI&p3v$fO%T06g^76`(@=7vuii?b~ zZhXruD^II5D>p7EE-%fg03Du}TT)i zT4h!m*o>l7&}M}6?3}#P^vv8m)7B6-D`#S*Cf(8Of!2 z1?i>P`KI~VnMI&A9~i}mrIERzxuvNgF~x|9X@+rCnvpT6RZgfFK}#3NwY-UGdYM@s z*chrLEHf)(Lp@`2)LT2@3Cjp{m`PP;Np4w1Ze@~ra$;hMQAv(D<`Fs>$>vFUW@*M{ zsb&?WnK@aBRfSdgi6sSDxrIh$m5Jsl*{LO_1xaODNjZt8*%?MdA=Ea#N z<;JOn#yKUah34sHg~d6b+W|2`-`K*^(9+l#vC$h(C)p_5)GQ?pe4jSi3Ck$kq%xxn zYzCg7NA6#n%#z%Eh|0q&6$%coEC$Ud zDWobCAKt7`k_x%KJ3Te81a!t^u0lp)Nvc9#YFWo=KXqNm@xxZdqk{Io5VZW=djZ zQf_vZd2y<7RhD_8k#SC0a#jIo{sX;$GP5u+GPE=>CvLE%*fb}}9Na!6w}2|GEH=$Z zg_uEk0cB!kY^i5vhFUS8^#D`SQZiHWi*n18s)|jDj4D&gvNQ4$i!-W{Qp+6{IF*W~NkRWM?IVPZUBMw@A+@D>hHdFRx5bD@irYEJ)8zO-m}N zs?0J^FDuD5$}a~k$v4Y1%1^DxEi@@FD>cc&I&V>4l9ZU9m6?-SWRh8Ko?2K~Y@TmY zl7lrhm|Gecnwk)|1HH1aGO^s4xXr3)SqZ7JP+D17X|EKr;YOUNiwM%D1q< zkr&I%GIO#`vMMr4^YgRvs{PV$l~%c{zgD$>);vP-JWi^`Jn)ACZ% z!0T(!OF{!9b3mMyICMG7DRVJpJ<(gNR6d7fJ?)fXIFwU+_Hm@kn$}ur7$jK;9GR;XXGEFm2O3o?D zFfB^N-bXj8sLHA;tt?8fGE2$Lsx+!d$;e48PtUD3Zb4dRZgIJBQ3~jwBGV*uvx35sBvY)VZdPiBQF3aI znR$9;VN$tiZca*No_S&tcmf(Nrp-((O%06=Oc4DI=oMgzSs7(!W%-Hel?5p!Y3aGS zMy4sINma%s1=%Gnz04>H*W^-eLB4rbN^T0YAi_SD>sqRinwOlPS5gE%j{|cg0l3gq$Jal5|bjM+_c<`oWd;A(!3M{45g_^E{JGvq~cz119PDMTO>>#<|6rMQQ03MHR_r+1WXHIXI7e zE;KDqE-y95HeixeRG4p8Xp(J~W0Gl}W1N?il2nugI^!ulIVr8E+|<-G*T}3YE2+fD zI3?3OxiCL32Wu9}$*QW%GD<8;EiX<^&dn(_I`b2Y8d3Lrb*bK^J+QiDxM9&1an8IAo)Ch06nVe&uXl|CBTv?Qy zmztfJY?5SZRG5;OT2zpfTa=TSo>*E@l%8KunNggPmXw~8nO108ioM({EHTT@EK1BP zG&8Tt%_`2UOvx@Ut;oT(>oX^(vLe3%di5<@`y@57G^;W*Gc_X}w7M~+w9v#Hbfs2G zR(f%1acZ(zo~dz0MP*`kNqTmPnMq21MWrd$_DNYmQASx#iK$s~ab|95VR=D@d1`tx zc&ie6q?=oSR?!$B?f<~joi;8n%`Gc80?%EO8R=!l<)s-_`Cv0Bk90FDBhX4#{Cm2} ziqZ>nOmj+1OEYtdN-7Fd%QMWgQ_a$|j0!W;vvSG`^FY%VnNgNenXyrJrcqi>QE^6Mnpp*o8Zsx%q_oPYxWp&}v$dLDn3rWrS7>ZvZfIzR6w{fY25V+YYFCiu&N)LgUlobvRHf<#l(^6cbf z^V~cm)BK#uqST_C%G}Zv)BK8T)1=B$)9l>x!nDMsqMRfgD-v>wO3RYWOY+JpO3QMx zjZ4jvjH`+(ii&Zhj%@Rsth|C;=y}Dc`<0TjvJ+Fw@{B91%yY9$(kly73X)1P(^5*) z3i7k_603^xijpf+$_mVq&5SC`j7!YRv0v0vTxObHl3tpUnv{~1Q*4x$Zd6#6S(t%! z-qO_A1hm+gsI@0Xpv`Fo;L%9pTFTH#exy!HnNdZGNeXCz1mSJNP~CWXD5h4Xpo_GP zL3vNH0_GtA6#)6|ytd<))@&ChC=>=H(ZGuNNyQsPHNe^3yG-$ai#fbad8F zDN0SuO-?LH&PXlNMdc*sX6mJ9rrBp?*czIe$CMfv7@3uq+Zut7^+Q?^k(#Fz3%^!3 zr?LoS7zn2pft$|xc_me4kX=qh)l=}fHlvA$E`>+y z%_Zd*=B4CklvNld7v|(ws5DM4ElJJI%S%crOHIteQ5xlz86}mN=7L(b7<CDk#cN z%gh0V95lXho>hcXV`5oiNn#PG(o}&Kfk`>}pyiuIscB$Q@SS&7N=b=n*+r?D8HJf? zmAS@gMp=oeX_Yy7M(L^9MHTrOX(ojwCMMY`r6k9+)F`dQ$fP9GJlQNYH8D9SCpRxCBP+SIq$n$~Bqi0v z1l0FWPc%16&MnExOCq9xF)}wXGcW_4)l`(4Sdxl-1&@WUk!6UXrImphXn~0_WJm+` z%Ee6eoE(MX)KrDa{8EL?JcY!({F02+B88mHv{Z%ijMO}Z@>GSyqEv;X{E`fXBsHxdxhko&Dy68h%q-nJBR{7w%Oo?kv^XczI3u;f z7{};`d3sTPMW$(5dYVa5Ri$}(rnz}#Zdn1&eGa7==EjL>*+y6nwof!OHYq4gOEfPv zDK@W2&Pz4UF)lAJGOMg8FG_jG zD~pRWi@|0Psyv8^X=6)0Q^*QPT+6aj%t}&BOU=tGtEx&0iVDh2b1L&IbMp!_($h<_ z&2lpG%2JZEP4ZI;vWkq-^0RXi3o<|jAf)O>S(a6jmu*&1S!QCAnU`*sWSX0tm|>P` zmWJz0tK5=Q^CF|dbR$#DZcbWZWs*r!R(fGhc2aJYky%zoaRKPGg{)+=?3|3!vZQ?D zf*hlqG?S9#g0!l%;~tQ;Z7=Q_M5cD@(9c9cCs5riRAF zsbx5;x{{nCqmoQhqY_N_CgogA<`!nghNk8w z#7q~J8CRqfC#4#J7dMev27%1TO)CYPL3zpqcfif@mq8iXrj^-Q=0$16xrvD-iRGDz z#py=LX6cow$tGEaDWz4J`6b55S!rek<&{RoRmoY!#yGlIq#jY5QBa;$Zkl0~hZ)nU zdHEIPMI}W>Sthyp`DrO-g&8KLCPw9XxkiOK<2nWk7{ zIyX75$|yG}(KMqZCAYBHyr>v-C`}o71Qe}NV`gY+Xk=t&ju@iGJ=9WKUQwJvX>C(p zQEW=J+QtYp)M8$#^fU zgzLxCL^1)bpEEYWAKQ6}#)a9H*%f6O#+6xF`4#zD8Rf>Q#)awW$t9+#W`!n6Rc2+W zWw|-|$;qWfY2{{VIGRXVCFVvIRVCRurN${n$r)uiIYt?oRYhjF`dld$WhEJAIap`V z5{pYr(sGMUt165OG7EByOY+Q$Q%qBh%96~}3eC!@Dl&4?(m;FpDocy2s?v?K%dw6R znx|!z=4MnGC6=0}8s%1Im}lpjW~3EhJL%Ef5VYOgg1FeOD#|Y~PXq6SB)f@JRg|9! z>bl|I?Tp;O!V}Y`R)%JJ28OUPf9Uw1iMgJo3Eml^JE`mFJ~nJN+>!tE9BBFfGHpBqJp=uc|a9sl>D_J=M50GdsW1Jh!wY zx1_Kjt1{iBFr_rjD8H~Gvk>cItMb%bBV*I#%&N?a!rY|f@(j>zX`n;OQ;I8!igGGTl8p<^)6=VR(u;~sOLJ3-(kpS~bMu^ZqufH%w8FBI z;)3LqQlrA`qO8&!oHONF<%!0{rRhc$n5BAhK}AYIaz;){R&H8hc~x;~l5u`{QK?aK zig9^WK}u?Jrg>6wfpI}@YL#hnMq#3926#FdV~0wmS$=9-c4|p!W?7keR!U-eaao#C zT46bs2C}&Y=x{ev1ELOyG_I;l%t!{G#!6gFL&s{7%Jeehs!Ee=h#7=p8mb#lJ~y*6 zvd}ZL0Iz_@KBaDCppcPoU}S=H_m45C?P6$&WAm?Rc}cR7ab|W(3TVG_N};()S$d{< z1?J}8q%$(#)T)AvjP&y2N^{f1 z)bv!Wg>rI|Q9(*-c5X^)RfV}}aamGXiCKD95@_@YqY5@KGcg1$AtGX}acOx@aaDdU zc-I}l$U%;eIPoRVzQw4AIoqr#-B ze6!ptv!b%Xw5lwuqXwzv1^HP;M&NrWD|3pCjZ$*6iz`e)EjaYhH?=f1H?**@ByQ9o zDL19C1TxM#p>J$uV6JCsh&t>H5B+pwW25vG)8dlsiqeYI zRCCZlv-v5-$x5JMyiBC#XG)=2SwT@wcD}iBd0w)yX<}(snMrP0PHAaYUUFtpS#nVh zXoAipBR8wqxU{^YG9?jf=$jdvR+wd(=B1bAWF{Mzm?oJe8Jkz-S70lsOhH=+OpS3>wp~NG>$XFG)!^ zGDGq5lRv zCVWcE^9l>nvcSjc5iA>!n_MVKGp{f!s|sueo}`Ja8&9cV3@XTt@ZW=xSX7djQJ9xf zQks`pno(L-k)2UoV4PW*QdVx7lapAUnqHP^RGw0hT4SQB_7& zs##%Ca(b0%im6d+~D!a-kJFzU^$fP*4*tj^u zyfP&_**q%=Yb{`5XbIZdG9D8CmIOgl=pmGN#R}OfB`y zEm4<`qK{LXBv+K@n&hSCXXlm`Wu;`48s#RLRG22EmKA0gm82JBmKBzyXO`zARTP2l z-7`-IpTLJDrVDd(%rdhe`+t**iYkoKva0fP%knd-DwBZSicuQ))DYqd7-(A$sel2Q zVP<9uHiJ+B1J#YEfH4DAd`8CL-XHdsVo0F7LPlmDXtiZ>W`15`jzUVJf^&Y(;VrpI zpwj`+R%a!pq^ISVn3bg@7N-~)m6=zXn--ar7iVPW8|RivewC1#eGm1ZSX zrYBYvR$|>_oLF30R9R}2S!ixvo|J1=k(iNMU{+Cv^-NB43j;$#V`E}gWt3JHRHf#Z z7=hdTgc=~oSr;h=KxUYgq=3yJ6a&Qc%8bqRjLq=&%91KG$|@=gi}K7%%9FE=3rlkH zb8@RHD)W+4j4M-9bIOd;vU9SEO-;;F($b1D%~H}*K{I(+GJbYNMNUywxlw6)QigE> zWHl1lo1mnkZ=3@qztr2uvSM2>%|}$85J2O6{bcN8K!x8#hFQF~uf7|2B;o_Rr;`51*sT28TP zj(N7RaanOmc3MtJc9~hSnQ2i~YFTPwR+*V`d5KwNW^tBDRYqy5aZz?kVs19pg@#p$ zrpZNx#ii!iiRG!;xn>1r#yM%}N!U&rGqp4aEyXt@YA~@pCDXJh-v~TRMO0FT4+bJ9 z=JJ$`igGisDTER;TtA)y#l*_MM9t=o0^rBQdp6jQc_TymX}kWQJ8B~n3`Fbnq_QUXp&`=U6zuUf_1K=AU`*) zFh8#(sjAqtG&QL#xvV&^C_M|?E=@B_BMUaVGc8O^1RwrRT-5_kQ2Xuvml8#bhaY1TwNf79AGmsG(shQ~+B}!IC2B3R4$}>|+GL)yP%UR9NGMpdR zG&ip(%gX~Ta>~UzHd9bhRZwPHT$)^3Ra%*qQkk5domG^c0cww;kIk4Hnwc0Hni&uk z(^aMUDHSH*O-01TG_=5kr%;pJ(yG#YbE8tQ8Ab5S3@e9FV;ZU(YfKx27=rE!v@$i+ NGcW*;mlJ#98UX)HUhe<^ diff --git a/legacy/cozy-setup (move to another repo).md b/legacy/cozy-setup (move to another repo).md deleted file mode 100644 index 71d50bd..0000000 --- a/legacy/cozy-setup (move to another repo).md +++ /dev/null @@ -1,496 +0,0 @@ -cozy-ucosm - - -## gateway - -- tailscale (exit node enabled) - -> allow ipv4 and ipv6 forwarding -- caddy - - ```bash - apt install golang - go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest - go/bin/xcaddy build \ - --with github.com/caddyserver/cache-handler \ - --with github.com/darkweak/storages/badger/caddy \ - --with github.com/mholt/caddy-ratelimit - # then https://caddyserver.com/docs/running#manual-installation - - mkdir /var/cache/caddy-badger - chown -R caddy:caddy /var/cache/caddy-badger/ - ``` - - - `/etc/caddy/Caddyfile` - - ``` - { - cache { - badger - api { - prometheus - } - } - } - - links.bsky.bad-example.com { - reverse_proxy link-aggregator:6789 - - @browser `{header.Origin.startsWith("Mozilla/5.0")` - rate_limit { - zone global_burst { - key {remote_host} - events 10 - window 1s - } - zone global_general { - key {remote_host} - events 100 - window 60s - log_key true - } - zone website_harsh_limit { - key {header.Origin} - match { - expression {header.User-Agent}.startsWith("Mozilla/5.0") - } - events 1000 - window 30s - log_key true - } - } - respond /souin-api/metrics "denied" 403 # does not work - cache { - ttl 3s - stale 1h - default_cache_control public, s-maxage=3 - badger { - path /var/cache/caddy-badger/links - } - } - } - - gateway:80 { - metrics - cache - } - ``` - well... the gateway fell over IMMEDIATELY with like 2 req/sec from deletions, with that ^^ config. for now i removed everything except the reverse proxy config + normal caddy metrics and it's running fine on vanilla caddy. i did try reducing the rate-limiting configs to a single, fixed-key global limit but it still ate all the ram and died. maybe badger w/ the cache config was still a problem. maybe it would have been ok on a machine with more than 1GB mem. - - - alternative proxies: - - - nginx. i should probably just use this. acme-client is a piece of cake to set up, and i know how to configure it. - - haproxy. also kind of familiar, it's old and stable. no idea how it handle low-mem (our 1gb) vs nginx. - - sozu. popular rust thing, fast. doesn't have rate-limiting or cache feature? - - rpxy. like caddy (auto-tls) but in rust and actually fast? has an "experimental" cache feature. but the cache feature looks good. - - rama. build-your-own proxy. not sure that it has both cache and limiter in their standard features? - - pingora. build-your-own cloudflare, so like, probably stable. has tools for cache and limiting. low-mem...? - - cache stuff in pingora seems a little... hit and miss (byeeeee). only a test impl for Storage for the main cache feature? - - but the rate-limiter has a guide: https://github.com/cloudflare/pingora/blob/main/docs/user_guide/rate_limiter.md - - what i want is low-resource reverse proxy with built-in rate-limiting and caching. but maybe cache (and/or ratelimiting) could be external to the reverse proxy - - varnish is a dedicated cache. has https://github.com/varnish/varnish-modules/blob/master/src/vmod_vsthrottle.vcc - - apache traffic control has experimental rate-limiting plugins - - -- victoriametrics - - ```bash - curl -LO https://github.com/VictoriaMetrics/VictoriaMetrics/releases/download/v1.109.1/victoria-metrics-linux-amd64-v1.109.1.tar.gz - tar xzf victoria-metrics-linux-amd64-v1.109.1.tar.gz - # and then https://docs.victoriametrics.com/quick-start/#starting-vm-single-from-a-binary - sudo mkdir /etc/victoria-metrics && sudo chown -R victoriametrics:victoriametrics /etc/victoria-metrics - - ``` - - - `/etc/victoria-metrics/prometheus.yml` - - ```yaml -global: - scrape_interval: '15s' - -scrape_configs: - - job_name: 'link_aggregator' - static_configs: - - targets: ['link-aggregator:8765'] - - job_name: 'gateway:caddy' - static_configs: - - targets: ['gateway:80/metrics'] - - job_name: 'gateway:cache' - static_configs: - - targets: ['gateway:80/souin-api/metrics'] - ``` - - - `ExecStart` in `/etc/systemd/system/victoriametrics.service`: - - ``` - ExecStart=/usr/local/bin/victoria-metrics-prod -storageDataPath=/var/lib/victoria-metrics -retentionPeriod=90d -selfScrapeInterval=1m -promscrape.config=/etc/victoria-metrics/prometheus.yml - ``` - -- grafana - - followed `https://grafana.com/docs/grafana/latest/setup-grafana/installation/debian/#install-grafana-on-debian-or-ubuntu` - - something something something then - - ``` - sudo grafana-cli --pluginUrl https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.11.1/victoriametrics-datasource-v0.11.1.zip plugins install victoriametrics - ``` - -- raspi node_exporter - - ```bash - curl -LO https://github.com/prometheus/node_exporter/releases/download/v1.8.2/node_exporter-1.8.2.linux-armv7.tar.gz - tar xzf node_exporter-1.8.2.linux-armv7.tar.gz - sudo cp node_exporter-1.8.2.linux-armv7/node_exporter /usr/local/bin/ - sudo useradd --no-create-home --shell /bin/false node_exporter - sudo nano /etc/systemd/system/node_exporter.service - # [Unit] - # Description=Node Exporter - # Wants=network-online.target - # After=network-online.target - - # [Service] - # User=node_exporter - # Group=node_exporter - # Type=simple - # ExecStart=/usr/local/bin/node_exporter - # Restart=always - # RestartSec=3 - - # [Install] - # WantedBy=multi-user.target - sudo systemctl daemon-reload - sudo systemctl enable node_exporter.service - sudo systemctl start node_exporter.service - ``` - - todo: get raspi vcgencmd outputs into metrics - -- nginx on gateway - - ```nginx - # in http - - ## - # cozy cache - ## - proxy_cache_path /var/cache/nginx keys_zone=cozy_zone:10m; - - ## - # cozy limit - ## - limit_req_zone $binary_remote_addr zone=cozy_ip_limit:10m rate=50r/s; - limit_req_zone $server_name zone=cozy_global_limit:10m rate=1000r/s; - - # in sites-available/constellation.microcosm.blue - - upstream cozy_link_aggregator { - server link-aggregator:6789; - keepalive 16; - } - - server { - listen 8080; - listen [::]:8080; - - server_name constellation.microcosm.blue; - - proxy_cache cozy_zone; - proxy_cache_background_update on; - proxy_cache_key "$scheme$proxy_host$uri$is_args$args$http_accept"; - proxy_cache_lock on; # make simlutaneous requests for the same uri wait for it to appear in cache instead of hitting origin - proxy_cache_lock_age 1s; - proxy_cache_lock_timeout 2s; - proxy_cache_valid 10s; # default -- should be explicitly set in the response headers - proxy_cache_valid any 15s; # non-200s default - proxy_read_timeout 5s; - proxy_send_timeout 15s; - proxy_socket_keepalive on; - - limit_req zone=cozy_ip_limit nodelay burst=100; - limit_req zone=cozy_global_limit; - limit_req_status 429; - - location / { - proxy_pass http://cozy_link_aggregator; - include proxy_params; - proxy_http_version 1.1; - proxy_set_header Connection ""; # for keepalive - } - } - ``` - - also `systemctl edit nginx` and paste - - ``` - [Service] - Restart=always - ``` - - —https://serverfault.com/a/1003373 - - now making browsers redirect to the microcosm.blue url: - - ``` - [...] - server_name links.bsky.bad-example.com; - - add_header Access-Control-Allow-Origin * always; # bit of hack to have it here but nginx doesn't like it in the `if` - if ($http_user_agent ~ ^Mozilla/) { - # for now send *browsers* to the new location, hopefully without impacting api requests - # (yeah we're doing UA test here and content-negotatiation in the app. whatever.) - return 301 https://constellation.microcosm.blue$request_uri; - } - [...] - ``` - -- nginx metrics - - - download nginx-prometheus-exporter - https://github.com/nginx/nginx-prometheus-exporter/releases/download/v1.4.1/nginx-prometheus-exporter_1.4.1_linux_amd64.tar.gz - - - err actually going to make mistakes and try with snap - `snap install nginx-prometheus-exporter` - - so it got a binary for me but no systemd task set up. boooo. - `snap remove nginx-prometheus-exporter` - - - ```bash - curl -LO https://github.com/nginx/nginx-prometheus-exporter/releases/download/v1.4.1/nginx-prometheus-exporter_1.4.1_linux_amd64.tar.gz - tar xzf nginx-prometheus-exporter_1.4.1_linux_amd64.tar.gz - mv nginx-prometheus-exporter /usr/local/bin - useradd --no-create-home --shell /bin/false nginx-prometheus-exporter - nano /etc/systemd/system/nginx-prometheus-exporter.service - # [Unit] - # Description=NGINX Exporter - # Wants=network-online.target - # After=network-online.target - - # [Service] - # User=nginx-prometheus-exporter - # Group=nginx-prometheus-exporter - # Type=simple - # ExecStart=/usr/local/bin/nginx-prometheus-exporter --nginx.scrape-uri=http://gateway:8080/stub_status --web.listen-address=gateway:9113 - # Restart=always - # RestartSec=3 - - # [Install] - # WantedBy=multi-user.target - systemctl daemon-reload - systemctl start nginx-prometheus-exporter.service - systemctl enable nginx-prometheus-exporter.service - ``` - - - nginx `/etc/nginx/sites-available/gateway-nginx-status` - - ```nginx - server { - listen 8080; - listen [::]:8080; - - server_name gateway; - - location /stub_status { - stub_status; - } - location / { - return 404; - } - } - ``` - - ```bash - ln -s /etc/nginx/sites-available/gateway-nginx-status /etc/nginx/sites-enabled/ - ``` - - -## bootes (pi5) - -- mount sd card, touch `ssh` file echo `echo "pi:$(echo raspberry | openssl passwd -6 -stdin)" > userconf.txt` -- raspi-config: enable pcie 3, set hostname, enable ssh -- put ssh key into `.ssh/authorized_keys` -- put `PasswordAuthentication no` in `/etc/ssh/sshd_config` -- `sudo apt update && sudo apt upgrade` -- `sudo apt install xfsprogs` -- `sudo mkfs.xfs -L c11n-kv /dev/nvme0n1` -- `sudo mount /dev/nvme0n1 /mnt` -- set up tailscale -- `sudo tailscale up` -- `git clone https://github.com/atcosm/links.git` -- tailscale: disable bootes key expiry -- rustup `curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh` -- `cd links/constellation` -- `sudo apt install libssl-dev` needed -- `sudo apt install clang` needed for bindgen -- (in tmux) `cargo build --release` -- `mkdir ~/backup` -- `sudo mount.cifs "//truenas.local/folks data" /home/pi/backup -o user=phil,uid=pi` -- `sudo chown pi:pi /mnt/` -- `RUST_BACKTRACE=full cargo run --bin rocks-restore-from-backup --release -- --from-backup-dir "/home/pi/backup/constellation-index" --to-data-dir /mnt/constellation-index` -etc -- follow above `- raspi node_exporter` -- configure victoriametrics to scrape the new pi -- configure ulimit before starting! `ulimit -n 16384` -- `RUST_BACKTRACE=full cargo run --release -- --backend rocks --data /mnt/constellation-index/ --jetstream us-east-2 --backup /home/pi/backup/constellation-index --backup-interval 6 --max-old-backups 20` -- add server to nginx gateway upstream: ` server 100.123.79.12:6789; # bootes` -- stop backups from running on the older instance! `RUST_BACKTRACE=full cargo run --release -- --backend rocks --data /mnt/links-2.rocks/ --jetstream us-east-1` -- stop upstreaming requests to older instance in nginx - - -- systemd unit for running: `sudo nano /etc/systemd/system/constellation.service` - - ```ini - [Unit] - Description=Constellation backlinks index - After=network.target - - [Service] - User=pi - WorkingDirectory=/home/pi/links/constellation - ExecStart=/home/pi/links/target/release/main --backend rocks --data /mnt/constellation-index/ --jetstream us-east-2 --backup /home/pi/backup/constellation-index --backup-interval 6 --max-old-backups 20 - LimitNOFILE=16384 - Restart=always - - [Install] - WantedBy=multi-user.target - ``` - - -- todo: overlayfs? would need to figure out builds/updates still, also i guess logs are currently written to sd? (oof) -- todo: cross-compile for raspi? - ---- - -some todos - -- [x] tailscale: exit node - - [!] link_aggregator: use exit node - -> worked, but reverted for now: tailscale on raspi was consuming ~50% cpu for the jetstream traffic. this might be near its max since it would have been catching up at the time (max jetstream throughput) but it feels a bit too much. we have to trust the jetstream server and link_aggregator doesn't (yet) make any other external connections, so for now the raspi connects directly from my home again. -- [x] caddy: reverse proxy - - [x] build with cache and rate-limit plugins - - [x] configure systemd to keep it alive -- [x] configure caddy cache -- [x] configure caddy rate-limit -- [ ] configure ~caddy~ nginx to use a health check (once it's added) -- [ ] ~configure caddy to only expose cache metrics to tailnet :/~ -- [x] make some grafana dashboards -- [ ] raspi: mount /dev/sda on boot -- [ ] raspi: run link_aggregator via systemd so it starts on startup (and restarts?) - -- [x] use nginx instead of caddy -- [x] nginx: enable cache -- [x] nginx: rate-limit -- [ ] nginx: get metrics - - - - ---- - -nginx cors for constellation + small burst bump - -```nginx -upstream cozy_constellation { - server :6789; # bootes; ip so that we don't race on reboot with tailscale coming up, which nginx doesn't like - keepalive 16; -} - -server { - server_name constellation.microcosm.blue; - - proxy_cache cozy_zone; - proxy_cache_background_update on; - proxy_cache_key "$scheme$proxy_host$uri$is_args$args$http_accept"; - proxy_cache_lock on; # make simlutaneous requests for the same uri wait for it to appear in cache instead of hitting origin - proxy_cache_lock_age 1s; - proxy_cache_lock_timeout 2s; - proxy_cache_valid 10s; # default -- should be explicitly set in the response headers - proxy_cache_valid any 2s; # non-200s default - proxy_read_timeout 5s; - proxy_send_timeout 15s; - proxy_socket_keepalive on; - - # take over cors responsibility from upsteram. `always` applies it to error responses. - proxy_hide_header 'Access-Control-Allow-Origin'; - proxy_hide_header 'Access-Control-Allowed-Methods'; - proxy_hide_header 'Access-Control-Allow-Headers'; - add_header 'Access-Control-Allow-Origin' '*' always; - add_header 'Access-Control-Allow-Methods' 'GET' always; - add_header 'Access-Control-Allow-Headers' '*' always; - - - limit_req zone=cozy_ip_limit nodelay burst=150; - limit_req zone=cozy_global_limit burst=1800; - limit_req_status 429; - - location / { - proxy_pass http://cozy_constellation; - include proxy_params; - proxy_http_version 1.1; - proxy_set_header Connection ""; # for keepalive - } - - - listen 443 ssl; # managed by Certbot - ssl_certificate /etc/letsencrypt/live/constellation.microcosm.blue/fullchain.pem; # managed by Certbot - ssl_certificate_key /etc/letsencrypt/live/constellation.microcosm.blue/privkey.pem; # managed by Certbot - include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot - ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot - -} - -server { - if ($host = constellation.microcosm.blue) { - return 301 https://$host$request_uri; - } # managed by Certbot - - - server_name constellation.microcosm.blue; - listen 80; - return 404; # managed by Certbot -} -``` - -re-reading about `nodelay`, i should probably remove it -- nginx would then queue requests to upstream, but still service them at the configured limit. it's fine for my internet since the global limit isn't nodelay, but probably less "fair" to clients if there's contention around the global limit (earlier requests would get all of theirs serviced before later ones can get in the queue) - -leaving it for now though. - - -### nginx logs to prom - -```bash -curl -LO https://github.com/martin-helmich/prometheus-nginxlog-exporter/releases/download/v1.11.0/prometheus-nginxlog-exporter_1.11.0_linux_amd64.deb -apt install ./prometheus-nginxlog-exporter_1.11.0_linux_amd64.deb -systemctl enable prometheus-nginxlog-exporter.service - -``` - -have it run as www-data (maybe not the best idea but...) -file `/usr/lib/systemd/system/prometheus-nginxlog-exporter.service` -set User under service and remove capabilities bounding - -```systemd -User=www-data -#CapabilityBoundingSet= -``` - -in `nginx.conf` in `http`: - -```nginx -log_format constellation_format "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\""; -``` - -in `sites-available/constellation.microcosm.blue` in `server`: - -```nginx -# log format must match prometheus-nginx-log-exporter -access_log /var/log/nginx/constellation-access.log constellation_format; -``` - -config at `/etc/prometheus-nginxlog-exporter.hcl` - - - -```bash -systemctl start prometheus-nginxlog-exporter.service -``` diff --git a/legacy/old-readme-details.md b/legacy/old-readme-details.md deleted file mode 100644 index 77a3b1b..0000000 --- a/legacy/old-readme-details.md +++ /dev/null @@ -1,35 +0,0 @@ -[Constellation](./constellation/) --------------------------------------------- - -A global atproto backlink index ✨ - -- Self hostable: handles the full write throughput of the global atproto firehose on a raspberry pi 4b + single SSD -- Storage efficient: less than 2GB/day disk consumption indexing all references in all lexicons and all non-atproto URLs -- Handles record deletion, account de/re-activation, and account deletion, ensuring accurate link counts and respecting users data choices -- Simple JSON API - -All social interactions in atproto tend to be represented by links (or references) between PDS records. This index can answer questions like "how many likes does a bsky post have", "who follows an account", "what are all the comments on a [frontpage](https://frontpage.fyi/) post", and more. - -- **status**: works! api is unstable and likely to change, and no known instances have a full network backfill yet. -- source: [./constellation/](./constellation/) -- public instance: [constellation.microcosm.blue](https://constellation.microcosm.blue/) - -_note: the public instance currently runs on a little raspberry pi in my house, feel free to use it! it comes with only with best-effort uptime, no commitment to not breaking the api for now, and possible rate-limiting. if you want to be nice you can put your project name and bsky username (or email) in your user-agent header for api requests._ - - -App: Spacedust --------------- - -A notification subscription service 💫 - -using the same "link source" concept as [constellation](./constellation/), offer webhook notifications for new references created to records - -- **status**: in design - - -Library: [links](./links/) ------------------------------------- - -A rust crate (not published on crates.io yet) for optimistically parsing links out of arbitrary atproto PDS records, and potentially canonicalizing them - -- **status**: unstable, might remain an internal lib for constellation (and spacedust, soon) diff --git a/legacy/original-notes.md b/legacy/original-notes.md deleted file mode 100644 index b321ea5..0000000 --- a/legacy/original-notes.md +++ /dev/null @@ -1,123 +0,0 @@ ---- - - -old notes follow, ignore ------------------------- - - -as far as i can tell, atproto lexicons today don't follow much of a convention for referencing across documents: sometimes it's a StrongRef, sometimes it's a DID, sometimes it's a bare at-uri. lexicon authors choose any old link-sounding key name for the key in their document. - -it's pretty messy so embrace the mess: atproto wants to be part of the web, so this library will also extract URLs and other URIs if you want it to. all the links. - - -why ---- - -the atproto firehose that bluesky sprays at you will contain raw _contents_ from peoples' pdses. these are isolated, decontextualized updates. it's very easy to build some kinds of interesting downstream apps off of this feed. - -- bluesky posts (firesky, deletions, ) -- blueksy post stats (emojis, ) -- trending keywords () - -but bringing almost kind of _context_ into your project requires a big step up in complexity and potentially cost: you're entering "appview" territory. _how many likes does a post have? who follows this account?_ - -you own your atproto data: it's kept in your personal data repository (PDS) and noone else can write to it. when someone likes your post, they create a "like" record in their _own_ pds, and that like belongs to _them_, not to you/your post. - -in the firehose you'll see a `app.bsky.feed.post` record created, with no details about who has liked it. then you'll see separate `app.bsky.feed.like` records show up for each like that comes in on that post, with no context about the post except a random-looking reference to it. storing these in order to do so is up to you! - -**so, why** - -everything is links, and they're a mess, but they all kinda work the same, so maybe some tooling can bring down that big step in complexity from firehose raw-content apps -> apps requiring any social context. - -everything is links: - -- likes -- follows -- blocks -- reposts -- quotes - -some low-level things you could make from links: - -- notification streams (part of ucosm) -- a global reverse index (part of ucosm) - -i think that making these low-level services as easy to use as jetstream could open up pathways for building more atproto apps that operate at full scale with interesting features for reasonable effort at low cost to operate. - - -extracting links ---------------- - - -- low-level: pass a &str of a field value and get a parsed link back - -- med-level: pass a &str of record in json form and get a list of parsed links + json paths back. (todo: should also handle dag-cbor prob?) - -- high-ish level: pass the json record and maybe apply some pre-loaded rules based on known lexicons to get the best result. - -for now, a link is only considered if it matches for the entire value of the record's field -- links embedded in text content are not included. note that urls in bluesky posts _will_ still be extracted, since they are broken out into facets. - - -resolving / canonicalizing links --------------------------------- - - -### at-uris - -every at-uri has at least two equivalent forms, one with a `DID`, and one with an account handle. the at-uri spec [illustrates this by example](https://atproto.com/specs/at-uri-scheme): - -- `at://did:plc:44ybard66vv44zksje25o7dz/app.bsky.feed.post/3jwdwj2ctlk26` -- `at://bnewbold.bsky.team/app.bsky.feed.post/3jwdwj2ctlk26` - -some applications, like a reverse link index, may wish to canonicalize at-uris to a single form. the `DID`-form is stable as an account changes its handle and probably the right choice to canonicalize to, but maybe some apps would actually perfer to canonicalise to handles? - -hopefully atrium will make it easy to resolve at-uris. - - -### urls - -canonicalizing URLs is more annoying but also a bit more established. lots of details. - -- do we have to deal with punycode? -- follow redirects (todo: only permanent ones, or all?) -- check for rel=canonical http header and possibly follow it -- check link rel=canonical meta tag and possibly follow it -- do we need to check site maps?? -- do we have to care at all about AMP? -- do we want anything to do with url shorteners?? -- how do multilingual sites affect this? -- do we have to care about `script type="application/ld+json"` ??? - -ugh. is there a crate for this. - - -### relative uris? - -links might be relative, in which case they might need to be made absolute before being useful. is that a concern for this library, or up to the user? (seems like we might not have context here to determine its absolute) - - -### canonicalizing - -there should be a few async functions available to canonicalize already-parsed links. - -- what happens if a link can't be resolved? - - ---- - -- using `tinyjson` because it's nice -- maybe should switch to serde_json to share deps with atrium? - -- would use atrium for parsing at-uris, but it's not in there. there's a did-only version in the non-lib commands.rs. its identifier parser is strict to did + handle, which makes sense, but for our purposes we might want to allow unknown methods too? - - - rsky-syntax has an aturi - - adenosyne also - - might come back to these - - -------- - -rocks - -```bash -ROCKSDB_LIB_DIR=/nix/store/z2chn0hsik0clridr8mlprx1cngh1g3c-rocksdb-9.7.3/lib/ cargo build -``` diff --git a/legacy/ufos ops (move to micro-ops).md b/legacy/ufos ops (move to micro-ops).md deleted file mode 100644 index 3c2b8f3..0000000 --- a/legacy/ufos ops (move to micro-ops).md +++ /dev/null @@ -1,196 +0,0 @@ -ufos ops - -btrfs snapshots: snapper - -```bash -sudo apt install snapper -sudo snapper -c ufos-db create-config /mnt/ufos-db - -# edit /etc/snapper/configs/ufos-db -# change -TIMELINE_MIN_AGE="1800" -TIMELINE_LIMIT_HOURLY="10" -TIMELINE_LIMIT_DAILY="10" -TIMELINE_LIMIT_WEEKLY="0" -TIMELINE_LIMIT_MONTHLY="10" -TIMELINE_LIMIT_YEARLY="10" -# to -TIMELINE_MIN_AGE="1800" -TIMELINE_LIMIT_HOURLY="22" -TIMELINE_LIMIT_DAILY="4" -TIMELINE_LIMIT_WEEKLY="0" -TIMELINE_LIMIT_MONTHLY="0" -TIMELINE_LIMIT_YEARLY="0" -``` - -this should be enough? - -list snapshots: - -```bash -sudo snapper -c ufos-db list -``` - -systemd - -create file: `/etc/systemd/system/ufos.service` - -```ini -[Unit] -Description=UFOs-API -After=network.target - -[Service] -User=pi -WorkingDirectory=/home/pi/ -ExecStart=/home/pi/ufos --jetstream us-west-2 --data /mnt/ufos-db/ -Environment="RUST_LOG=info" -LimitNOFILE=16384 -Restart=always - -[Install] -WantedBy=multi-user.target -``` - -then - -```bash -sudo systemctl daemon-reload -sudo systemctl enable ufos -sudo systemctl start ufos -``` - -monitor with - -```bash -journalctl -u ufos -f -``` - -make sure a backup dir exists - -```bash -mkdir /home/pi/backup -``` - -mount the NAS - -```bash -sudo mount.cifs "//truenas.local/folks data" /home/pi/backup -o user=phil,uid=pi -``` - -manual rsync - -```bash -sudo rsync -ahP --delete /mnt/ufos-db/.snapshots/1/snapshot/ backup/ufos/ -``` - -backup script sketch - -```bash -NUM=$(sudo snapper --csvout -c ufos-db list --type single --columns number | tail -n1) -sudo rsync -ahP --delete "/mnt/ufos-db/.snapshots/${NUM}/snapshot/" backup/ufos/ -``` - -just crontab it? - -`sudo crontab -e` -```bash -0 1/6 * * * rsync -ahP --delete "/mnt/ufos-db/.snapshots/$(sudo snapper --csvout -c ufos-db list --columns number | tail -n1)/snapshot/" backup/ufos/ -``` - -^^ try once initial backup is done - - ---columns subvolume,number - -subvolume -number - - - - -gateway: follow constellation for nginx->prom thing - -config at `/etc/prometheus-nginxlog-exporter.hcl` - -before: `/etc/prometheus-nginxlog-exporter.hcl` - -```hcl -listen { - port = 4044 -} - -namespace "nginx" { - source = { - files = [ - "/var/log/nginx/constellation-access.log" - ] - } - - format = "$remote_addr - $remote_user [$time_local] \"$request\" $status $upstream_cache_status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\"" - - labels { - app = "constellation" - } - - relabel "cache_status" { - from = "upstream_cache_status" - } -} -``` - -after: - -```hcl -listen { - port = 4044 -} - -namespace "constellation" { - source = { - files = [ - "/var/log/nginx/constellation-access.log" - ] - } - - format = "$remote_addr - $remote_user [$time_local] \"$request\" $status $upstream_cache_status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\"" - - labels { - app = "constellation" - } - - relabel "cache_status" { - from = "upstream_cache_status" - } - - namespace_label = "vhost" - metrics_override = { prefix = "nginx" } -} - -namespace "ufos" { - source = { - files = [ - "/var/log/nginx/ufos-access.log" - ] - } - - format = "$remote_addr - $remote_user [$time_local] \"$request\" $status $upstream_cache_status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\"" - - labels { - app = "ufos" - } - - relabel "cache_status" { - from = "upstream_cache_status" - } - - namespace_label = "vhost" - metrics_override = { prefix = "nginx" } -} -``` - - -```bash -systemctl start prometheus-nginxlog-exporter.service -``` - diff --git a/links/Cargo.toml b/links/Cargo.toml deleted file mode 100644 index 212fb35..0000000 --- a/links/Cargo.toml +++ /dev/null @@ -1,11 +0,0 @@ -[package] -name = "links" -version = "0.1.0" -edition = "2021" - -[dependencies] -anyhow = "1.0.95" -fluent-uri = "0.3.2" -nom = "7.1.3" -thiserror = "2.0.9" -tinyjson = "2.5.1" diff --git a/links/src/at_uri.rs b/links/src/at_uri.rs deleted file mode 100644 index 2437884..0000000 --- a/links/src/at_uri.rs +++ /dev/null @@ -1,306 +0,0 @@ -use fluent_uri::{Uri, UriRef}; -use std::sync::LazyLock; - -static BASE: LazyLock> = LazyLock::new(|| Uri::parse("https://example.com").unwrap()); - -// normalizing is a bit opinionated but eh -/// see "Full AT URI Syntax" at https://atproto.com/specs/at-uri-scheme -/// this parser is intentinonally lax: it should accept all valid at-uris, and -/// may accept some invalid at-uris. -/// -/// at the moment this implementation is quite bad and incomplete -pub fn parse_at_uri(s: &str) -> Option { - // for now, just working through the rules laid out in the docs in order, - // without much regard for efficiency for now. - - // The overall URI is restricted to a subset of ASCII characters - if !s.is_ascii() { - return None; - } - - // Maximum overall length is 8 kilobytes (which may be shortened in the future) - if s.len() > (8 * 2_usize.pow(10)) { - return None; - } - - // Hex-encoding of characters is permitted (but in practice not necessary) - // -> decode any unreserved characters. from rfc 3986: - // -> For consistency, percent-encoded octets in the ranges of ALPHA - // -> (%41-%5A and %61-%7A), DIGIT (%30-%39), hyphen (%2D), period (%2E), - // -> underscore (%5F), or tilde (%7E) should not be created by URI - // -> producers and, when found in a URI, should be decoded to their - // -> corresponding unreserved characters by URI normalizers. - let s = if let Some((unencoded_prefix, rest)) = s.split_once('%') { - let mut out = String::with_capacity(s.len()); - out.push_str(unencoded_prefix); - for segment in rest.split('%') { - let Some((hex2, unencoded_suffix)) = segment.split_at_checked(2) else { - return None; // bail: % must always be followed by 2 hex digits - }; - let Ok(decoded) = u8::from_str_radix(hex2, 16).map(char::from) else { - return None; // bail: % must be followed by decodable hex - }; - if matches!(decoded, 'A'..='Z' | 'a'..='z' | '0'..='9' | '.' | '-' | '_' | '~') { - out.push(decoded); - } else { - out.push('%'); - out.push_str(&hex2.to_ascii_uppercase()); // norm - } - out.push_str(unencoded_suffix); - } - out - } else { - s.to_string() - }; - - // The URI scheme is `at`, and an authority part preceded with double slashes is always - // required, so the URI always starts at:// - // -> the spec doesn't explicitly say, but uri schemes can be case-insensitive? - let (proto, rest) = s.split_at_checked(5)?; - if !proto.eq_ignore_ascii_case("at://") { - return None; - } - - // An authority section is required and must be non-empty. the authority can be either an - // atproto Handle, or a DID meeting the restrictions for use with atproto. note that the - // authority part can not be interpreted as a host:port pair, because of the use of colon - // characters (:) in DIDs. Colons and unreserved characters should not be escaped in DIDs, - // but other reserved characters (including #, /, $, &, @) must be escaped. - // Note that none of the current "blessed" DID methods for atproto allow these - // characters in DID identifiers - - // An optional path section may follow the authority. The path may contain multiple segments - // separated by a single slash (/). Generic URI path normalization rules may be used. - - // An optional query part is allowed, following generic URI syntax restrictions - - // An optional fragment part is allowed, using JSON Path syntax - - // -> work backwards from fragment, query, path -> authority - let mut base = rest; - let (mut fragment, mut query, mut path) = (None, None, None); - if let Some((pre, f)) = base.split_once('#') { - base = pre; - fragment = Some(f); - } - if let Some((pre, q)) = base.split_once('?') { - base = pre; - query = Some(q); - } - if let Some((pre, p)) = base.split_once('/') { - base = pre; - path = Some(p); - } - let mut authority = base.to_string(); - - if authority.is_empty() { - return None; - } - - // Normalization: Authority as handle: lowercased - if !authority.starts_with("did:") { - // lowercase handles - authority.make_ascii_lowercase(); - } - - // Normalization: No trailing slashes in path part - // Normalization: No duplicate slashes or "dot" sections in path part (/./ or /abc/../ for example) - // -> be so lazy - let path = match path { - Some(p) => { - let p = p.trim_end_matches('/'); - let uri_ref = UriRef::parse(p).ok()?; // fully bail if we can't parse path - let resolved = uri_ref.resolve_against(&*BASE).unwrap(); // both fail conditions are specific to BASE - let normalized = resolved.normalize().path().to_string(); - let without_trailing_slashes = normalized.trim_end_matches('/'); - Some(without_trailing_slashes.to_string()) - } - None => None, - }; - - let mut out = format!("at://{authority}"); - if let Some(p) = path { - // no need for `/` -- it's added by fluent_uri normalization - out.push_str(&p); - } - if let Some(q) = query { - out.push('?'); - out.push_str(q); - } - if let Some(f) = fragment { - out.push('#'); - out.push_str(f); - } - - Some(out) - - // there's a more normalization to do still. ugh. -} - -pub fn at_uri_collection(at_uri: &str) -> Option { - let (proto, rest) = at_uri.split_at_checked(5)?; - if !proto.eq_ignore_ascii_case("at://") { - return None; - } - let (_did, rest) = rest.split_once('/')?; - if let Some((collection, _path_rest)) = rest.split_once('/') { - return Some(collection.to_string()); - } - if let Some((collection, _query_rest)) = rest.split_once('?') { - return Some(collection.to_string()); - } - if let Some((collection, _hash_rest)) = rest.split_once('#') { - return Some(collection.to_string()); - } - Some(rest.to_string()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_at_uri_parse() { - for (case, expected, detail) in vec![ - ("", None, "empty"), - (" ", None, "whitespace"), - ("https://bad-example.com", None, "not at scheme"), - ("at://µcosm.bad-example.com", None, "not ascii"), - ( - "at://bad-example.com", - Some("at://bad-example.com"), - "handle, authority-only", - ), - ( - "at://did:plc:hdhoaan3xa3jiuq4fg4mefid", - Some("at://did:plc:hdhoaan3xa3jiuq4fg4mefid"), - "DID, authority-only", - ), - ( - "at://bad-example.com/app.bsky.feed.post/3jwdwj2ctlk26", - Some("at://bad-example.com/app.bsky.feed.post/3jwdwj2ctlk26"), - "bsky post (handle)", - ), - ( - "at://did:plc:hdhoaan3xa3jiuq4fg4mefid/app.bsky.feed.post/3ldqksainxc27", - Some("at://did:plc:hdhoaan3xa3jiuq4fg4mefid/app.bsky.feed.post/3ldqksainxc27"), - "bsky post (DID)", - ), - ( - "AT://bad-example.com", - Some("at://bad-example.com"), - "scheme case is normalized", - ), - ( - "at://bad-example.com", - Some("at://bad-example.com"), - "scheme case is normalized", - ), - ( - "at://bad-example.com?q=z", - Some("at://bad-example.com?q=z"), - "query is allowed", - ), - ( - "at://bad-example.com#a", - Some("at://bad-example.com#a"), - "fragment is allowed", - ), - ( - "at://bad-example.com/%", - None, - "invalid percent-encoding: ends with %", - ), - ( - "at://bad-example.com/%2", - None, - "invalid percent-encoding: ends with only one digit after %", - ), - ( - "at://bad-example.com/%ZZ", - None, - "invalid percent-encoding: non-hex after %", - ), - ( - "at://bad-example.com/%3A", - Some("at://bad-example.com/%3A"), - "valid percent-encoding is left", - ), - ( - "at://bad-example.com/%3a", - Some("at://bad-example.com/%3A"), - "valid percent-encoding is hex-uppercased", - ), - ( - "at://bad-example.com/%61/%62", - Some("at://bad-example.com/a/b"), - "unreserved characters are percent-decoded", - ), - ( - "at://bad-example.com/a/../b", - Some("at://bad-example.com/b"), - "paths have traversals resolved (oof)", // reminder to self: we are normalizing, not sanitizing - ), - ( - "at://bad-example.com/../", - Some("at://bad-example.com"), - "paths always have trailing slashes removed", - ), - ] { - assert_eq!( - parse_at_uri(case), - expected.map(|s| s.to_string()), - "{detail}" - ); - } - } - - #[test] - fn test_at_uri_collection() { - for (case, expected, detail) in vec![ - ("", None, "empty"), - ("at://did:plc:vc7f4oafdgxsihk4cry2xpze", None, "did only"), - ( - "at://did:plc:vc7f4oafdgxsihk4cry2xpze/collec.tion", - Some("collec.tion"), - "no path (weird)", - ), - ( - "at://did:plc:vc7f4oafdgxsihk4cry2xpze/collec.tion/path", - Some("collec.tion"), - "normal at-uri", - ), - ( - "at://did:plc:vc7f4oafdgxsihk4cry2xpze/collec.tion?query", - Some("collec.tion"), - "colleciton with query", - ), - ( - "at://did:plc:vc7f4oafdgxsihk4cry2xpze/collec.tion#hash", - Some("collec.tion"), - "colleciton with hash", - ), - ( - "at://did:plc:vc7f4oafdgxsihk4cry2xpze/collec.tion/path?query#hash", - Some("collec.tion"), - "colleciton with everything", - ), - ( - "at://did:web:example.com/collec.tion/path", - Some("collec.tion"), - "did:web", - ), - ( - "at://did:web:example.com/col.lec.tio.ns.so.long.going.on.and.on", - Some("col.lec.tio.ns.so.long.going.on.and.on"), - "long collection", - ), - ] { - assert_eq!( - at_uri_collection(case), - expected.map(|s| s.to_string()), - "{detail}" - ); - } - } -} diff --git a/links/src/did.rs b/links/src/did.rs deleted file mode 100644 index 7db5f8d..0000000 --- a/links/src/did.rs +++ /dev/null @@ -1,152 +0,0 @@ -/// see https://atproto.com/specs/did#at-protocol-did-identifier-syntax -/// this parser is intentinonally lax: it should accept all valid DIDs, and -/// may accept some invalid DIDs. -/// -/// at the moment this implementation might also be quite bad and incomplete -pub fn parse_did(s: &str) -> Option { - // for now, just working through the rules laid out in the docs in order, - // without much regard for efficiency for now. - - // The entire URI is made up of a subset of ASCII, containing letters (A-Z, a-z), - // digits (0-9), period, underscore, colon, percent sign, or hyphen (._:%-) - if !s - .chars() - .all(|c| matches!(c, 'A'..='Z' | 'a'..='z' | '0'..='9' | '.' | '_' | ':' | '%' | '-')) - { - return None; - } - - // The URI is case-sensitive - // -> (nothing to check) - - // The URI starts with lowercase `did:` - let unprefixed = s.strip_prefix("did:")?; - - // The method segment is one or more lowercase letters (a-z), followed by : - let (method, identifier) = unprefixed.split_once(':')?; - if !method.chars().all(|c| c.is_ascii_lowercase()) { - return None; - } - - // The remainder of the URI (the identifier) may contain any of the above-allowed - // ASCII characters, except for percent-sign (%) - // -> ok, ugh, gotta know our encoding context for this - - // The URI (and thus the remaining identifier) may not end in ':'. - if identifier.ends_with(':') { - return None; - } - - // Percent-sign (%) is used for "percent encoding" in the identifier section, and - // must always be followed by two hex characters - // -> again incoding context (bleh) - - // Query (?) and fragment (#) sections are allowed in DID URIs, but not in DID - // identifiers. In the context of atproto, the query and fragment parts are not - // allowed. - // -> disallow here -- the uri decoder should already split them out first. - - // DID identifiers do not generally have a maximum length restriction, but in the - // context of atproto, there is an initial hard limit of 2 KB. - // -> we're in atproto, so sure, let's enforce it. (would be sensible to do this - // -> first but we're following doc order) - if s.len() > (2 * 2_usize.pow(10)) { - return None; - } - - // -> it's not actually written in the spec, but by example in the spec, the - // -> identifier cannot be empty - if identifier.is_empty() { - return None; - } - - Some(s.to_string()) - // the only normalization we might want would be percent-decoding, but we - // probably leave that to the uri decoder -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_did_parse() { - for (case, expected, detail) in vec![ - ("", None, "empty str"), - (" ", None, "whitespace str"), - ("z", None, "not a did"), - ("did:plc", None, "no identifier separator colon"), - ("did:plc:", None, "missing identifier"), - ( - "did:web:bad-example.com", - Some("did:web:bad-example.com"), - "web did", - ), - ( - "did:plc:hdhoaan3xa3jiuq4fg4mefid", - Some("did:plc:hdhoaan3xa3jiuq4fg4mefid"), - "plc did", - ), - ( - "DID:plc:hdhoaan3xa3jiuq4fg4mefid", - None, - "'did:' prefix must be lowercase", - ), - ( - "did:ok:z", - Some("did:ok:z"), - "unknown did methods are allowed", - ), - ("did:BAD:z", None, "non-lowercase methods are not allowed"), - ("did:bad:z$z", None, "invalid chars are not allowed"), - ( - "did:ok:z:z", - Some("did:ok:z:z"), - "colons are allowed in identifier", - ), - ("did:bad:z:", None, "colons not are allowed at the end"), - ("did:bad:z?q=y", None, "queries are not allowed in atproto"), - ("did:bad:z#a", None, "anchors are not allowed in atproto"), - ] { - assert_eq!(parse_did(case), expected.map(|s| s.to_string()), "{detail}"); - } - } - - #[test] - fn test_doc_exmples_atproto() { - // https://atproto.com/specs/did#at-protocol-did-identifier-syntax - for case in ["did:plc:z72i7hdynmk6r22z27h6tvur", "did:web:blueskyweb.xyz"] { - assert!(parse_did(case).is_some(), "should pass: {case}") - } - } - - #[test] - fn test_doc_exmples_lexicon() { - // https://atproto.com/specs/did#at-protocol-did-identifier-syntax - for case in [ - "did:method:val:two", - "did:m:v", - "did:method::::val", - "did:method:-:_:.", - "did:key:zQ3shZc2QzApp2oymGvQbzP8eKheVshBHbU4ZYjeXqwSKEn6N", - ] { - assert!(parse_did(case).is_some(), "should pass: {case}") - } - } - - #[test] - fn test_doc_exmples_invalid() { - // https://atproto.com/specs/did#at-protocol-did-identifier-syntax - for case in [ - "did:METHOD:val", - "did:m123:val", - "DID:method:val", - "did:method:", - "did:method:val/two", - "did:method:val?two", - "did:method:val#two", - ] { - assert!(parse_did(case).is_none(), "should fail: {case}") - } - } -} diff --git a/links/src/lib.rs b/links/src/lib.rs deleted file mode 100644 index 112019b..0000000 --- a/links/src/lib.rs +++ /dev/null @@ -1,147 +0,0 @@ -use fluent_uri::Uri; - -pub mod at_uri; -pub mod did; -pub mod record; - -pub use record::collect_links; - -#[derive(Debug, Clone, Ord, Eq, PartialOrd, PartialEq)] -pub enum Link { - AtUri(String), - Uri(String), - Did(String), -} - -impl Link { - pub fn into_string(self) -> String { - match self { - Link::AtUri(s) => s, - Link::Uri(s) => s, - Link::Did(s) => s, - } - } - pub fn as_str(&self) -> &str { - match self { - Link::AtUri(s) => s, - Link::Uri(s) => s, - Link::Did(s) => s, - } - } - pub fn name(&self) -> &'static str { - match self { - Link::AtUri(_) => "at-uri", - Link::Uri(_) => "uri", - Link::Did(_) => "did", - } - } - pub fn at_uri_collection(&self) -> Option { - if let Link::AtUri(at_uri) = self { - at_uri::at_uri_collection(at_uri) - } else { - None - } - } - pub fn did(&self) -> Option { - let did = match self { - Link::AtUri(s) => { - let rest = s.strip_prefix("at://")?; // todo: this might be safe to unwrap? - if let Some((did, _)) = rest.split_once("/") { - did - } else { - rest - } - } - Link::Uri(_) => return None, - Link::Did(did) => did, - }; - Some(did.to_string()) - } -} - -#[derive(Debug, PartialEq)] -pub struct CollectedLink { - pub path: String, - pub target: Link, -} - -// normalizing is a bit opinionated but eh -pub fn parse_uri(s: &str) -> Option { - Uri::parse(s).map(|u| u.normalize().into_string()).ok() -} - -pub fn parse_any_link(s: &str) -> Option { - at_uri::parse_at_uri(s).map(Link::AtUri).or_else(|| { - did::parse_did(s) - .map(Link::Did) - .or_else(|| parse_uri(s).map(Link::Uri)) - }) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_uri_parse() { - let s = "https://example.com"; - let uri = parse_uri(s).unwrap(); - assert_eq!(uri.as_str(), s); - } - - #[test] - fn test_uri_normalizes() { - let s = "HTTPS://example.com/../"; - let uri = parse_uri(s).unwrap(); - assert_eq!(uri.as_str(), "https://example.com/"); - } - - #[test] - fn test_uri_invalid() { - assert!(parse_uri("https:\\bad-example.com").is_none()); - } - - #[test] - fn test_any_parse() { - assert_eq!( - parse_any_link("https://example.com"), - Some(Link::Uri("https://example.com".into())) - ); - - assert_eq!( - parse_any_link( - "at://did:plc:44ybard66vv44zksje25o7dz/app.bsky.feed.post/3jwdwj2ctlk26" - ), - Some(Link::AtUri( - "at://did:plc:44ybard66vv44zksje25o7dz/app.bsky.feed.post/3jwdwj2ctlk26".into() - )), - ); - - assert_eq!( - parse_any_link("did:plc:44ybard66vv44zksje25o7dz"), - Some(Link::Did("did:plc:44ybard66vv44zksje25o7dz".into())) - ) - } - - #[test] - fn test_at_uri_collection() { - assert_eq!( - parse_any_link("https://example.com") - .unwrap() - .at_uri_collection(), - None - ); - assert_eq!( - parse_any_link("did:web:bad-example.com") - .unwrap() - .at_uri_collection(), - None - ); - assert_eq!( - parse_any_link("at://did:web:bad-example.com/my.collection/3jwdwj2ctlk26") - .unwrap() - .at_uri_collection(), - Some("my.collection".into()) - ); - } -} diff --git a/links/src/record.rs b/links/src/record.rs deleted file mode 100644 index d83a171..0000000 --- a/links/src/record.rs +++ /dev/null @@ -1,163 +0,0 @@ -use tinyjson::JsonValue; - -use crate::{parse_any_link, CollectedLink}; - -pub fn walk_record(path: &str, v: &JsonValue, found: &mut Vec) { - match v { - JsonValue::Object(o) => { - for (key, child) in o { - walk_record(&format!("{path}.{key}"), child, found) - } - } - JsonValue::Array(a) => { - for child in a { - let child_p = match child { - JsonValue::Object(o) => { - if let Some(JsonValue::String(t)) = o.get("$type") { - format!("{path}[{t}]") - } else { - format!("{path}[]") - } - } - _ => format!("{path}[]"), - }; - walk_record(&child_p, child, found) - } - } - JsonValue::String(s) => { - if let Some(link) = parse_any_link(s) { - found.push(CollectedLink { - path: path.to_string(), - target: link, - }); - } - } - _ => {} - } -} - -pub fn collect_links(v: &JsonValue) -> Vec { - let mut found = vec![]; - walk_record("", v, &mut found); - found -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::Link; - - fn l(path: &str, target: Link) -> CollectedLink { - CollectedLink { - path: path.into(), - target, - } - } - - #[test] - fn test_collect_links() { - let rec = r#"{"a": "https://example.com", "b": "not a link"}"#; - let json = collect_links(&rec.parse().unwrap()); - assert_eq!(json, vec![l(".a", Link::Uri("https://example.com".into()))]); - } - - #[test] - fn test_bsky_feed_post_record_reply() { - let rec = r#"{ - "$type": "app.bsky.feed.post", - "createdAt": "2025-01-08T20:52:43.041Z", - "langs": [ - "en" - ], - "reply": { - "parent": { - "cid": "bafyreifk3bwnmulk37ezrarg4ouheqnhgucypynftqafl4limssogvzk6i", - "uri": "at://did:plc:b3rzzkblqsxhr3dgcueymkqe/app.bsky.feed.post/3lf6yc4drhk2f" - }, - "root": { - "cid": "bafyreifk3bwnmulk37ezrarg4ouheqnhgucypynftqafl4limssogvzk6i", - "uri": "at://did:plc:b3rzzkblqsxhr3dgcueymkqe/app.bsky.feed.post/3lf6yc4drhk2f" - } - }, - "text": "Yup!" - }"#; - let mut json = collect_links(&rec.parse().unwrap()); - json.sort_by_key(|c| (c.path.clone(), c.target.clone())); - assert_eq!( - json, - vec![ - l( - ".reply.parent.uri", - Link::AtUri( - "at://did:plc:b3rzzkblqsxhr3dgcueymkqe/app.bsky.feed.post/3lf6yc4drhk2f" - .into() - ) - ), - l( - ".reply.root.uri", - Link::AtUri( - "at://did:plc:b3rzzkblqsxhr3dgcueymkqe/app.bsky.feed.post/3lf6yc4drhk2f" - .into() - ) - ), - ] - ) - } - - #[test] - fn test_bsky_feed_post_record_embed() { - let rec = r#"{ - "$type": "app.bsky.feed.post", - "createdAt": "2025-01-08T20:52:39.539Z", - "embed": { - "$type": "app.bsky.embed.external", - "external": { - "description": "YouTube video by More Perfect Union", - "thumb": { - "$type": "blob", - "ref": { - "$link": "bafkreifxuvkbqksq5usi4cryex37o4absjexuouvgenlb62ojsx443b2tm" - }, - "mimeType": "image/jpeg", - "size": 477460 - }, - "title": "Corporations & Wealthy Elites Are Coopting Our Government. Who Can Stop Them?", - "uri": "https://youtu.be/oKXm4szEP1Q?si=_0n_uPu4qNKokMnq" - } - }, - "facets": [ - { - "features": [ - { - "$type": "app.bsky.richtext.facet#link", - "uri": "https://youtu.be/oKXm4szEP1Q?si=_0n_uPu4qNKokMnq" - } - ], - "index": { - "byteEnd": 24, - "byteStart": 0 - } - } - ], - "langs": [ - "en" - ], - "text": "youtu.be/oKXm4szEP1Q?..." - }"#; - let mut json = collect_links(&rec.parse().unwrap()); - json.sort_by_key(|c| (c.path.clone(), c.target.clone())); - assert_eq!( - json, - vec![ - l( - ".embed.external.uri", - Link::Uri("https://youtu.be/oKXm4szEP1Q?si=_0n_uPu4qNKokMnq".into()), - ), - l( - ".facets[].features[app.bsky.richtext.facet#link].uri", - Link::Uri("https://youtu.be/oKXm4szEP1Q?si=_0n_uPu4qNKokMnq".into()), - ), - ] - ) - } -} diff --git a/readme.md b/readme.md index 045652b..a158cd7 100644 --- a/readme.md +++ b/readme.md @@ -1,95 +1,6 @@ -microcosm HTTP APIs + rust crates -================================= -[![@microcosm.blue: bluesky](https://img.shields.io/badge/@microcosm.blue-bluesky-blue)](https://bsky.app/profile/microcosm.blue) -[![microcosm discord: join](https://img.shields.io/badge/microcosm_discord-join-purple)](https://discord.gg/tcDfe4PGVB) -[![github sponsors: support](https://img.shields.io/badge/github_sponsors-support-pink)](https://github.com/sponsors/uniphil/) -[![ko-fi: support](https://img.shields.io/badge/ko--fi-support-pink)](https://ko-fi.com/bad_example) +Moved +===== -Welcome! Documentation is under active development. If you like reading API docs, you'll probably hit the ground running! +This repo is now under the [@microcosm.blue](https://tangled.sh/@microcosm.blue) org account: https://tangled.sh/@microcosm.blue/microcosm-rs -Tutorials, how-to guides, and client SDK libraries are all in the works for gentler on-ramps, but are not quite ready yet. But don't let that stop you! Hop in the [microcosm discord](https://discord.gg/tcDfe4PGVB), or post questions and tag [@bad-example.com](https://bsky.app/profile/bad-example.com) on Bluesky if you get stuck anywhere. - -> [!tip] -> This repository's primary home is moving to tangled: [@microcosm.blue/microcosm-rs](https://tangled.sh/@microcosm.blue/microcosm-rs). It will continue to be mirrored on [github](https://github.com/at-microcosm/microcosm-rs) for the forseeable future, and it's fine to open issues or pulls in either place! - - -🌌 [Constellation](./constellation/) ------------------------------------- - -A global atproto interactions backlink index as a simple JSON API. Works with every lexicon, runs on a raspberry pi, consumes less than 2GiB of disk per day. Handles record deletion, account de/re-activation, and account deletion, ensuring accurate link counts while respecting users' data choices. - -- Source: [./constellation/](./constellation/) -- [Public instance/API docs](https://constellation.microcosm.blue/) -- Status: used in production. APIs will change but backwards compatibility will be maintained as long as needed. - - -🎇 [Spacedust](./spacedust/) ----------------------------- - -A global atproto interactions firehose. Extracts all at-uris, DIDs, and URLs from every lexicon in the firehose, and exposes them over a websocket modelled after [jetstream](github.com/bluesky-social/jetstream). - -- Source: [./spacedust/](./spacedust/) -- [Public instance/API docs](https://spacedust.microcosm.blue/) -- Status: v0: the basics work and the APIs are in place! missing cursor replay, forward link storage, and delete event link hydration. - -### Demos: - -- [Spacedust notifications](https://notifications.microcosm.blue/): web push notifications for _every_ atproto app -- [Zero-Bluesky real-time interaction-updating post embed](https://bsky.bad-example.com/zero-bluesky-realtime-embed/) - - -🛰️ [Slingshot](./slingshot) ---------------------------- - -A fast, eager, production-grade edge cache for atproto records and identities. Pre-caches all records from the firehose and maintains a longer-term cache of requested records on disk. - -- Source: [./slingshot/](./slingshot/) -- [Public instance/API docs](https://slingshot.microcosm.blue/) -- Status: v0: most XRPC APIs are working. cache storage is being reworked. - - -🛸 [UFOs API](./ufos) ---------------------- - -Timeseries stats and sample records for every [collection](https://atproto.com/guides/glossary#collection) ever seen in the atproto firehose. Unique users are counted in hyperloglog sketches enabling arbitrary cardinality aggregation across time buckets and/or NSIDs. - -- Source: [./ufos/](./ufos/) -- [Public instance/API docs](https://ufos-api.microcosm.blue/) -- Status: Used in production. It has APIs and they work! Needs improvement on indexing; needs more indexes and some more APIs to the data exposed. - -> [!tip] -> See also: [UFOs atproto explorer](https://ufos.microcosm.blue/) built on UFOs API. ([source](github.com/at-microcosm/spacedust-utils)) - - -💫 [Links](./links) -------------------- - -Rust library for parsing and extracting links (at-uris, DIDs, and URLs) from atproto records. - -- Source: [./links/](./links/) -- Status: not yet published to crates.io; needs some rework - - -🛩️ [Jetstream](./jetstream) ---------------------------- - -A low-overhead jetstream client with cursor handling and automatic reconnect. - -- Source: [./links/](./links/) -- Status: used in multiple apps in production, but not yet published to crates.io; some rework planned - -> [!tip] -> See also: [Rocketman](https://github.com/teal-fm/cadet/tree/main/rocketman), another excellent rust jetstream client which shares some lineage and _is_ published on crates.io. - - - -🔭 Deprecated: [Who am I](./who-am-i) -------------------------------------- - -An identity bridge for microcosm demos, that kinda worked. Fixing its problems is about equivalent to reinventing a lot of OIDC, so it's being retired. - -- Source: [./who-am-i/](./who-am-i/) -- Status: ready for retirement. - -> [!warning] -> `who-am-i` is still in use for the Spacedust Notifications demo, but that will hopefully be migrated to use atproto oauth directly instead. +It should have been there from the start, sorry! diff --git a/rust-toolchain.toml b/rust-toolchain.toml deleted file mode 100644 index aa2092b..0000000 --- a/rust-toolchain.toml +++ /dev/null @@ -1,2 +0,0 @@ -[toolchain] -components = [ "rustfmt", "clippy" ] diff --git a/slingshot/.gitignore b/slingshot/.gitignore deleted file mode 100644 index 01c6be7..0000000 --- a/slingshot/.gitignore +++ /dev/null @@ -1 +0,0 @@ -foyer diff --git a/slingshot/Cargo.toml b/slingshot/Cargo.toml deleted file mode 100644 index 06a97a5..0000000 --- a/slingshot/Cargo.toml +++ /dev/null @@ -1,31 +0,0 @@ -[package] -name = "slingshot" -version = "0.1.0" -edition = "2024" - -[dependencies] -atrium-api = { version = "0.25.4", default-features = false } -atrium-common = "0.1.2" -atrium-identity = "0.1.5" -atrium-oauth = "0.1.3" -clap = { version = "4.5.41", features = ["derive"] } -ctrlc = "3.4.7" -foyer = { version = "0.18.0", features = ["serde"] } -hickory-resolver = "0.25.2" -jetstream = { path = "../jetstream", features = ["metrics"] } -links = { path = "../links" } -log = "0.4.27" -metrics = "0.24.2" -metrics-exporter-prometheus = { version = "0.17.1", features = ["http-listener"] } -poem = { version = "3.1.12", features = ["acme", "static-files"] } -poem-openapi = { version = "5.1.16", features = ["scalar"] } -reqwest = { version = "0.12.22", features = ["json"] } -rustls = "0.23.31" -serde = { version = "1.0.219", features = ["derive"] } -serde_json = { version = "1.0.141", features = ["raw_value"] } -thiserror = "2.0.12" -time = { version = "0.3.41", features = ["serde"] } -tokio = { version = "1.47.0", features = ["full"] } -tokio-util = "0.7.15" -tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } -url = "2.5.4" diff --git a/slingshot/api-description.md b/slingshot/api-description.md deleted file mode 100644 index e029693..0000000 --- a/slingshot/api-description.md +++ /dev/null @@ -1,93 +0,0 @@ -_A [gravitational slingshot](https://en.wikipedia.org/wiki/Gravity_assist) makes use of the gravity and relative movements of celestial bodies to accelerate a spacecraft and change its trajectory._ - - -# Slingshot: edge record cache - -Applications in [ATProtocol](https://atproto.com/) store data in users' own [PDS](https://atproto.com/guides/self-hosting) (Personal Data Server), which are distributed across thousands of independently-run servers all over the world. Trying to access this data poses challenges for client applications: - -- A PDS might be far away with long network latency -- or may be on an unreliable connection -- or overloaded when you need it, or offline, or… - -Large projects like [Bluesky](https://bsky.app/) control their performance and reliability by syncing all app-relevant data from PDSs into first-party databases. But for new apps, building out this additional data infrastructure adds significant effort and complexity up front. - -**Slingshot is a fast, eager, production-grade cache of data in the [ATmosphere](https://atproto.com/)**, offering performance and reliability without custom infrastructure. - - -### Current status - -> [!important] -> Slingshot is currently in a **v0, pre-release state**. There is one production instance and you can use it! Expect short downtimes for restarts as development progresses and lower cache hit-rates as the internal storage caches are adjusted and reset. - -The core APIs will not change, since they are standard third-party `com.atproto` query APIs from ATProtocol. - - -## Eager caching - -In many cases, Slingshot can cache the data you need *before* first request! - -Slingshot subscribes to the global [Firehose](https://atproto.com/specs/sync#firehose) of data updates. It keeps a short-term rolling indexed window of *all* data, and automatically promotes content likely to be requested to its longer-term main cache. _(automatic promotion is still a work in progress)_ - -When there is a cache miss, Slingshot can often still accelerate record fetching, since it keeps a large cache of resolved identities: it can usually request from the correct PDS without extra lookups. - - -## Precise invalidation - -The fireshose includes **update** and **delete** events, which Slingshot uses to ensure stale and deleted data is removed within a very short window. Additonally, identity and account-level events can trigger rapid cleanup of data for deactivated and deleted accounts. _(some of this is still a work in progress)_ - - -## Low-trust - -The "AT" in ATProtocol [stands for _Authenticated Transfer_](https://atproto.com/guides/glossary#at-protocol): all data is cryptographically signed, which makes it possible to broadcast data through third parties and trust that it's real _without_ having to directly contact the originating server. - -Two core standard query APIs are supported to balance convenience and trust. They both fetch [records](https://atproto.com/guides/glossary#record): - -### [`com.atproto.repo.getRecord`](#tag/comatproto-queries/get/xrpc/com.atproto.repo.getRecord) - -- convenient `JSON` response format -- cannot be proven authentic - -### [`com.atproto.sync.getRecord`](#tag/comatproto-queries/get/xrpc/com.atproto.sync.getRecord) - -- [`DAG-CBOR`](https://atproto.com/specs/data-model)-encoded response requires extra libraries to decode, but -- includes a cryptographic proof of authenticity! - -_(work on this endpoint is in progress)_ - - -## Service proxying - -Clients can proxy atproto queries through their own PDS with [Service Proxying](https://atproto.com/specs/xrpc#service-proxying), and this is supported by Slingshot. The Slingshot instance must be started the `--domain` argument specified. - -Service-proxied requests can specify a Slingshot instance via the `atproto-proxy` header: - -```http -GET /xrpc/com.bad-example.identity.resolveMiniDoc?identifier=bad-example.com -Host: -atproto-proxy: did:web:#slingshot -``` - -Where `` is the user's own PDS host, and `` is the domain that the slingshot instance is deployed at (eg. `slingshot.microcosm.blue`). See the [Service Proxying](https://atproto.com/specs/xrpc#service-proxying) docs for more. - -> [!tip] -> Service proxying is supported but completely optional. All APIs are directly accessible over the public internet, and GeoDNS helps route users to the closest instance to them for the lowest possible latency. (_note: deploying multiple slingshot instances with GeoDNS is still TODO_) - - -## Ergonomic APIs - -- Slingshot also offers variants of the `getRecord` endpoints that accept a full `at-uri` as a parameter, to save clients from needing to parse and validate all parts of a record location. - -- Bi-directionally verifying identity endpoints, so you can directly exchange atproto [`handle`](https://atproto.com/guides/glossary#handle)s for [`DID`](https://atproto.com/guides/glossary#did-decentralized-id)s without extra steps, plus a convenient [Mini-Doc](#tag/slingshot-specific-queries/get/xrpc/com.bad-example.identity.resolveMiniDoc) verified identity summary. - - -## Part of microcosm - -[Microcosm](https://www.microcosm.blue/) is a collection of services and independent community-run infrastructure for ATProtocol. - -Slingshot excels when combined with _shallow indexing_ services, which offer fast queries of global data relationships but with only references to the data records. Microcosm has a few! - -- [🌌 Constellation](https://constellation.microcosm.blue/), a global backlink index (all social interactions in atproto are links!) -- [🎇 Spacedust](https://spacedust.microcosm.blue/), a firehose of all social interactions - -> [!success] -> All microcosm projects are [open source](https://tangled.sh/@bad-example.com/microcosm-links). **You can help sustain Slingshot** and all of microcosm by becoming a [Github sponsor](https://github.com/sponsors/uniphil/) or a [Ko-fi supporter](https://ko-fi.com/bad_example)! diff --git a/slingshot/readme.md b/slingshot/readme.md deleted file mode 100644 index beb4de8..0000000 --- a/slingshot/readme.md +++ /dev/null @@ -1,7 +0,0 @@ -# slingshot: atproto record edge cache - -local dev running: - -```bash -RUST_LOG=info,slingshot=trace ulimit -n 4096 && RUST_LOG=info cargo run -- --jetstream us-east-1 --cache-dir ./foyer -``` diff --git a/slingshot/src/consumer.rs b/slingshot/src/consumer.rs deleted file mode 100644 index ee270d2..0000000 --- a/slingshot/src/consumer.rs +++ /dev/null @@ -1,80 +0,0 @@ -use crate::CachedRecord; -use crate::error::ConsumerError; -use foyer::HybridCache; -use jetstream::{ - DefaultJetstreamEndpoints, JetstreamCompression, JetstreamConfig, JetstreamConnector, - events::{CommitOp, Cursor, EventKind}, -}; -use tokio_util::sync::CancellationToken; - -pub async fn consume( - jetstream_endpoint: String, - cursor: Option, - no_zstd: bool, - shutdown: CancellationToken, - cache: HybridCache, -) -> Result<(), ConsumerError> { - let endpoint = DefaultJetstreamEndpoints::endpoint_or_shortcut(&jetstream_endpoint); - if endpoint == jetstream_endpoint { - log::info!("consumer: connecting jetstream at {endpoint}"); - } else { - log::info!("consumer: connecting jetstream at {jetstream_endpoint} => {endpoint}"); - } - let config: JetstreamConfig = JetstreamConfig { - endpoint, - compression: if no_zstd { - JetstreamCompression::None - } else { - JetstreamCompression::Zstd - }, - replay_on_reconnect: true, - channel_size: 1024, // buffer up to ~1s of jetstream events - ..Default::default() - }; - let mut receiver = JetstreamConnector::new(config)? - .connect_cursor(cursor) - .await?; - - log::info!("consumer: receiving messages.."); - loop { - if shutdown.is_cancelled() { - log::info!("consumer: exiting for shutdown"); - return Ok(()); - } - let Some(mut event) = receiver.recv().await else { - log::error!("consumer: could not receive event, bailing"); - break; - }; - - if event.kind != EventKind::Commit { - continue; - } - let Some(ref mut commit) = event.commit else { - log::warn!("consumer: commit event missing commit data, ignoring"); - continue; - }; - - // TODO: something a bit more robust - let at_uri = format!( - "at://{}/{}/{}", - &*event.did, &*commit.collection, &*commit.rkey - ); - - if commit.operation == CommitOp::Delete { - cache.insert(at_uri, CachedRecord::Deleted); - } else { - let Some(record) = commit.record.take() else { - log::warn!("consumer: commit insert or update missing record, ignoring"); - continue; - }; - let Some(cid) = commit.cid.take() else { - log::warn!("consumer: commit insert or update missing CID, ignoring"); - continue; - }; - - cache.insert(at_uri, CachedRecord::Found((cid, record).into())); - } - } - - Err(ConsumerError::JetstreamEnded) -} diff --git a/slingshot/src/error.rs b/slingshot/src/error.rs deleted file mode 100644 index 67b3e07..0000000 --- a/slingshot/src/error.rs +++ /dev/null @@ -1,93 +0,0 @@ -use crate::ErrorResponseObject; -use thiserror::Error; - -#[derive(Debug, Error)] -pub enum ConsumerError { - #[error(transparent)] - JetstreamConnectionError(#[from] jetstream::error::ConnectionError), - #[error(transparent)] - JetstreamConfigValidationError(#[from] jetstream::error::ConfigValidationError), - #[error("jetstream ended")] - JetstreamEnded, - #[error("delay queue output dropped")] - DelayQueueOutputDropped, -} - -#[derive(Debug, Error)] -pub enum ServerError { - #[error("server build error: {0}")] - AcmeBuildError(std::io::Error), - #[error("server exited: {0}")] - ServerExited(std::io::Error), -} - -#[derive(Debug, Error)] -pub enum IdentityError { - #[error("whatever: {0}")] - WhateverError(String), - #[error("bad DID: {0}")] - BadDid(&'static str), - #[error("identity types got mixed up: {0}")] - IdentityValTypeMixup(String), - #[error("foyer error: {0}")] - FoyerError(#[from] foyer::Error), - - #[error("failed to resolve: {0}")] - ResolutionFailed(#[from] atrium_identity::Error), - // #[error("identity resolved but no handle found for user")] - // NoHandle, - #[error("found handle {0:?} but it appears invalid: {1}")] - InvalidHandle(String, &'static str), - - #[error("could not convert atrium did doc to partial mini doc: {0}")] - BadDidDoc(String), - - #[error("wrong key for clearing refresh queue: {0}")] - RefreshQueueKeyError(&'static str), -} - -#[derive(Debug, Error)] -pub enum HealthCheckError { - #[error("failed to send checkin: {0}")] - HealthCheckError(#[from] reqwest::Error), -} - -#[derive(Debug, Error)] -pub enum MainTaskError { - #[error(transparent)] - ConsumerTaskError(#[from] ConsumerError), - #[error(transparent)] - ServerTaskError(#[from] ServerError), - #[error(transparent)] - IdentityTaskError(#[from] IdentityError), - #[error(transparent)] - HealthCheckError(#[from] HealthCheckError), - #[error("firehose cache failed to close: {0}")] - FirehoseCacheCloseError(foyer::Error), -} - -#[derive(Debug, Error)] -pub enum RecordError { - #[error("identity error: {0}")] - IdentityError(#[from] IdentityError), - #[error("repo could not be validated as either a DID or an atproto handle")] - BadRepo, - #[error("could not get record: {0}")] - NotFound(&'static str), - #[error("could nto parse pds url: {0}")] - UrlParseError(#[from] url::ParseError), - #[error("reqwest send failed: {0}")] - SendError(reqwest::Error), - #[error("reqwest raised for status: {0}")] - StatusError(reqwest::Error), - #[error("reqwest failed to parse json: {0}")] - ParseJsonError(reqwest::Error), - #[error("upstream getRecord did not include a CID")] - MissingUpstreamCid, - #[error("upstream CID was not valid: {0}")] - BadUpstreamCid(String), - #[error("upstream atproto-looking bad request")] - UpstreamBadRequest(ErrorResponseObject), - #[error("upstream non-atproto bad request")] - UpstreamBadBadNotGoodRequest(reqwest::Error), -} diff --git a/slingshot/src/firehose_cache.rs b/slingshot/src/firehose_cache.rs deleted file mode 100644 index 1ff33da..0000000 --- a/slingshot/src/firehose_cache.rs +++ /dev/null @@ -1,22 +0,0 @@ -use crate::CachedRecord; -use foyer::{DirectFsDeviceOptions, Engine, HybridCache, HybridCacheBuilder}; -use std::path::Path; - -pub async fn firehose_cache( - cache_dir: impl AsRef, -) -> Result, String> { - let cache = HybridCacheBuilder::new() - .with_name("firehose") - .memory(64 * 2_usize.pow(20)) - .with_weighter(|k: &String, v| k.len() + std::mem::size_of_val(v)) - .storage(Engine::large()) - .with_device_options( - DirectFsDeviceOptions::new(cache_dir) - .with_capacity(2_usize.pow(30)) // TODO: configurable (1GB to have something) - .with_file_size(16 * 2_usize.pow(20)), // note: this does limit the max cached item size, warning jumbo records - ) - .build() - .await - .map_err(|e| format!("foyer setup error: {e:?}"))?; - Ok(cache) -} diff --git a/slingshot/src/healthcheck.rs b/slingshot/src/healthcheck.rs deleted file mode 100644 index fd60758..0000000 --- a/slingshot/src/healthcheck.rs +++ /dev/null @@ -1,32 +0,0 @@ -use crate::error::HealthCheckError; -use reqwest::Client; -use std::time::Duration; -use tokio::time::sleep; -use tokio_util::sync::CancellationToken; - -pub async fn healthcheck( - endpoint: String, - shutdown: CancellationToken, -) -> Result<(), HealthCheckError> { - let client = Client::builder() - .user_agent(format!( - "microcosm slingshot v{} (dev: @bad-example.com)", - env!("CARGO_PKG_VERSION") - )) - .no_proxy() - .timeout(Duration::from_secs(10)) - .build()?; - - loop { - tokio::select! { - res = client.get(&endpoint).send() => { - let _ = res - .and_then(|r| r.error_for_status()) - .inspect_err(|e| log::error!("failed to send healthcheck: {e}")); - }, - _ = shutdown.cancelled() => break, - } - sleep(Duration::from_secs(51)).await; - } - Ok(()) -} diff --git a/slingshot/src/identity.rs b/slingshot/src/identity.rs deleted file mode 100644 index 1abd0d0..0000000 --- a/slingshot/src/identity.rs +++ /dev/null @@ -1,524 +0,0 @@ -use hickory_resolver::{ResolveError, TokioResolver}; -use std::collections::{HashSet, VecDeque}; -use std::path::Path; -use std::sync::Arc; -/// for now we're gonna just keep doing more cache -/// -/// plc.director x foyer, ttl kept with data, refresh deferred to background on fetch -/// -/// things we need: -/// -/// 1. handle -> DID resolution: getRecord must accept a handle for `repo` param -/// 2. DID -> PDS resolution: so we know where to getRecord -/// 3. DID -> handle resolution: for bidirectional handle validation and in case we want to offer this -use std::time::Duration; -use tokio::sync::Mutex; -use tokio_util::sync::CancellationToken; - -use crate::error::IdentityError; -use atrium_api::{ - did_doc::DidDocument, - types::string::{Did, Handle}, -}; -use atrium_common::resolver::Resolver; -use atrium_identity::{ - did::{CommonDidResolver, CommonDidResolverConfig, DEFAULT_PLC_DIRECTORY_URL}, - handle::{AtprotoHandleResolver, AtprotoHandleResolverConfig, DnsTxtResolver}, -}; -use atrium_oauth::DefaultHttpClient; // it's probably not worth bringing all of atrium_oauth for this but -use foyer::{DirectFsDeviceOptions, Engine, HybridCache, HybridCacheBuilder}; -use serde::{Deserialize, Serialize}; -use time::UtcDateTime; - -/// once we have something resolved, don't re-resolve until after this period -const MIN_TTL: Duration = Duration::from_secs(4 * 3600); // probably shoudl have a max ttl -const MIN_NOT_FOUND_TTL: Duration = Duration::from_secs(60); - -#[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] -enum IdentityKey { - Handle(Handle), - Did(Did), -} - -#[derive(Debug, Serialize, Deserialize)] -struct IdentityVal(UtcDateTime, IdentityData); - -#[derive(Debug, Serialize, Deserialize)] -enum IdentityData { - NotFound, - Did(Did), - Doc(PartialMiniDoc), -} - -/// partial representation of a com.bad-example.identity mini atproto doc -/// -/// partial because the handle is not verified -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PartialMiniDoc { - /// an atproto handle (**unverified**) - /// - /// the first valid atproto handle from the did doc's aka - pub unverified_handle: Handle, - /// the did's atproto pds url (TODO: type this?) - /// - /// note: atrium *does* actually parse it into a URI, it just doesn't return - /// that for some reason - pub pds: String, - /// for now we're just pulling this straight from the did doc - /// - /// would be nice to type and validate it - /// - /// this is the publicKeyMultibase from the did doc. - /// legacy key encoding not supported. - /// `id`, `type`, and `controller` must be checked, but aren't stored. - pub signing_key: String, -} - -impl TryFrom for PartialMiniDoc { - type Error = String; - fn try_from(did_doc: DidDocument) -> Result { - // must use the first valid handle - let mut unverified_handle = None; - let Some(ref doc_akas) = did_doc.also_known_as else { - return Err("did doc missing `also_known_as`".to_string()); - }; - for aka in doc_akas { - let Some(maybe_handle) = aka.strip_prefix("at://") else { - continue; - }; - let Ok(valid_handle) = Handle::new(maybe_handle.to_string()) else { - continue; - }; - unverified_handle = Some(valid_handle); - break; - } - let Some(unverified_handle) = unverified_handle else { - return Err("no valid atproto handles in `also_known_as`".to_string()); - }; - - // atrium seems to get service endpoint getters - let Some(pds) = did_doc.get_pds_endpoint() else { - return Err("no valid pds service found".to_string()); - }; - - // TODO can't use atrium's get_signing_key() becuase it fails to check type and controller - // so if we check those and reject it, we might miss a later valid key in the array - // (todo is to fix atrium) - // actually: atrium might be flexible for legacy reps. for now we're rejecting legacy rep. - - // must use the first valid signing key - let mut signing_key = None; - let Some(verification_methods) = did_doc.verification_method else { - return Err("no verification methods found".to_string()); - }; - for method in verification_methods { - if method.id != format!("{}#atproto", did_doc.id) { - continue; - } - if method.r#type != "Multikey" { - continue; - } - if method.controller != did_doc.id { - continue; - } - let Some(key) = method.public_key_multibase else { - continue; - }; - signing_key = Some(key); - break; - } - let Some(signing_key) = signing_key else { - return Err("no valid atproto signing key found in verification methods".to_string()); - }; - - Ok(PartialMiniDoc { - unverified_handle, - pds, - signing_key, - }) - } -} - -/// multi-producer *single-consumer* queue structures (wrap in arc-mutex plz) -/// -/// the hashset allows testing for presense of items in the queue. -/// this has absolutely no support for multiple queue consumers. -#[derive(Debug, Default)] -struct RefreshQueue { - queue: VecDeque, - items: HashSet, -} - -#[derive(Clone)] -pub struct Identity { - handle_resolver: Arc>, - did_resolver: Arc>, - cache: HybridCache, - /// multi-producer *single consumer* queue - refresh_queue: Arc>, - /// just a lock to ensure only one refresher (queue consumer) is running (to be improved with a better refresher) - refresher: Arc>, -} - -impl Identity { - pub async fn new(cache_dir: impl AsRef) -> Result { - let http_client = Arc::new(DefaultHttpClient::default()); - let handle_resolver = AtprotoHandleResolver::new(AtprotoHandleResolverConfig { - dns_txt_resolver: HickoryDnsTxtResolver::new().unwrap(), - http_client: http_client.clone(), - }); - let did_resolver = CommonDidResolver::new(CommonDidResolverConfig { - plc_directory_url: DEFAULT_PLC_DIRECTORY_URL.to_string(), - http_client: http_client.clone(), - }); - - let cache = HybridCacheBuilder::new() - .with_name("identity") - .memory(16 * 2_usize.pow(20)) - .with_weighter(|k, v| std::mem::size_of_val(k) + std::mem::size_of_val(v)) - .storage(Engine::small()) - .with_device_options( - DirectFsDeviceOptions::new(cache_dir) - .with_capacity(2_usize.pow(30)) // TODO: configurable (1GB to have something) - .with_file_size(2_usize.pow(20)), // note: this does limit the max cached item size, warning jumbo records - ) - .build() - .await?; - - Ok(Self { - handle_resolver: Arc::new(handle_resolver), - did_resolver: Arc::new(did_resolver), - cache, - refresh_queue: Default::default(), - refresher: Default::default(), - }) - } - - /// Resolve (and verify!) an atproto handle to a DID - /// - /// The result can be stale - /// - /// `None` if the handle can't be found or verification fails - pub async fn handle_to_did(&self, handle: Handle) -> Result, IdentityError> { - let Some(did) = self.handle_to_unverified_did(&handle).await? else { - return Ok(None); - }; - let Some(doc) = self.did_to_partial_mini_doc(&did).await? else { - return Ok(None); - }; - if doc.unverified_handle != handle { - return Ok(None); - } - Ok(Some(did)) - } - - /// Resolve a DID to a pds url - /// - /// This *also* incidentally resolves and verifies the handle, which might - /// make it slower than expected - pub async fn did_to_pds(&self, did: Did) -> Result, IdentityError> { - let Some(mini_doc) = self.did_to_partial_mini_doc(&did).await? else { - return Ok(None); - }; - Ok(Some(mini_doc.pds)) - } - - /// Resolve (and cache but **not verify**) a handle to a DID - async fn handle_to_unverified_did( - &self, - handle: &Handle, - ) -> Result, IdentityError> { - let key = IdentityKey::Handle(handle.clone()); - let entry = self - .cache - .fetch(key.clone(), { - let handle = handle.clone(); - let resolver = self.handle_resolver.clone(); - || async move { - match resolver.resolve(&handle).await { - Ok(did) => Ok(IdentityVal(UtcDateTime::now(), IdentityData::Did(did))), - Err(atrium_identity::Error::NotFound) => { - Ok(IdentityVal(UtcDateTime::now(), IdentityData::NotFound)) - } - Err(other) => Err(foyer::Error::Other(Box::new( - IdentityError::ResolutionFailed(other), - ))), - } - } - }) - .await?; - - let now = UtcDateTime::now(); - let IdentityVal(last_fetch, data) = entry.value(); - match data { - IdentityData::Doc(_) => { - log::error!("identity value mixup: got a doc from a handle key (should be a did)"); - Err(IdentityError::IdentityValTypeMixup(handle.to_string())) - } - IdentityData::NotFound => { - if (now - *last_fetch) >= MIN_NOT_FOUND_TTL { - self.queue_refresh(key).await; - } - Ok(None) - } - IdentityData::Did(did) => { - if (now - *last_fetch) >= MIN_TTL { - self.queue_refresh(key).await; - } - Ok(Some(did.clone())) - } - } - } - - /// Fetch (and cache) a partial mini doc from a did - pub async fn did_to_partial_mini_doc( - &self, - did: &Did, - ) -> Result, IdentityError> { - let key = IdentityKey::Did(did.clone()); - let entry = self - .cache - .fetch(key.clone(), { - let did = did.clone(); - let resolver = self.did_resolver.clone(); - || async move { - match resolver.resolve(&did).await { - Ok(did_doc) => { - // TODO: fix in atrium: should verify id is did - if did_doc.id != did.to_string() { - return Err(foyer::Error::other(Box::new( - IdentityError::BadDidDoc( - "did doc's id did not match did".to_string(), - ), - ))); - } - let mini_doc = did_doc.try_into().map_err(|e| { - foyer::Error::Other(Box::new(IdentityError::BadDidDoc(e))) - })?; - Ok(IdentityVal(UtcDateTime::now(), IdentityData::Doc(mini_doc))) - } - Err(atrium_identity::Error::NotFound) => { - Ok(IdentityVal(UtcDateTime::now(), IdentityData::NotFound)) - } - Err(other) => Err(foyer::Error::Other(Box::new( - IdentityError::ResolutionFailed(other), - ))), - } - } - }) - .await?; - - let now = UtcDateTime::now(); - let IdentityVal(last_fetch, data) = entry.value(); - match data { - IdentityData::Did(_) => { - log::error!("identity value mixup: got a did from a did key (should be a doc)"); - Err(IdentityError::IdentityValTypeMixup(did.to_string())) - } - IdentityData::NotFound => { - if (now - *last_fetch) >= MIN_NOT_FOUND_TTL { - self.queue_refresh(key).await; - } - Ok(None) - } - IdentityData::Doc(mini_did) => { - if (now - *last_fetch) >= MIN_TTL { - self.queue_refresh(key).await; - } - Ok(Some(mini_did.clone())) - } - } - } - - /// put a refresh task on the queue - /// - /// this can be safely called from multiple concurrent tasks - async fn queue_refresh(&self, key: IdentityKey) { - // todo: max queue size - let mut q = self.refresh_queue.lock().await; - if !q.items.contains(&key) { - q.items.insert(key.clone()); - q.queue.push_back(key); - } - } - - /// find out what's next in the queue. concurrent consumers are not allowed. - /// - /// intent is to leave the item in the queue while refreshing, so that a - /// producer will not re-add it if it's in progress. there's definitely - /// better ways to do this, but this is ~simple for as far as a single - /// consumer can take us. - /// - /// we could take it from the queue but leave it in the set and remove from - /// set later, but splitting them apart feels more bug-prone. - async fn peek_refresh(&self) -> Option { - let q = self.refresh_queue.lock().await; - q.queue.front().cloned() - } - - /// call to clear the latest key from the refresh queue. concurrent consumers not allowed. - /// - /// must provide the last peeked refresh queue item as a small safety check - async fn complete_refresh(&self, key: &IdentityKey) -> Result<(), IdentityError> { - let mut q = self.refresh_queue.lock().await; - - let Some(queue_key) = q.queue.pop_front() else { - // gone from queue + since we're in an error condition, make sure it's not stuck in items - // (not toctou because we have the lock) - // bolder here than below and removing from items because if the queue is *empty*, then we - // know it hasn't been re-added since losing sync. - if q.items.remove(key) { - log::error!("identity refresh: queue de-sync: not in "); - } else { - log::warn!( - "identity refresh: tried to complete with wrong key. are multiple queue consumers running?" - ); - } - return Err(IdentityError::RefreshQueueKeyError("no key in queue")); - }; - - if queue_key != *key { - // extra weird case here, what's the most defensive behaviour? - // we have two keys: ours should have been first but isn't. this shouldn't happen, so let's - // just leave items alone for it. risks unbounded growth but we're in a bad place already. - // the other key is the one we just popped. we didn't want it, so maybe we should put it - // back, BUT if we somehow ended up with concurrent consumers, we have bigger problems. take - // responsibility for taking it instead: remove it from items as well, and just drop it. - // - // hope that whoever calls us takes this error seriously. - if q.items.remove(&queue_key) { - log::warn!( - "identity refresh: queue de-sync + dropping a bystander key without refreshing it!" - ); - } else { - // you thought things couldn't get weirder? (i mean hopefully they can't) - log::error!("identity refresh: queue de-sync + bystander key also de-sync!?"); - } - return Err(IdentityError::RefreshQueueKeyError( - "wrong key at front of queue", - )); - } - - if q.items.remove(key) { - Ok(()) - } else { - log::error!("identity refresh: queue de-sync: key not in items"); - Err(IdentityError::RefreshQueueKeyError("key not in items")) - } - } - - /// run the refresh queue consumer - pub async fn run_refresher(&self, shutdown: CancellationToken) -> Result<(), IdentityError> { - let _guard = self - .refresher - .try_lock() - .expect("there to only be one refresher running"); - loop { - if shutdown.is_cancelled() { - log::info!("identity refresher: exiting for shutdown: closing cache..."); - if let Err(e) = self.cache.close().await { - log::error!("cache close errored: {e}"); - } else { - log::info!("identity cache closed.") - } - return Ok(()); - } - let Some(task_key) = self.peek_refresh().await else { - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - continue; - }; - match task_key { - IdentityKey::Handle(ref handle) => { - log::trace!("refreshing handle {handle:?}"); - match self.handle_resolver.resolve(handle).await { - Ok(did) => { - self.cache.insert( - task_key.clone(), - IdentityVal(UtcDateTime::now(), IdentityData::Did(did)), - ); - } - Err(atrium_identity::Error::NotFound) => { - self.cache.insert( - task_key.clone(), - IdentityVal(UtcDateTime::now(), IdentityData::NotFound), - ); - } - Err(err) => { - log::warn!( - "failed to refresh handle: {err:?}. leaving stale (should we eventually do something?)" - ); - } - } - self.complete_refresh(&task_key).await?; // failures are bugs, so break loop - } - IdentityKey::Did(ref did) => { - log::trace!("refreshing did doc: {did:?}"); - - match self.did_resolver.resolve(did).await { - Ok(did_doc) => { - // TODO: fix in atrium: should verify id is did - if did_doc.id != did.to_string() { - log::warn!( - "refreshed did doc failed: wrong did doc id. dropping refresh." - ); - continue; - } - let mini_doc = match did_doc.try_into() { - Ok(md) => md, - Err(e) => { - log::warn!( - "converting mini doc failed: {e:?}. dropping refresh." - ); - continue; - } - }; - self.cache.insert( - task_key.clone(), - IdentityVal(UtcDateTime::now(), IdentityData::Doc(mini_doc)), - ); - } - Err(atrium_identity::Error::NotFound) => { - self.cache.insert( - task_key.clone(), - IdentityVal(UtcDateTime::now(), IdentityData::NotFound), - ); - } - Err(err) => { - log::warn!( - "failed to refresh did doc: {err:?}. leaving stale (should we eventually do something?)" - ); - } - } - - self.complete_refresh(&task_key).await?; // failures are bugs, so break loop - } - } - } - } -} - -pub struct HickoryDnsTxtResolver(TokioResolver); - -impl HickoryDnsTxtResolver { - fn new() -> Result { - Ok(Self(TokioResolver::builder_tokio()?.build())) - } -} - -impl DnsTxtResolver for HickoryDnsTxtResolver { - async fn resolve( - &self, - query: &str, - ) -> core::result::Result, Box> { - match self.0.txt_lookup(query).await { - Ok(r) => { - metrics::counter!("whoami_resolve_dns_txt", "success" => "true").increment(1); - Ok(r.iter().map(|r| r.to_string()).collect()) - } - Err(e) => { - metrics::counter!("whoami_resolve_dns_txt", "success" => "false").increment(1); - Err(e.into()) - } - } - } -} diff --git a/slingshot/src/lib.rs b/slingshot/src/lib.rs deleted file mode 100644 index 7737374..0000000 --- a/slingshot/src/lib.rs +++ /dev/null @@ -1,14 +0,0 @@ -mod consumer; -pub mod error; -mod firehose_cache; -mod healthcheck; -mod identity; -mod record; -mod server; - -pub use consumer::consume; -pub use firehose_cache::firehose_cache; -pub use healthcheck::healthcheck; -pub use identity::Identity; -pub use record::{CachedRecord, ErrorResponseObject, Repo}; -pub use server::serve; diff --git a/slingshot/src/main.rs b/slingshot/src/main.rs deleted file mode 100644 index ae5fdbb..0000000 --- a/slingshot/src/main.rs +++ /dev/null @@ -1,194 +0,0 @@ -// use foyer::HybridCache; -// use foyer::{Engine, DirectFsDeviceOptions, HybridCacheBuilder}; -use metrics_exporter_prometheus::PrometheusBuilder; -use slingshot::{ - Identity, Repo, consume, error::MainTaskError, firehose_cache, healthcheck, serve, -}; -use std::path::PathBuf; - -use clap::Parser; -use tokio_util::sync::CancellationToken; - -/// Slingshot record edge cache -#[derive(Parser, Debug, Clone)] -#[command(version, about, long_about = None)] -struct Args { - /// Jetstream server to connect to (exclusive with --fixture). Provide either a wss:// URL, or a shorhand value: - /// 'us-east-1', 'us-east-2', 'us-west-1', or 'us-west-2' - #[arg(long)] - jetstream: String, - /// don't request zstd-compressed jetstream events - /// - /// reduces CPU at the expense of more ingress bandwidth - #[arg(long, action)] - jetstream_no_zstd: bool, - /// where to keep disk caches - #[arg(long)] - cache_dir: PathBuf, - /// the domain pointing to this server - /// - /// if present: - /// - a did:web document will be served at /.well-known/did.json - /// - an HTTPS certs will be automatically configured with Acme/letsencrypt - /// - TODO: a rate-limiter will be installed - #[arg(long)] - domain: Option, - /// email address for letsencrypt contact - /// - /// recommended in production, i guess? - #[arg(long)] - acme_contact: Option, - /// a location to cache acme https certs - /// - /// only used if --host is specified. omitting requires re-requesting certs - /// on every restart, and letsencrypt has rate limits that are easy to hit. - /// - /// recommended in production, but mind the file permissions. - #[arg(long)] - certs: Option, - /// an web address to send healtcheck pings to every ~51s or so - #[arg(long)] - healthcheck: Option, -} - -#[tokio::main] -async fn main() -> Result<(), String> { - tracing_subscriber::fmt::init(); - - let shutdown = CancellationToken::new(); - - let ctrlc_shutdown = shutdown.clone(); - ctrlc::set_handler(move || ctrlc_shutdown.cancel()).expect("failed to set ctrl-c handler"); - - let args = Args::parse(); - - if let Err(e) = install_metrics_server() { - log::error!("failed to install metrics server: {e:?}"); - } else { - log::info!("metrics listening at http://0.0.0.0:8765"); - } - - std::fs::create_dir_all(&args.cache_dir).map_err(|e| { - format!( - "failed to ensure cache parent dir: {e:?} (dir: {:?})", - args.cache_dir - ) - })?; - let cache_dir = args.cache_dir.canonicalize().map_err(|e| { - format!( - "failed to canonicalize cache_dir: {e:?} (dir: {:?})", - args.cache_dir - ) - })?; - log::info!("cache dir ready at at {cache_dir:?}."); - - log::info!("setting up firehose cache..."); - let cache = firehose_cache(cache_dir.join("./firehose")).await?; - log::info!("firehose cache ready."); - - let mut tasks: tokio::task::JoinSet> = tokio::task::JoinSet::new(); - - log::info!("starting identity service..."); - let identity = Identity::new(cache_dir.join("./identity")) - .await - .map_err(|e| format!("identity setup failed: {e:?}"))?; - log::info!("identity service ready."); - let identity_refresher = identity.clone(); - let identity_shutdown = shutdown.clone(); - tasks.spawn(async move { - identity_refresher.run_refresher(identity_shutdown).await?; - Ok(()) - }); - - let repo = Repo::new(identity.clone()); - - let server_shutdown = shutdown.clone(); - let server_cache_handle = cache.clone(); - tasks.spawn(async move { - serve( - server_cache_handle, - identity, - repo, - args.domain, - args.acme_contact, - args.certs, - server_shutdown, - ) - .await?; - Ok(()) - }); - - let consumer_shutdown = shutdown.clone(); - let consumer_cache = cache.clone(); - tasks.spawn(async move { - consume( - args.jetstream, - None, - args.jetstream_no_zstd, - consumer_shutdown, - consumer_cache, - ) - .await?; - Ok(()) - }); - - if let Some(hc) = args.healthcheck { - let healthcheck_shutdown = shutdown.clone(); - tasks.spawn(async move { - healthcheck(hc, healthcheck_shutdown).await?; - Ok(()) - }); - } - - tokio::select! { - _ = shutdown.cancelled() => log::warn!("shutdown requested"), - Some(r) = tasks.join_next() => { - log::warn!("a task exited, shutting down: {r:?}"); - shutdown.cancel(); - } - } - - tasks.spawn(async move { - cache - .close() - .await - .map_err(MainTaskError::FirehoseCacheCloseError) - }); - - tokio::select! { - _ = async { - while let Some(completed) = tasks.join_next().await { - log::info!("shutdown: task completed: {completed:?}"); - } - } => {}, - _ = tokio::time::sleep(std::time::Duration::from_secs(30)) => { - log::info!("shutdown: not all tasks completed on time. aborting..."); - tasks.shutdown().await; - }, - } - - log::info!("bye!"); - - Ok(()) -} - -fn install_metrics_server() -> Result<(), metrics_exporter_prometheus::BuildError> { - log::info!("installing metrics server..."); - let host = [0, 0, 0, 0]; - let port = 8765; - PrometheusBuilder::new() - .set_quantiles(&[0.5, 0.9, 0.99, 1.0])? - .set_bucket_duration(std::time::Duration::from_secs(300))? - .set_bucket_count(std::num::NonZero::new(12).unwrap()) // count * duration = 60 mins. stuff doesn't happen that fast here. - .set_enable_unit_suffix(false) // this seemed buggy for constellation (sometimes wouldn't engage) - .with_http_listener((host, port)) - .install()?; - log::info!( - "metrics server installed! listening on http://{}.{}.{}.{}:{port}", - host[0], - host[1], - host[2], - host[3] - ); - Ok(()) -} diff --git a/slingshot/src/record.rs b/slingshot/src/record.rs deleted file mode 100644 index 971c648..0000000 --- a/slingshot/src/record.rs +++ /dev/null @@ -1,155 +0,0 @@ -//! cached record storage - -use crate::{Identity, error::RecordError}; -use atrium_api::types::string::{Cid, Did, Nsid, RecordKey}; -use reqwest::{Client, StatusCode}; -use serde::{Deserialize, Serialize}; -use serde_json::value::RawValue; -use std::str::FromStr; -use std::time::Duration; -use url::Url; - -#[derive(Debug, Serialize, Deserialize)] -pub struct RawRecord { - cid: Cid, - record: String, -} - -// TODO: should be able to do typed CID -impl From<(Cid, Box)> for RawRecord { - fn from((cid, rv): (Cid, Box)) -> Self { - Self { - cid, - record: rv.get().to_string(), - } - } -} - -/// only for use with stored (validated) values, not general strings -impl From<&RawRecord> for (Cid, Box) { - fn from(RawRecord { cid, record }: &RawRecord) -> Self { - ( - cid.clone(), - RawValue::from_string(record.to_string()) - .expect("stored string from RawValue to be valid"), - ) - } -} - -#[derive(Debug, Serialize, Deserialize)] -pub enum CachedRecord { - Found(RawRecord), - Deleted, -} - -//////// upstream record fetching - -#[derive(Deserialize)] -struct RecordResponseObject { - #[allow(dead_code)] // expect it to be there but we ignore it - uri: String, - /// CID for this exact version of the record - /// - /// this is optional in the spec and that's potentially TODO for slingshot - cid: Option, - /// the record itself as JSON - value: Box, -} - -#[derive(Debug, Deserialize)] -pub struct ErrorResponseObject { - pub error: String, - pub message: String, -} - -#[derive(Clone)] -pub struct Repo { - identity: Identity, - client: Client, -} - -impl Repo { - pub fn new(identity: Identity) -> Self { - let client = Client::builder() - .user_agent(format!( - "microcosm slingshot v{} (dev: @bad-example.com)", - env!("CARGO_PKG_VERSION") - )) - .no_proxy() - .timeout(Duration::from_secs(10)) - .build() - .unwrap(); - Repo { identity, client } - } - - pub async fn get_record( - &self, - did: &Did, - collection: &Nsid, - rkey: &RecordKey, - cid: &Option, - ) -> Result { - let Some(pds) = self.identity.did_to_pds(did.clone()).await? else { - return Err(RecordError::NotFound("could not get pds for DID")); - }; - - // cid gets set to None for a retry, if it's Some and we got NotFound - let mut cid = cid; - - let res = loop { - // TODO: throttle outgoing requests by host probably, generally guard against outgoing requests - let mut params = vec![ - ("repo", did.to_string()), - ("collection", collection.to_string()), - ("rkey", rkey.to_string()), - ]; - if let Some(cid) = cid { - params.push(("cid", cid.as_ref().to_string())); - } - let mut url = Url::parse_with_params(&pds, ¶ms)?; - url.set_path("/xrpc/com.atproto.repo.getRecord"); - - let res = self - .client - .get(url.clone()) - .send() - .await - .map_err(RecordError::SendError)?; - - if res.status() == StatusCode::BAD_REQUEST { - // 1. if we're not able to parse json, it's not something we can handle - let err = res - .json::() - .await - .map_err(RecordError::UpstreamBadBadNotGoodRequest)?; - // 2. if we are, is it a NotFound? and if so, did we try with a CID? - // if so, retry with no CID (api handler will reject for mismatch but - // with a nice error + warm cache) - if err.error == "NotFound" && cid.is_some() { - cid = &None; - continue; - } else { - return Err(RecordError::UpstreamBadRequest(err)); - } - } - break res; - }; - - let data = res - .error_for_status() - .map_err(RecordError::StatusError)? // TODO atproto error handling (think about handling not found) - .json::() - .await - .map_err(RecordError::ParseJsonError)?; // todo... - - let Some(cid) = data.cid else { - return Err(RecordError::MissingUpstreamCid); - }; - let cid = Cid::from_str(&cid).map_err(|e| RecordError::BadUpstreamCid(e.to_string()))?; - - Ok(CachedRecord::Found(RawRecord { - cid, - record: data.value.to_string(), - })) - } -} diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs deleted file mode 100644 index 9066da1..0000000 --- a/slingshot/src/server.rs +++ /dev/null @@ -1,769 +0,0 @@ -use crate::{ - CachedRecord, ErrorResponseObject, Identity, Repo, - error::{RecordError, ServerError}, -}; -use atrium_api::types::string::{Cid, Did, Handle, Nsid, RecordKey}; -use foyer::HybridCache; -use links::at_uri::parse_at_uri as normalize_at_uri; -use serde::Serialize; -use std::path::PathBuf; -use std::str::FromStr; -use std::sync::Arc; -use tokio_util::sync::CancellationToken; - -use poem::{ - Endpoint, EndpointExt, Route, Server, - endpoint::{StaticFileEndpoint, make_sync}, - http::Method, - listener::{ - Listener, TcpListener, - acme::{AutoCert, LETS_ENCRYPT_PRODUCTION}, - }, - middleware::{CatchPanic, Cors, Tracing}, -}; -use poem_openapi::{ - ApiResponse, ContactObject, ExternalDocumentObject, Object, OpenApi, OpenApiService, Tags, - param::Query, payload::Json, types::Example, -}; - -fn example_handle() -> String { - "bad-example.com".to_string() -} -fn example_did() -> String { - "did:plc:hdhoaan3xa3jiuq4fg4mefid".to_string() -} -fn example_collection() -> String { - "app.bsky.feed.like".to_string() -} -fn example_rkey() -> String { - "3lv4ouczo2b2a".to_string() -} -fn example_uri() -> String { - format!( - "at://{}/{}/{}", - example_did(), - example_collection(), - example_rkey() - ) -} -fn example_pds() -> String { - "https://porcini.us-east.host.bsky.network".to_string() -} -fn example_signing_key() -> String { - "zQ3shpq1g134o7HGDb86CtQFxnHqzx5pZWknrVX2Waum3fF6j".to_string() -} - -#[derive(Object)] -#[oai(example = true)] -struct XrpcErrorResponseObject { - /// Should correspond an error `name` in the lexicon errors array - error: String, - /// Human-readable description and possibly additonal context - message: String, -} -impl Example for XrpcErrorResponseObject { - fn example() -> Self { - Self { - error: "RecordNotFound".to_string(), - message: "This record was deleted".to_string(), - } - } -} -type XrpcError = Json; -fn xrpc_error(error: impl AsRef, message: impl AsRef) -> XrpcError { - Json(XrpcErrorResponseObject { - error: error.as_ref().to_string(), - message: message.as_ref().to_string(), - }) -} - -fn bad_request_handler_get_record(err: poem::Error) -> GetRecordResponse { - GetRecordResponse::BadRequest(Json(XrpcErrorResponseObject { - error: "InvalidRequest".to_string(), - message: format!("Bad request, here's some info that maybe should not be exposed: {err}"), - })) -} - -fn bad_request_handler_resolve_mini(err: poem::Error) -> ResolveMiniIDResponse { - ResolveMiniIDResponse::BadRequest(Json(XrpcErrorResponseObject { - error: "InvalidRequest".to_string(), - message: format!("Bad request, here's some info that maybe should not be exposed: {err}"), - })) -} - -fn bad_request_handler_resolve_handle(err: poem::Error) -> JustDidResponse { - JustDidResponse::BadRequest(Json(XrpcErrorResponseObject { - error: "InvalidRequest".to_string(), - message: format!("Bad request, here's some info that maybe should not be exposed: {err}"), - })) -} - -#[derive(Object)] -#[oai(example = true)] -struct FoundRecordResponseObject { - /// at-uri for this record - uri: String, - /// CID for this exact version of the record - /// - /// Slingshot will always return the CID, despite it not being a required - /// response property in the official lexicon. - /// - /// TODO: probably actually let it be optional, idk are some pds's weirdly - /// not returning it? - cid: Option, - /// the record itself as JSON - value: serde_json::Value, -} -impl Example for FoundRecordResponseObject { - fn example() -> Self { - Self { - uri: example_uri(), - cid: Some("bafyreialv3mzvvxaoyrfrwoer3xmabbmdchvrbyhayd7bga47qjbycy74e".to_string()), - value: serde_json::json!({ - "$type": "app.bsky.feed.like", - "createdAt": "2025-07-29T18:02:02.327Z", - "subject": { - "cid": "bafyreia2gy6eyk5qfetgahvshpq35vtbwy6negpy3gnuulcdi723mi7vxy", - "uri": "at://did:plc:vwzwgnygau7ed7b7wt5ux7y2/app.bsky.feed.post/3lv4lkb4vgs2k" - } - }), - } - } -} - -#[derive(ApiResponse)] -#[oai(bad_request_handler = "bad_request_handler_get_record")] -enum GetRecordResponse { - /// Record found - #[oai(status = 200)] - Ok(Json), - /// Bad request or no record to return - /// - /// The only error name in the repo.getRecord lexicon is `RecordNotFound`, - /// but the [canonical api docs](https://docs.bsky.app/docs/api/com-atproto-repo-get-record) - /// also list `InvalidRequest`, `ExpiredToken`, and `InvalidToken`. Of - /// these, slingshot will only generate `RecordNotFound` or `InvalidRequest`, - /// but may return any proxied error code from the upstream repo. - #[oai(status = 400)] - BadRequest(XrpcError), - /// Server errors - #[oai(status = 500)] - ServerError(XrpcError), -} - -#[derive(Object)] -#[oai(example = true)] -struct MiniDocResponseObject { - /// DID, bi-directionally verified if a handle was provided in the query. - did: String, - /// The validated handle of the account or `handle.invalid` if the handle - /// did not bi-directionally match the DID document. - handle: String, - /// The identity's PDS URL - pds: String, - /// The atproto signing key publicKeyMultibase - /// - /// Legacy key encoding not supported. the key is returned directly; `id`, - /// `type`, and `controller` are omitted. - signing_key: String, -} -impl Example for MiniDocResponseObject { - fn example() -> Self { - Self { - did: example_did(), - handle: example_handle(), - pds: example_pds(), - signing_key: example_signing_key(), - } - } -} - -#[derive(ApiResponse)] -#[oai(bad_request_handler = "bad_request_handler_resolve_mini")] -enum ResolveMiniIDResponse { - /// Identity resolved - #[oai(status = 200)] - Ok(Json), - /// Bad request or identity not resolved - #[oai(status = 400)] - BadRequest(XrpcError), -} - -#[derive(Object)] -#[oai(example = true)] -struct FoundDidResponseObject { - /// the DID, bi-directionally verified if using Slingshot - did: String, -} -impl Example for FoundDidResponseObject { - fn example() -> Self { - Self { did: example_did() } - } -} - -#[derive(ApiResponse)] -#[oai(bad_request_handler = "bad_request_handler_resolve_handle")] -enum JustDidResponse { - /// Resolution succeeded - #[oai(status = 200)] - Ok(Json), - /// Bad request, failed to resolve, or failed to verify - /// - /// `error` will be one of `InvalidRequest`, `HandleNotFound`. - #[oai(status = 400)] - BadRequest(XrpcError), - /// Something went wrong trying to complete the request - #[oai(status = 500)] - ServerError(XrpcError), -} - -struct Xrpc { - cache: HybridCache, - identity: Identity, - repo: Arc, -} - -#[derive(Tags)] -enum ApiTags { - /// Core ATProtocol-compatible APIs. - /// - /// > [!tip] - /// > Upstream documentation is available at - /// > https://docs.bsky.app/docs/category/http-reference - /// - /// These queries are usually executed directly against the PDS containing - /// the data being requested. Slingshot offers a caching view of the same - /// contents with better expected performance and reliability. - #[oai(rename = "com.atproto.* queries")] - ComAtproto, - /// Additional and improved APIs. - /// - /// These APIs offer small tweaks to the core ATProtocol APIs, with more - /// more convenient [request parameters](#tag/slingshot-specific-queries/GET/xrpc/com.bad-example.repo.getUriRecord) - /// or [response formats](#tag/slingshot-specific-queries/GET/xrpc/com.bad-example.identity.resolveMiniDoc). - /// - /// > [!important] - /// > At the moment, these are namespaced under the `com.bad-example.*` NSID - /// > prefix, but as they stabilize they may be migrated to an org namespace - /// > like `blue.microcosm.*`. Support for asliasing to `com.bad-example.*` - /// > will be maintained as long as it's in use. - #[oai(rename = "slingshot-specific queries")] - Custom, -} - -#[OpenApi] -impl Xrpc { - /// com.atproto.repo.getRecord - /// - /// Get a single record from a repository. Does not require auth. - /// - /// > [!tip] - /// > See also the [canonical `com.atproto` XRPC documentation](https://docs.bsky.app/docs/api/com-atproto-repo-get-record) - /// > that this endpoint aims to be compatible with. - #[oai( - path = "/com.atproto.repo.getRecord", - method = "get", - tag = "ApiTags::ComAtproto" - )] - async fn get_record( - &self, - /// The DID or handle of the repo - #[oai(example = "example_did")] - Query(repo): Query, - /// The NSID of the record collection - #[oai(example = "example_collection")] - Query(collection): Query, - /// The Record key - #[oai(example = "example_rkey")] - Query(rkey): Query, - /// Optional: the CID of the version of the record. - /// - /// If not specified, then return the most recent version. - /// - /// If a stale `CID` is specified and a newer version of the record - /// exists, Slingshot returns a `NotFound` error. That is: Slingshot - /// only retains the most recent version of a record. - Query(cid): Query>, - ) -> GetRecordResponse { - self.get_record_impl(repo, collection, rkey, cid).await - } - - /// com.bad-example.repo.getUriRecord - /// - /// Ergonomic complement to [`com.atproto.repo.getRecord`](https://docs.bsky.app/docs/api/com-atproto-repo-get-record) - /// which accepts an `at-uri` instead of individual repo/collection/rkey params - #[oai( - path = "/com.bad-example.repo.getUriRecord", - method = "get", - tag = "ApiTags::Custom" - )] - async fn get_uri_record( - &self, - /// The at-uri of the record - /// - /// The identifier can be a DID or an atproto handle, and the collection - /// and rkey segments must be present. - #[oai(example = "example_uri")] - Query(at_uri): Query, - /// Optional: the CID of the version of the record. - /// - /// If not specified, then return the most recent version. - /// - /// > [!tip] - /// > If specified and a newer version of the record exists, returns 404 not - /// > found. That is: slingshot only retains the most recent version of a - /// > record. - Query(cid): Query>, - ) -> GetRecordResponse { - let bad_at_uri = || { - GetRecordResponse::BadRequest(xrpc_error( - "InvalidRequest", - "at-uri does not appear to be valid", - )) - }; - - let Some(normalized) = normalize_at_uri(&at_uri) else { - return bad_at_uri(); - }; - - // TODO: move this to links - let Some(rest) = normalized.strip_prefix("at://") else { - return bad_at_uri(); - }; - let Some((repo, rest)) = rest.split_once('/') else { - return bad_at_uri(); - }; - let Some((collection, rest)) = rest.split_once('/') else { - return bad_at_uri(); - }; - let rkey = if let Some((rkey, _rest)) = rest.split_once('?') { - rkey - } else { - rest - }; - - self.get_record_impl( - repo.to_string(), - collection.to_string(), - rkey.to_string(), - cid, - ) - .await - } - - /// com.atproto.identity.resolveHandle - /// - /// Resolves an atproto [`handle`](https://atproto.com/guides/glossary#handle) - /// (hostname) to a [`DID`](https://atproto.com/guides/glossary#did-decentralized-id). - /// - /// > [!tip] - /// > Compatibility note: Slingshot will **always bi-directionally verify - /// > against the DID document**, which is optional according to the - /// > authoritative lexicon. - /// - /// > [!tip] - /// > See the [canonical `com.atproto` XRPC documentation](https://docs.bsky.app/docs/api/com-atproto-identity-resolve-handle) - /// > that this endpoint aims to be compatible with. - #[oai( - path = "/com.atproto.identity.resolveHandle", - method = "get", - tag = "ApiTags::ComAtproto" - )] - async fn resolve_handle( - &self, - /// The handle to resolve. - #[oai(example = "example_handle")] - Query(handle): Query, - ) -> JustDidResponse { - let Ok(handle) = Handle::new(handle) else { - return JustDidResponse::BadRequest(xrpc_error("InvalidRequest", "not a valid handle")); - }; - - let Ok(alleged_did) = self.identity.handle_to_did(handle.clone()).await else { - return JustDidResponse::ServerError(xrpc_error("Failed", "Could not resolve handle")); - }; - - let Some(alleged_did) = alleged_did else { - return JustDidResponse::BadRequest(xrpc_error( - "HandleNotFound", - "Could not resolve handle to a DID", - )); - }; - - let Ok(partial_doc) = self.identity.did_to_partial_mini_doc(&alleged_did).await else { - return JustDidResponse::ServerError(xrpc_error("Failed", "Could not fetch DID doc")); - }; - - let Some(partial_doc) = partial_doc else { - return JustDidResponse::BadRequest(xrpc_error( - "HandleNotFound", - "Resolved handle but could not find DID doc for the DID", - )); - }; - - if partial_doc.unverified_handle != handle { - return JustDidResponse::BadRequest(xrpc_error( - "HandleNotFound", - "Resolved handle failed bi-directional validation", - )); - } - - JustDidResponse::Ok(Json(FoundDidResponseObject { - did: alleged_did.to_string(), - })) - } - - /// com.bad-example.identity.resolveMiniDoc - /// - /// Like [com.atproto.identity.resolveIdentity](https://docs.bsky.app/docs/api/com-atproto-identity-resolve-identity) - /// but instead of the full `didDoc` it returns an atproto-relevant subset. - #[oai( - path = "/com.bad-example.identity.resolveMiniDoc", - method = "get", - tag = "ApiTags::Custom" - )] - async fn resolve_mini_id( - &self, - /// Handle or DID to resolve - #[oai(example = "example_handle")] - Query(identifier): Query, - ) -> ResolveMiniIDResponse { - let invalid = |reason: &'static str| { - ResolveMiniIDResponse::BadRequest(xrpc_error("InvalidRequest", reason)) - }; - - let mut unverified_handle = None; - let did = match Did::new(identifier.clone()) { - Ok(did) => did, - Err(_) => { - let Ok(alleged_handle) = Handle::new(identifier) else { - return invalid("identifier was not a valid DID or handle"); - }; - if let Ok(res) = self.identity.handle_to_did(alleged_handle.clone()).await { - if let Some(did) = res { - // we did it joe - unverified_handle = Some(alleged_handle); - did - } else { - return invalid("Could not resolve handle identifier to a DID"); - } - } else { - // TODO: ServerError not BadRequest - return invalid("errored while trying to resolve handle to DID"); - } - } - }; - let Ok(partial_doc) = self.identity.did_to_partial_mini_doc(&did).await else { - return invalid("failed to get DID doc"); - }; - let Some(partial_doc) = partial_doc else { - return invalid("failed to find DID doc"); - }; - - // ok so here's where we're at: - // ✅ we have a DID - // ✅ we have a partial doc - // 🔶 if we have a handle, it's from the `identifier` (user-input) - // -> then we just need to compare to the partial doc to confirm - // -> else we need to resolve the DID doc's to a handle and check - let handle = if let Some(h) = unverified_handle { - if h == partial_doc.unverified_handle { - h.to_string() - } else { - "handle.invalid".to_string() - } - } else { - let Ok(handle_did) = self - .identity - .handle_to_did(partial_doc.unverified_handle.clone()) - .await - else { - return invalid("failed to get did doc's handle"); - }; - let Some(handle_did) = handle_did else { - return invalid("failed to resolve did doc's handle"); - }; - if handle_did == did { - partial_doc.unverified_handle.to_string() - } else { - "handle.invalid".to_string() - } - }; - - ResolveMiniIDResponse::Ok(Json(MiniDocResponseObject { - did: did.to_string(), - handle, - pds: partial_doc.pds, - signing_key: partial_doc.signing_key, - })) - } - - async fn get_record_impl( - &self, - repo: String, - collection: String, - rkey: String, - cid: Option, - ) -> GetRecordResponse { - let did = match Did::new(repo.clone()) { - Ok(did) => did, - Err(_) => { - let Ok(handle) = Handle::new(repo) else { - return GetRecordResponse::BadRequest(xrpc_error( - "InvalidRequest", - "repo was not a valid DID or handle", - )); - }; - if let Ok(res) = self.identity.handle_to_did(handle).await { - if let Some(did) = res { - did - } else { - return GetRecordResponse::BadRequest(xrpc_error( - "InvalidRequest", - "Could not resolve handle repo to a DID", - )); - } - } else { - return GetRecordResponse::ServerError(xrpc_error( - "ResolutionFailed", - "errored while trying to resolve handle to DID", - )); - } - } - }; - - let Ok(collection) = Nsid::new(collection) else { - return GetRecordResponse::BadRequest(xrpc_error( - "InvalidRequest", - "invalid NSID for collection", - )); - }; - - let Ok(rkey) = RecordKey::new(rkey) else { - return GetRecordResponse::BadRequest(xrpc_error("InvalidRequest", "invalid rkey")); - }; - - let cid: Option = if let Some(cid) = cid { - let Ok(cid) = Cid::from_str(&cid) else { - return GetRecordResponse::BadRequest(xrpc_error("InvalidRequest", "invalid CID")); - }; - Some(cid) - } else { - None - }; - - let at_uri = format!("at://{}/{}/{}", &*did, &*collection, &*rkey); - - let fr = self - .cache - .fetch(at_uri.clone(), { - let cid = cid.clone(); - let repo_api = self.repo.clone(); - || async move { - repo_api - .get_record(&did, &collection, &rkey, &cid) - .await - .map_err(|e| foyer::Error::Other(Box::new(e))) - } - }) - .await; - - let entry = match fr { - Ok(e) => e, - Err(foyer::Error::Other(e)) => { - let record_error = match e.downcast::() { - Ok(e) => e, - Err(e) => { - log::error!("error (foyer other) getting cache entry, {e:?}"); - return GetRecordResponse::ServerError(xrpc_error( - "ServerError", - "sorry, something went wrong", - )); - } - }; - let RecordError::UpstreamBadRequest(ErrorResponseObject { error, message }) = - *record_error - else { - log::error!("RecordError getting cache entry, {record_error:?}"); - return GetRecordResponse::ServerError(xrpc_error( - "ServerError", - "sorry, something went wrong", - )); - }; - - // all of the noise around here is so that we can ultimately reach this: - // upstream BadRequest extracted from the foyer result which we can proxy back - return GetRecordResponse::BadRequest(xrpc_error( - error, - format!("Upstream bad request: {message}"), - )); - } - Err(e) => { - log::error!("error (foyer) getting cache entry, {e:?}"); - return GetRecordResponse::ServerError(xrpc_error( - "ServerError", - "sorry, something went wrong", - )); - } - }; - - match *entry { - CachedRecord::Found(ref raw) => { - let (found_cid, raw_value) = raw.into(); - if cid.clone().map(|c| c != found_cid).unwrap_or(false) { - return GetRecordResponse::BadRequest(Json(XrpcErrorResponseObject { - error: "RecordNotFound".to_string(), - message: "A record was found but its CID did not match that requested" - .to_string(), - })); - } - // TODO: thank u stellz: https://gist.github.com/stella3d/51e679e55b264adff89d00a1e58d0272 - let value = - serde_json::from_str(raw_value.get()).expect("RawValue to be valid json"); - GetRecordResponse::Ok(Json(FoundRecordResponseObject { - uri: at_uri, - cid: Some(found_cid.as_ref().to_string()), - value, - })) - } - CachedRecord::Deleted => GetRecordResponse::BadRequest(Json(XrpcErrorResponseObject { - error: "RecordNotFound".to_string(), - message: "This record was deleted".to_string(), - })), - } - } - - // TODO - // #[oai(path = "/com.atproto.identity.resolveHandle", method = "get")] - // #[oai(path = "/com.atproto.identity.resolveDid", method = "get")] - // but these are both not specified to do bidirectional validation, which is what we want to offer - // com.atproto.identity.resolveIdentity seems right, but requires returning the full did-doc - // would be nice if there were two queries: - // did -> verified handle + pds url - // handle -> verified did + pds url - // - // we could do horrible things and implement resolveIdentity with only a stripped-down fake did doc - // but this will *definitely* cause problems because eg. we're not currently storing pubkeys and - // those are a little bit important -} - -#[derive(Debug, Clone, Serialize)] -#[serde(rename_all = "camelCase")] -struct AppViewService { - id: String, - r#type: String, - service_endpoint: String, -} -#[derive(Debug, Clone, Serialize)] -struct AppViewDoc { - id: String, - service: [AppViewService; 1], -} -/// Serve a did document for did:web for this to be an xrpc appview -/// -/// No slingshot endpoints currently require auth, so it's not necessary to do -/// service proxying, however clients may wish to: -/// -/// - PDS proxying offers a level of client IP anonymity from slingshot -/// - slingshot *may* implement more generous per-user rate-limits for proxied requests in the future -fn get_did_doc(domain: &str) -> impl Endpoint + use<> { - let doc = poem::web::Json(AppViewDoc { - id: format!("did:web:{domain}"), - service: [AppViewService { - id: "#slingshot".to_string(), - r#type: "SlingshotRecordProxy".to_string(), - service_endpoint: format!("https://{domain}"), - }], - }); - make_sync(move |_| doc.clone()) -} - -pub async fn serve( - cache: HybridCache, - identity: Identity, - repo: Repo, - domain: Option, - acme_contact: Option, - certs: Option, - shutdown: CancellationToken, -) -> Result<(), ServerError> { - let repo = Arc::new(repo); - let api_service = OpenApiService::new( - Xrpc { - cache, - identity, - repo, - }, - "Slingshot", - env!("CARGO_PKG_VERSION"), - ) - .server(if let Some(ref h) = domain { - format!("https://{h}") - } else { - "http://localhost:3000".to_string() - }) - .url_prefix("/xrpc") - .contact( - ContactObject::new() - .name("@microcosm.blue") - .url("https://bsky.app/profile/microcosm.blue"), - ) - .description(include_str!("../api-description.md")) - .external_document(ExternalDocumentObject::new( - "https://microcosm.blue/slingshot", - )); - - let mut app = Route::new() - .at("/", StaticFileEndpoint::new("./static/index.html")) - .nest("/openapi", api_service.spec_endpoint()) - .nest("/xrpc/", api_service); - - if let Some(domain) = domain { - rustls::crypto::aws_lc_rs::default_provider() - .install_default() - .expect("alskfjalksdjf"); - - app = app.at("/.well-known/did.json", get_did_doc(&domain)); - - let mut auto_cert = AutoCert::builder() - .directory_url(LETS_ENCRYPT_PRODUCTION) - .domain(&domain); - if let Some(contact) = acme_contact { - auto_cert = auto_cert.contact(contact); - } - if let Some(certs) = certs { - auto_cert = auto_cert.cache_path(certs); - } - let auto_cert = auto_cert.build().map_err(ServerError::AcmeBuildError)?; - - run( - TcpListener::bind("0.0.0.0:443").acme(auto_cert), - app, - shutdown, - ) - .await - } else { - run(TcpListener::bind("127.0.0.1:3000"), app, shutdown).await - } -} - -async fn run(listener: L, app: Route, shutdown: CancellationToken) -> Result<(), ServerError> -where - L: Listener + 'static, -{ - let app = app - .with( - Cors::new() - .allow_origin_regex("*") - .allow_methods([Method::GET]) - .allow_credentials(false), - ) - .with(CatchPanic::new()) - .with(Tracing); - Server::new(listener) - .name("slingshot") - .run_with_graceful_shutdown(app, shutdown.cancelled(), None) - .await - .map_err(ServerError::ServerExited) - .inspect(|()| log::info!("server ended. goodbye.")) -} diff --git a/slingshot/static/favicon.ico b/slingshot/static/favicon.ico deleted file mode 100644 index e69de29..0000000 diff --git a/slingshot/static/index.html b/slingshot/static/index.html deleted file mode 100644 index 74b0a3d..0000000 --- a/slingshot/static/index.html +++ /dev/null @@ -1,67 +0,0 @@ - - - - - Slingshot: atproto edge record cache - - - - - -
-

- TODO: thing -

- -
- - - - - - - - diff --git a/spacedust/Cargo.toml b/spacedust/Cargo.toml deleted file mode 100644 index d30a08c..0000000 --- a/spacedust/Cargo.toml +++ /dev/null @@ -1,29 +0,0 @@ -[package] -name = "spacedust" -version = "0.1.0" -edition = "2024" - -[dependencies] -async-trait = "0.1.88" -clap = { version = "4.5.40", features = ["derive"] } -ctrlc = "3.4.7" -dropshot = "0.16.2" -env_logger = "0.11.8" -futures = "0.3.31" -http = "1.3.1" -jetstream = { path = "../jetstream", features = ["metrics"] } -links = { path = "../links" } -log = "0.4.27" -metrics = "0.24.2" -metrics-exporter-prometheus = { version = "0.17.1", features = ["http-listener"] } -rand = "0.9.1" -schemars = "0.8.22" -semver = "1.0.26" -serde = { version = "1.0.219", features = ["derive"] } -serde_json = "1.0.140" -serde_qs = "1.0.0-rc.3" -thiserror = "2.0.12" -tinyjson = "2.5.1" -tokio = { version = "1.45.1", features = ["full"] } -tokio-tungstenite = "0.27.0" -tokio-util = "0.7.15" diff --git a/spacedust/src/consumer.rs b/spacedust/src/consumer.rs deleted file mode 100644 index fdad8ac..0000000 --- a/spacedust/src/consumer.rs +++ /dev/null @@ -1,116 +0,0 @@ -use crate::ClientMessage; -use crate::error::ConsumerError; -use crate::removable_delay_queue; -use jetstream::{ - DefaultJetstreamEndpoints, JetstreamCompression, JetstreamConfig, JetstreamConnector, - events::{CommitOp, Cursor, EventKind}, -}; -use links::collect_links; -use std::sync::Arc; -use tokio::sync::broadcast; -use tokio_util::sync::CancellationToken; - -const MAX_LINKS_PER_EVENT: usize = 100; - -pub async fn consume( - b: broadcast::Sender>, - d: removable_delay_queue::Input<(String, usize), Arc>, - jetstream_endpoint: String, - cursor: Option, - no_zstd: bool, - shutdown: CancellationToken, -) -> Result<(), ConsumerError> { - let endpoint = DefaultJetstreamEndpoints::endpoint_or_shortcut(&jetstream_endpoint); - if endpoint == jetstream_endpoint { - log::info!("consumer: connecting jetstream at {endpoint}"); - } else { - log::info!("consumer: connecting jetstream at {jetstream_endpoint} => {endpoint}"); - } - let config: JetstreamConfig = JetstreamConfig { - endpoint, - compression: if no_zstd { - JetstreamCompression::None - } else { - JetstreamCompression::Zstd - }, - replay_on_reconnect: true, - channel_size: 1024, // buffer up to ~1s of jetstream events - ..Default::default() - }; - let mut receiver = JetstreamConnector::new(config)? - .connect_cursor(cursor) - .await?; - - log::info!("consumer: receiving messages.."); - loop { - if shutdown.is_cancelled() { - log::info!("consumer: exiting for shutdown"); - return Ok(()); - } - let Some(event) = receiver.recv().await else { - log::error!("consumer: could not receive event, bailing"); - break; - }; - - if event.kind != EventKind::Commit { - continue; - } - let Some(ref commit) = event.commit else { - log::warn!("consumer: commit event missing commit data, ignoring"); - continue; - }; - - // TODO: something a bit more robust - let at_uri = format!( - "at://{}/{}/{}", - &*event.did, &*commit.collection, &*commit.rkey - ); - - // TODO: keep a buffer and remove quick deletes to debounce notifs - // for now we just drop all deletes eek - if commit.operation == CommitOp::Delete { - d.remove_range((at_uri.clone(), 0)..=(at_uri.clone(), MAX_LINKS_PER_EVENT)) - .await; - continue; - } - let Some(ref record) = commit.record else { - log::warn!("consumer: commit update/delete missing record, ignoring"); - continue; - }; - - let jv = match record.get().parse() { - Ok(v) => v, - Err(e) => { - log::warn!("consumer: record failed to parse, ignoring: {e}"); - continue; - } - }; - - for (i, link) in collect_links(&jv).into_iter().enumerate() { - if i >= MAX_LINKS_PER_EVENT { - // todo: indicate if the link limit was reached (-> links omitted) - log::warn!("consumer: event has too many links, ignoring the rest"); - metrics::counter!("consumer_dropped_links", "reason" => "too_many_links") - .increment(1); - break; - } - let client_message = match ClientMessage::new_link(link, &at_uri, commit) { - Ok(m) => m, - Err(e) => { - // TODO indicate to clients that a link has been dropped - log::warn!("consumer: failed to serialize link to json: {e:?}"); - metrics::counter!("consumer_dropped_links", "reason" => "failed_to_serialize") - .increment(1); - continue; - } - }; - let message = Arc::new(client_message); - let _ = b.send(message.clone()); // only errors if no subscribers are connected, which is just fine. - d.enqueue((at_uri.clone(), i), message) - .await - .map_err(|_| ConsumerError::DelayQueueOutputDropped)?; - } - } - - Err(ConsumerError::JetstreamEnded) -} diff --git a/spacedust/src/delay.rs b/spacedust/src/delay.rs deleted file mode 100644 index 94044df..0000000 --- a/spacedust/src/delay.rs +++ /dev/null @@ -1,22 +0,0 @@ -use crate::error::DelayError; -use crate::removable_delay_queue; -use tokio::sync::broadcast; -use tokio_util::sync::CancellationToken; - -pub async fn to_broadcast( - source: removable_delay_queue::Output<(String, usize), T>, - dest: broadcast::Sender, - shutdown: CancellationToken, -) -> Result<(), DelayError> { - loop { - tokio::select! { - ev = source.next() => match ev { - Some(event) => { - let _ = dest.send(event); // only errors of there are no listeners, but that's normal - }, - None => return Err(DelayError::DelayEnded), - }, - _ = shutdown.cancelled() => return Ok(()), - } - } -} diff --git a/spacedust/src/error.rs b/spacedust/src/error.rs deleted file mode 100644 index cbfe445..0000000 --- a/spacedust/src/error.rs +++ /dev/null @@ -1,55 +0,0 @@ -use thiserror::Error; - -#[derive(Debug, Error)] -pub enum MainTaskError { - #[error(transparent)] - ConsumerTaskError(#[from] ConsumerError), - #[error(transparent)] - ServerTaskError(#[from] ServerError), - #[error(transparent)] - DelayTaskError(#[from] DelayError), -} - -#[derive(Debug, Error)] -pub enum ConsumerError { - #[error(transparent)] - JetstreamConnectionError(#[from] jetstream::error::ConnectionError), - #[error(transparent)] - JetstreamConfigValidationError(#[from] jetstream::error::ConfigValidationError), - #[error("jetstream ended")] - JetstreamEnded, - #[error("delay queue output dropped")] - DelayQueueOutputDropped, -} - -#[derive(Debug, Error)] -pub enum SubscriberUpdateError { - #[error("failed to parse json for subscriber update: {0}")] - FailedToParseMessage(serde_json::Error), - #[error("more wantedSources were requested than allowed (max 1,000)")] - TooManySourcesWanted, - #[error("more wantedSubjectDids were requested than allowed (max 10,000)")] - TooManyDidsWanted, - #[error("more wantedSubjects were requested than allowed (max 50,000)")] - TooManySubjectsWanted, -} - -#[derive(Debug, Error)] -pub enum DelayError { - #[error("delay ended")] - DelayEnded, -} - -#[derive(Debug, Error)] -pub enum ServerError { - #[error("failed to configure server logger: {0}")] - ConfigLogError(std::io::Error), - #[error("failed to render json for openapi: {0}")] - OpenApiJsonFail(serde_json::Error), - #[error(transparent)] - FailedToBuildServer(#[from] dropshot::BuildError), - #[error("server exited: {0}")] - ServerExited(String), - #[error("server closed badly: {0}")] - BadClose(String), -} diff --git a/spacedust/src/lib.rs b/spacedust/src/lib.rs deleted file mode 100644 index 915f061..0000000 --- a/spacedust/src/lib.rs +++ /dev/null @@ -1,104 +0,0 @@ -pub mod consumer; -pub mod delay; -pub mod error; -pub mod removable_delay_queue; -pub mod server; -pub mod subscriber; - -use jetstream::events::CommitEvent; -use links::CollectedLink; -use serde::{Deserialize, Serialize}; -use server::MultiSubscribeQuery; -use tokio_tungstenite::tungstenite::Message; - -#[derive(Debug)] -pub struct FilterableProperties { - /// Full unmodified DID, at-uri, or url - pub subject: String, - /// User/identity DID. - /// - /// Will match both bare-DIDs and DIDs extracted from at-uris. - /// `None` for any URL. - pub subject_did: Option, - /// Link source -- collection NSID joined with `:` to the record property path. - pub source: String, -} - -/// A serialized message with filterable properties attached -#[derive(Debug)] -pub struct ClientMessage { - pub message: Message, // always Message::Text - pub properties: FilterableProperties, -} - -impl ClientMessage { - pub fn new_link( - link: CollectedLink, - at_uri: &str, - commit: &CommitEvent, - ) -> Result { - let subject_did = link.target.did(); - - let subject = link.target.into_string(); - - let undotted = link.path.strip_prefix('.').unwrap_or_else(|| { - eprintln!("link path did not have expected '.' prefix: {}", link.path); - "" - }); - let source = format!("{}:{undotted}", &*commit.collection); - - let client_link_event = ClientLinkEvent { - operation: "create", - source: source.clone(), - source_record: at_uri.to_string(), - source_rev: commit.rev.to_string(), - subject: subject.clone(), - }; - - let client_event = ClientEvent { - kind: "link", - origin: "live", // TODO: indicate when we're locally replaying jetstream on reconnect?? maybe not. - link: client_link_event, - }; - - let client_event_json = serde_json::to_string(&client_event)?; - - let message = Message::Text(client_event_json.into()); - - let properties = FilterableProperties { - subject, - subject_did, - source, - }; - - Ok(ClientMessage { - message, - properties, - }) - } -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "snake_case")] -pub struct ClientEvent { - kind: &'static str, // "link" - origin: &'static str, // "live", "replay", "backfill" - link: ClientLinkEvent, -} - -#[derive(Debug, Serialize)] -struct ClientLinkEvent { - operation: &'static str, // "create", "delete" (prob no update, though maybe for rev?) - source: String, - source_record: String, - source_rev: String, - subject: String, - // TODO: include the record too? would save clients a level of hydration - // ^^ no, not for now. until we backfill + support broader deletes at *least*. -} - -#[derive(Debug, Deserialize)] -#[serde(tag = "type", content = "payload", rename_all = "snake_case")] -pub enum SubscriberSourcedMessage { - OptionsUpdate(MultiSubscribeQuery), -} diff --git a/spacedust/src/main.rs b/spacedust/src/main.rs deleted file mode 100644 index 54db30e..0000000 --- a/spacedust/src/main.rs +++ /dev/null @@ -1,144 +0,0 @@ -use spacedust::consumer; -use spacedust::delay; -use spacedust::error::MainTaskError; -use spacedust::removable_delay_queue::removable_delay_queue; -use spacedust::server; - -use clap::Parser; -use metrics_exporter_prometheus::PrometheusBuilder; -use std::time::Duration; -use tokio::sync::broadcast; -use tokio_util::sync::CancellationToken; - -/// Aggregate links in the at-mosphere -#[derive(Parser, Debug, Clone)] -#[command(version, about, long_about = None)] -struct Args { - /// Jetstream server to connect to (exclusive with --fixture). Provide either a wss:// URL, or a shorhand value: - /// 'us-east-1', 'us-east-2', 'us-west-1', or 'us-west-2' - #[arg(long)] - jetstream: String, - /// don't request zstd-compressed jetstream events - /// - /// reduces CPU at the expense of more ingress bandwidth - #[arg(long, action)] - jetstream_no_zstd: bool, -} - -#[tokio::main] -async fn main() -> Result<(), String> { - env_logger::init(); - - // tokio broadcast keeps a single main output queue for all subscribers. - // each subscriber clones off a copy of an individual value for each recv. - // since there's no large per-client buffer, we can make this one kind of - // big and accommodate more slow/bursty clients. - // - // in fact, we *could* even keep lagging clients alive, inserting lag- - // indicating messages to their output.... but for now we'll drop them to - // avoid accumulating zombies. - // - // events on the channel are individual links as they are discovered. a link - // contains a source and a target. the target is an at-uri, so it's up to - // ~1KB max; source is a collection + link path, which can be more but in - // practice the whole link rarely approaches 1KB total. - // - // TODO: determine if a pathological case could blow this up (eg 1MB link - // paths + slow subscriber -> 16GiB queue) - let (b, _) = broadcast::channel(16_384); - let consumer_sender = b.clone(); - let (d, _) = broadcast::channel(16_384); - let consumer_delayed_sender = d.clone(); - - let delay = Duration::from_secs(21); - let (delay_queue_sender, delay_queue_receiver) = removable_delay_queue(delay); - - let shutdown = CancellationToken::new(); - - let ctrlc_shutdown = shutdown.clone(); - ctrlc::set_handler(move || ctrlc_shutdown.cancel()).expect("failed to set ctrl-c handler"); - - let args = Args::parse(); - - if let Err(e) = install_metrics_server() { - log::error!("failed to install metrics server: {e:?}"); - }; - - let mut tasks: tokio::task::JoinSet> = tokio::task::JoinSet::new(); - - let server_shutdown = shutdown.clone(); - tasks.spawn(async move { - server::serve(b, d, server_shutdown).await?; - Ok(()) - }); - - let consumer_shutdown = shutdown.clone(); - tasks.spawn(async move { - consumer::consume( - consumer_sender, - delay_queue_sender, - args.jetstream, - None, - args.jetstream_no_zstd, - consumer_shutdown, - ) - .await?; - Ok(()) - }); - - let delay_shutdown = shutdown.clone(); - tasks.spawn(async move { - delay::to_broadcast( - delay_queue_receiver, - consumer_delayed_sender, - delay_shutdown, - ) - .await?; - Ok(()) - }); - - tokio::select! { - _ = shutdown.cancelled() => log::warn!("shutdown requested"), - Some(r) = tasks.join_next() => { - log::warn!("a task exited, shutting down: {r:?}"); - shutdown.cancel(); - } - } - - tokio::select! { - _ = async { - while let Some(completed) = tasks.join_next().await { - log::info!("shutdown: task completed: {completed:?}"); - } - } => {}, - _ = tokio::time::sleep(std::time::Duration::from_secs(3)) => { - log::info!("shutdown: not all tasks completed on time. aborting..."); - tasks.shutdown().await; - }, - } - - log::info!("bye!"); - - Ok(()) -} - -fn install_metrics_server() -> Result<(), metrics_exporter_prometheus::BuildError> { - log::info!("installing metrics server..."); - let host = [0, 0, 0, 0]; - let port = 8765; - PrometheusBuilder::new() - .set_quantiles(&[0.5, 0.9, 0.99, 1.0])? - .set_bucket_duration(std::time::Duration::from_secs(300))? - .set_bucket_count(std::num::NonZero::new(12).unwrap()) // count * duration = 60 mins. stuff doesn't happen that fast here. - .set_enable_unit_suffix(false) // this seemed buggy for constellation (sometimes wouldn't engage) - .with_http_listener((host, port)) - .install()?; - log::info!( - "metrics server installed! listening on http://{}.{}.{}.{}:{port}", - host[0], - host[1], - host[2], - host[3] - ); - Ok(()) -} diff --git a/spacedust/src/removable_delay_queue.rs b/spacedust/src/removable_delay_queue.rs deleted file mode 100644 index f528677..0000000 --- a/spacedust/src/removable_delay_queue.rs +++ /dev/null @@ -1,125 +0,0 @@ -use std::collections::{BTreeMap, VecDeque}; -use std::ops::RangeBounds; -use std::sync::Arc; -use std::time::{Duration, Instant}; -use thiserror::Error; -use tokio::sync::Mutex; - -#[derive(Debug, Error)] -pub enum EnqueueError { - #[error("queue ouput dropped")] - OutputDropped(T), -} - -pub trait Key: Eq + Ord + Clone {} -impl Key for T {} - -#[derive(Debug)] -struct Queue { - queue: VecDeque<(Instant, K)>, - items: BTreeMap, -} - -pub struct Input { - q: Arc>>, -} - -impl Input { - /// if a key is already present, its previous item will be overwritten and - /// its delay time will be reset for the new item. - /// - /// errors if the remover has been dropped - pub async fn enqueue(&self, key: K, item: T) -> Result<(), EnqueueError> { - if Arc::strong_count(&self.q) == 1 { - return Err(EnqueueError::OutputDropped(item)); - } - // TODO: try to push out an old element first - // for now we just hope there's a listener - let now = Instant::now(); - let mut q = self.q.lock().await; - q.queue.push_back((now, key.clone())); - q.items.insert(key, item); - Ok(()) - } - /// remove an item from the queue, by key - /// - /// the item itself is removed, but the key will remain in the queue -- it - /// will simply be skipped over when a new output item is requested. this - /// keeps the removal cheap (=btreemap remove), for a bit of space overhead - pub async fn remove_range(&self, range: impl RangeBounds) { - let n = { - let mut q = self.q.lock().await; - let keys = q - .items - .range(range) - .map(|(k, _)| k) - .cloned() - .collect::>(); - for k in &keys { - q.items.remove(k); - } - keys.len() - }; - if n == 0 { - metrics::counter!("delay_queue_remove_not_found").increment(1); - } else { - metrics::counter!("delay_queue_remove_total_records").increment(1); - metrics::counter!("delay_queue_remove_total_links").increment(n as u64); - } - } -} - -pub struct Output { - delay: Duration, - q: Arc>>, -} - -impl Output { - pub async fn next(&self) -> Option { - let get = || async { - let mut q = self.q.lock().await; - metrics::gauge!("delay_queue_queue_len").set(q.queue.len() as f64); - metrics::gauge!("delay_queue_queue_capacity").set(q.queue.capacity() as f64); - while let Some((t, k)) = q.queue.pop_front() { - // skip over queued keys that were removed from items - if let Some(item) = q.items.remove(&k) { - return Some((t, item)); - } - } - None - }; - loop { - if let Some((t, item)) = get().await { - let now = Instant::now(); - let expected_release = t + self.delay; - if expected_release.saturating_duration_since(now) > Duration::from_millis(1) { - tokio::time::sleep_until(expected_release.into()).await; - metrics::counter!("delay_queue_emit_total", "early" => "yes").increment(1); - metrics::histogram!("delay_queue_emit_overshoot").record(0); - } else { - let overshoot = now.saturating_duration_since(expected_release); - metrics::counter!("delay_queue_emit_total", "early" => "no").increment(1); - metrics::histogram!("delay_queue_emit_overshoot") - .record(overshoot.as_secs_f64()); - } - return Some(item); - } else if Arc::strong_count(&self.q) == 1 { - return None; - } - // the queue is *empty*, so we need to wait at least as long as the current delay - tokio::time::sleep(self.delay).await; - metrics::counter!("delay_queue_entirely_empty_total").increment(1); - } - } -} - -pub fn removable_delay_queue(delay: Duration) -> (Input, Output) { - let q: Arc>> = Arc::new(Mutex::new(Queue { - queue: VecDeque::new(), - items: BTreeMap::new(), - })); - - let input = Input:: { q: q.clone() }; - let output = Output:: { q, delay }; - (input, output) -} diff --git a/spacedust/src/server.rs b/spacedust/src/server.rs deleted file mode 100644 index 9d3dcf6..0000000 --- a/spacedust/src/server.rs +++ /dev/null @@ -1,339 +0,0 @@ -use crate::ClientMessage; -use crate::error::ServerError; -use crate::subscriber::Subscriber; -use dropshot::{ - ApiDescription, ApiEndpointBodyContentType, Body, ConfigDropshot, ConfigLogging, - ConfigLoggingLevel, ExtractorMetadata, HttpError, HttpResponse, Query, RequestContext, - ServerBuilder, ServerContext, SharedExtractor, WebsocketConnection, channel, endpoint, -}; -use http::{ - Response, StatusCode, - header::{ORIGIN, USER_AGENT}, -}; -use metrics::{counter, histogram}; -use std::sync::Arc; - -use async_trait::async_trait; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; -use std::collections::HashSet; -use tokio::sync::broadcast; -use tokio::time::Instant; -use tokio_tungstenite::tungstenite::protocol::{Role, WebSocketConfig}; -use tokio_util::sync::CancellationToken; - -const INDEX_HTML: &str = include_str!("../static/index.html"); -const FAVICON: &[u8] = include_bytes!("../static/favicon.ico"); - -pub async fn serve( - b: broadcast::Sender>, - d: broadcast::Sender>, - shutdown: CancellationToken, -) -> Result<(), ServerError> { - let config_logging = ConfigLogging::StderrTerminal { - level: ConfigLoggingLevel::Info, - }; - - let log = config_logging - .to_logger("example-basic") - .map_err(ServerError::ConfigLogError)?; - - let mut api = ApiDescription::new(); - api.register(index).unwrap(); - api.register(favicon).unwrap(); - api.register(openapi).unwrap(); - api.register(subscribe).unwrap(); - - // TODO: put spec in a once cell / lazy lock thing? - let spec = Arc::new( - api.openapi( - "Spacedust", - env!("CARGO_PKG_VERSION") - .parse() - .inspect_err(|e| { - eprintln!("failed to parse cargo package version for openapi: {e:?}") - }) - .unwrap_or(semver::Version::new(0, 0, 1)), - ) - .description("A configurable ATProto notifications firehose.") - .contact_name("part of @microcosm.blue") - .contact_url("https://microcosm.blue") - .json() - .map_err(ServerError::OpenApiJsonFail)?, - ); - - let sub_shutdown = shutdown.clone(); - let ctx = Context { - spec, - b, - d, - shutdown: sub_shutdown, - }; - - let server = ServerBuilder::new(api, ctx, log) - .config(ConfigDropshot { - bind_address: "0.0.0.0:9998".parse().unwrap(), - ..Default::default() - }) - .start()?; - - tokio::select! { - s = server.wait_for_shutdown() => { - s.map_err(ServerError::ServerExited)?; - log::info!("server shut down normally."); - }, - _ = shutdown.cancelled() => { - log::info!("shutting down: closing server"); - server.close().await.map_err(ServerError::BadClose)?; - }, - } - Ok(()) -} - -#[derive(Debug, Clone)] -struct Context { - pub spec: Arc, - pub b: broadcast::Sender>, - pub d: broadcast::Sender>, - pub shutdown: CancellationToken, -} - -async fn instrument_handler(ctx: &RequestContext, handler: H) -> Result -where - R: HttpResponse, - H: Future>, - T: ServerContext, -{ - let start = Instant::now(); - let result = handler.await; - let latency = start.elapsed(); - let status_code = match &result { - Ok(response) => response.status_code(), - Err(e) => e.status_code.as_status(), - } - .as_str() // just the number (.to_string()'s Display does eg `200 OK`) - .to_string(); - let endpoint = ctx.endpoint.operation_id.clone(); - let headers = ctx.request.headers(); - let origin = headers - .get(ORIGIN) - .and_then(|v| v.to_str().ok()) - .unwrap_or("") - .to_string(); - let ua = headers - .get(USER_AGENT) - .and_then(|v| v.to_str().ok()) - .map(|ua| { - if ua.starts_with("Mozilla/5.0 ") { - "browser" - } else { - ua - } - }) - .unwrap_or("") - .to_string(); - counter!("server_requests_total", - "endpoint" => endpoint.clone(), - "origin" => origin, - "ua" => ua, - "status_code" => status_code, - ) - .increment(1); - histogram!("server_handler_latency", "endpoint" => endpoint).record(latency.as_micros() as f64); - result -} - -use dropshot::{HttpResponseHeaders, HttpResponseOk}; - -pub type OkCorsResponse = Result>, HttpError>; - -/// Helper for constructing Ok responses: return OkCors(T).into() -/// (not happy with this yet) -pub struct OkCors(pub T); - -impl From> for OkCorsResponse -where - T: Serialize + JsonSchema + Send + Sync, -{ - fn from(ok: OkCors) -> OkCorsResponse { - let mut res = HttpResponseHeaders::new_unnamed(HttpResponseOk(ok.0)); - res.headers_mut() - .insert("access-control-allow-origin", "*".parse().unwrap()); - Ok(res) - } -} - -// TODO: cors for HttpError - -/// Serve index page as html -#[endpoint { - method = GET, - path = "/", - /* - * not useful to have this in openapi - */ - unpublished = true, -}] -async fn index(ctx: RequestContext) -> Result, HttpError> { - instrument_handler(&ctx, async { - Ok(Response::builder() - .status(StatusCode::OK) - .header(http::header::CONTENT_TYPE, "text/html") - .body(INDEX_HTML.into())?) - }) - .await -} - -/// Serve index page as html -#[endpoint { - method = GET, - path = "/favicon.ico", - /* - * not useful to have this in openapi - */ - unpublished = true, -}] -async fn favicon(ctx: RequestContext) -> Result, HttpError> { - instrument_handler(&ctx, async { - Ok(Response::builder() - .status(StatusCode::OK) - .header(http::header::CONTENT_TYPE, "image/x-icon") - .body(FAVICON.to_vec().into())?) - }) - .await -} - -/// Meta: get the openapi spec for this api -#[endpoint { - method = GET, - path = "/openapi", - /* - * not useful to have this in openapi - */ - unpublished = true, -}] -async fn openapi(ctx: RequestContext) -> OkCorsResponse { - instrument_handler(&ctx, async { - let spec = (*ctx.context().spec).clone(); - OkCors(spec).into() - }) - .await -} - -/// The real type that gets deserialized -#[derive(Debug, Deserialize, JsonSchema)] -#[serde(rename_all = "camelCase")] -pub struct MultiSubscribeQuery { - #[serde(default)] - pub wanted_subjects: HashSet, - #[serde(default)] - pub wanted_subject_dids: HashSet, - #[serde(default)] - pub wanted_sources: HashSet, -} -/// The fake corresponding type for docs that dropshot won't freak out about a -/// vec for -#[derive(Deserialize, JsonSchema)] -#[allow(dead_code)] -#[serde(rename_all = "camelCase")] -struct MultiSubscribeQueryForDocs { - /// One or more at-uris to receive links about - /// - /// The at-uri must be url-encoded - /// - /// Pass this parameter multiple times to specify multiple collections, like - /// `wantedSubjects=[...]&wantedSubjects=[...]` - pub wanted_subjects: String, - /// One or more DIDs to receive links about - /// - /// Pass this parameter multiple times to specify multiple collections - pub wanted_subject_dids: String, - /// One or more link sources to receive links about - /// - /// TODO: docs about link sources - /// - /// eg, a bluesky like's link source: `app.bsky.feed.like:subject.uri` - /// - /// Pass this parameter multiple times to specify multiple sources - pub wanted_sources: String, -} - -// The `SharedExtractor` implementation for Query describes how to -// construct an instance of `Query` from an HTTP request: namely, by -// parsing the query string to an instance of `QueryType`. -#[async_trait] -impl SharedExtractor for MultiSubscribeQuery { - async fn from_request( - ctx: &RequestContext, - ) -> Result { - let raw_query = ctx.request.uri().query().unwrap_or(""); - let q = serde_qs::from_str(raw_query).map_err(|e| { - HttpError::for_bad_request(None, format!("unable to parse query string: {e}")) - })?; - Ok(q) - } - - fn metadata(body_content_type: ApiEndpointBodyContentType) -> ExtractorMetadata { - // HACK: query type switcheroo: passing MultiSubscribeQuery to - // `metadata` would "helpfully" panic because dropshot believes we can - // only have scalar types in a query. - // - // so instead we have a fake second type whose only job is to look the - // same as MultiSubscribeQuery exept that it has `String` instead of - // `Vec`, which dropshot will accept, and generate ~close-enough - // docs for. - as SharedExtractor>::metadata(body_content_type) - } -} - -#[derive(Deserialize, JsonSchema)] -#[serde(rename_all = "camelCase")] -struct ScalarSubscribeQuery { - /// Bypass the 21-sec delay buffer - /// - /// By default, spacedust holds all firehose links for 21 seconds before - /// emitting them, to prevent quickly- undone interactions from generating - /// notifications. - /// - /// Setting `instant` to true bypasses this buffer, allowing faster (and - /// noisier) notification delivery. - /// - /// Typically [a little less than 1%](https://bsky.app/profile/bad-example.com/post/3ls32wctsrs2l) - /// of links links get deleted within 21s of being created. - #[serde(default)] - pub instant: bool, -} - -#[channel { - protocol = WEBSOCKETS, - path = "/subscribe", -}] -async fn subscribe( - reqctx: RequestContext, - query: MultiSubscribeQuery, - scalar_query: Query, - upgraded: WebsocketConnection, -) -> dropshot::WebsocketChannelResult { - let ws = tokio_tungstenite::WebSocketStream::from_raw_socket( - upgraded.into_inner(), - Role::Server, - Some(WebSocketConfig::default().max_message_size( - Some(10 * 2_usize.pow(20)), // 10MiB, matching jetstream - )), - ) - .await; - - let Context { b, d, shutdown, .. } = reqctx.context(); - let sub_token = shutdown.child_token(); - - let q = scalar_query.into_inner(); - let subscription = if q.instant { b } else { d }.subscribe(); - log::info!("starting subscriber with broadcast: instant={}", q.instant); - - Subscriber::new(query, sub_token) - .start(ws, subscription) - .await - .map_err(|e| format!("boo: {e:?}"))?; - - Ok(()) -} diff --git a/spacedust/src/subscriber.rs b/spacedust/src/subscriber.rs deleted file mode 100644 index 2e6a963..0000000 --- a/spacedust/src/subscriber.rs +++ /dev/null @@ -1,164 +0,0 @@ -use crate::error::SubscriberUpdateError; -use crate::server::MultiSubscribeQuery; -use crate::{ClientMessage, FilterableProperties, SubscriberSourcedMessage}; -use dropshot::WebsocketConnectionRaw; -use futures::SinkExt; -use futures::StreamExt; -use std::error::Error; -use std::sync::Arc; -use std::time::Duration; -use tokio::sync::broadcast::{self, error::RecvError}; -use tokio::time::interval; -use tokio_tungstenite::{WebSocketStream, tungstenite::Message}; -use tokio_util::sync::CancellationToken; - -const PING_PERIOD: Duration = Duration::from_secs(30); - -pub struct Subscriber { - query: MultiSubscribeQuery, - shutdown: CancellationToken, -} - -impl Subscriber { - pub fn new(query: MultiSubscribeQuery, shutdown: CancellationToken) -> Self { - Self { query, shutdown } - } - - pub async fn start( - mut self, - ws: WebSocketStream, - mut receiver: broadcast::Receiver>, - ) -> Result<(), Box> { - let mut ping_state = None; - let (mut ws_sender, mut ws_receiver) = ws.split(); - let mut ping_interval = interval(PING_PERIOD); - let _guard = self.shutdown.clone().drop_guard(); - - // TODO: do we need to timeout ws sends?? - - metrics::counter!("subscribers_connected_total").increment(1); - metrics::gauge!("subscribers_connected").increment(1); - - loop { - tokio::select! { - l = receiver.recv() => match l { - Ok(link) => if self.filter(&link.properties) { - if let Err(e) = ws_sender.send(link.message.clone()).await { - log::warn!("failed to send link, dropping subscriber: {e:?}"); - break; - } - }, - Err(RecvError::Closed) => self.shutdown.cancel(), - Err(RecvError::Lagged(n)) => { - log::warn!("dropping lagging subscriber (missed {n} messages already)"); - self.shutdown.cancel(); - } - }, - cm = ws_receiver.next() => match cm { - Some(Ok(Message::Ping(state))) => { - if let Err(e) = ws_sender.send(Message::Pong(state)).await { - log::error!("failed to reply pong to subscriber: {e:?}"); - break; - } - } - Some(Ok(Message::Pong(state))) => { - if let Some(expected_state) = ping_state { - if *state == expected_state { - ping_state = None; // good - } else { - log::error!("subscriber returned a pong with the wrong state, dropping"); - self.shutdown.cancel(); - } - } else { - log::error!("subscriber sent a pong when none was expected"); - self.shutdown.cancel(); - } - } - Some(Ok(Message::Text(raw))) => { - if let Err(e) = self.query.update_from_raw(&raw) { - log::error!("subscriber options could not be updated, dropping: {e:?}"); - // TODO: send client an explanation - self.shutdown.cancel(); - } - log::trace!("subscriber updated with opts: {:?}", self.query); - }, - Some(Ok(m)) => log::trace!("subscriber sent an unexpected message: {m:?}"), - Some(Err(e)) => { - log::error!("failed to receive subscriber message: {e:?}"); - break; - } - None => { - log::trace!("end of subscriber messages. bye!"); - break; - } - }, - _ = ping_interval.tick() => { - if ping_state.is_some() { - log::warn!("did not recieve pong within {PING_PERIOD:?}, dropping subscriber"); - self.shutdown.cancel(); - } else { - let new_state: [u8; 8] = rand::random(); - let ping = new_state.to_vec().into(); - ping_state = Some(new_state); - if let Err(e) = ws_sender.send(Message::Ping(ping)).await { - log::error!("failed to send ping to subscriber, dropping: {e:?}"); - self.shutdown.cancel(); - } - } - } - _ = self.shutdown.cancelled() => { - log::info!("subscriber shutdown requested, bye!"); - if let Err(e) = ws_sender.close().await { - log::warn!("failed to close subscriber: {e:?}"); - } - break; - }, - } - } - log::trace!("end of subscriber. bye!"); - metrics::gauge!("subscribers_connected").decrement(1); - Ok(()) - } - - fn filter(&self, properties: &FilterableProperties) -> bool { - let query = &self.query; - - // subject + subject DIDs are logical OR - if !(query.wanted_subjects.is_empty() && query.wanted_subject_dids.is_empty() - || query.wanted_subjects.contains(&properties.subject) - || properties - .subject_did - .as_ref() - .map(|did| query.wanted_subject_dids.contains(did)) - .unwrap_or(false)) - { - // wowwww ^^ fix that - return false; - } - - // subjects together with sources are logical AND - if !(query.wanted_sources.is_empty() || query.wanted_sources.contains(&properties.source)) { - return false; - } - - true - } -} - -impl MultiSubscribeQuery { - pub fn update_from_raw(&mut self, s: &str) -> Result<(), SubscriberUpdateError> { - let SubscriberSourcedMessage::OptionsUpdate(opts) = - serde_json::from_str(s).map_err(SubscriberUpdateError::FailedToParseMessage)?; - if opts.wanted_sources.len() > 1_000 { - return Err(SubscriberUpdateError::TooManySourcesWanted); - } - if opts.wanted_subject_dids.len() > 10_000 { - return Err(SubscriberUpdateError::TooManyDidsWanted); - } - if opts.wanted_subjects.len() > 50_000 { - return Err(SubscriberUpdateError::TooManySubjectsWanted); - } - *self = opts; - Ok(()) - } -} diff --git a/spacedust/static/favicon.ico b/spacedust/static/favicon.ico deleted file mode 100644 index 5db484406fbdcb26760f9c29070dfa98fc5d4463..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 15406 zcmZQzU}Rus5D);-3Je)63=C!r3=9ei5Wa>W1H(KP1_lEI2tPxOf#H}a1A_(w1A_oa z9Roz10XO*1z`*>Ufr0x!1GB_`26nCg%zQ@w8MsaV>)6Hr*R_uMFKdwSpI657KLeY} ze`XGu{|t;={}~vV{xdKzK=d;5{pS_a`_I6s|6kj!_P>Bm+J6@L=>Jwxz5fk7O8<+Q zl>KLt3jQys7XF`2$mTz{sPTUWMn14!CKkc}(i*}4c@={H%iGufmvwCW&#qhjpUtr2 ze?-^W|3;Cs{<9f%|5x-~@L$xj?mw4m%74d%h5tp>LjN-`^Z#dHmi*7iWA$Istn`0S z_3{52@!S5(gzfsTmT~gG|IDZVgC|}6ub+4Ef83PE|J5S4{8#pw_n%2B>c5~$=zj(l z$^Q(js{a`T9R6!~PyR3NvHZVT$+`b#&5!=O%=qv>eCPN7ky}6hH*9|OUoi6gf7z%* z|Aky;{pT~O`Ohrn_n(1N<3FpQ&3`7Tu>XvjmH)+qw*8mMy#C*Q#<%~`+yDOWdiv*o z!@Xbsjc5M;uif$Szfi)(|17qP|Fh_~{8zB+{LjE?@Sj!0<-dSQ-G2d>dH;F*cl}qZ zefi&O&ENk$xBmUV_2cjVeP90mpK|Zt|AOOx|0|R}`!Abx=D%Ixx&N|Glm0V{IR9sr zaQV-npZ%Z9aoT^O*i-+_CVl$9`tiU2mp=dd|Mu5^Fn;>u-~Ym6fB&mDefTdNe&Ro) z@#OzZs;U1O_-+0(@LK<8kc;J)C$QT3hw{wsg{pM2}z|MN)tx4irJKYahc|7MfF z{uhWm@t@JC|38CV!hZ%noBs^l7XLXlv;Vu69{#VHd+xtl&FlZkyZ`>*_UhmN7eD{~ zfBgO5{|lf0{jWav@4w5u@BhskZu}SVTJWFOu;Rb4cG`ai4#WQp5&{3!f*1d1cUkpc zw(!RPpq0P=cU}4Szw^ew|1B5({jWd&_kZHH-~W}1ZvGbx-twQ@y!AhWMBsmBA-n$! zY?}WWgq;2}sV4ttGU@oQmU-&GZ0^nfA%J;vpb^CuV>nZ=)Jy!lVsJ`}JF6YXBp6HYR zdBP6=XSQ4RU)+E3e}nkV{|&>I{%7E_`p>{D`=4LI`9A}@?tfmT=>ME5ssDwoTK)?; zPxvqHv+zH+>)ik5`N#izl%4(0Z$I%rgM1R$uZCVT{)=j5{byj8_|L$=@t=WF^gpYR z-G3F^&i@knh5z|9)BiKb#QtZ~&Hm4*lJlQOujD_oLd1Vo8Bm%t{?8*9@SlN6_&);! z+kXZImj4V4oc|dZMgB8#X#8j3(EZQAY51Rk*Wy0|D15+iZ26yo+xWkzLHd724yFGL zOnm6p8@0! zX5s$~jNH(;0oetz6XXUE4dR3B=KIepqWYhKRSx0@PQCxK#wq_3>URGZGRphUA?y2} zLCohrlT;wM+>EQ8_dhbf`#-x((0>Mr(EldEQ~xuFh5Tm_bOO6m-YDt6fQ-j~RzB_j zT*6xa8JNZXGcfZ0XJBLpm$8s|289R2ZU$!Y|NIgL|0OkD!QsikXZ2rDBkaF|L(_kG zhlc;0DslfAI2V{|vmg|5=0_{;ODKg5wZm2Lltx4WKj%vYYik1Czji24*F2zGGn5`7fa2@n6Cy z;Xj{x+>C71rk)GYeXu3i0~!=T|m zk5R*a-SEZ#S+$D(^XX;$XIF^+&miLYpGhL*zgPD9|Ey9`{~353z;zh|6F)SqbN=TP z)c9}WUh&^Du=~HFMfrag`K14LNh|(KJNEq-wrv0JRJiv)vv$jWHlz0c%m#h`SWl;BjX_JKiOgwUs`jV0VKLd;Ge+G7=|7_y^|7ESJ{<~zX`(MBC@&D}U zkN%6hFZl1cd^=(hjN=5znEIj#DypLP1bcK((BmAhX5kDYY%zkAc={|vU9{`>dA^WK;h$2)h4g5^(y@z@`JqSD-Ker3qGL zaGnB}`68bGnU#|Nv+A|}kM6wkUnP0Rf2qJN|9yJy{bzCB{$Doz^ncI3r~j2oAN?;o z{`r6O(p&$1Cp`Mk5q9o>`r2y~uakN9zkJxP|MI~b{@dj2|IcmI`kz5E@;|3? z>VMPl8UF>;0{=5`s6x^svoxfv<~IG$An5X+Niy=kir1|FeAZL`vzW~IAJ%>2zkJ4t z|E{ff|0@;V`LA61@PESk@BcO1Km9Md`u%^*&X@lK=Dq)K(f#_rdeMXbc1;idGdS)3 z&tSFozkceG|I+@e{|ni5{O32V|1W9Z_Mb^K=s&Bh$A4x~b4VDlfbzHjIKMDT`2JUN z>-rzjcIm%Y-NpZmW()qaxor6#H|N=ZrP2rggBE@IFJAigf90va|NU40{awm+!fB!4b|NCF{{MY}|JKz4NAOHJ5Y~$bmT)AKVi|V2wyXdB*Ps3GzuT(6|Mj}R{^v`7^j|#V!hdGxP5;@Qmi%Wl?ElYaG4p?9^`-v| zlIh@b0o0~sVAuaIZj}FD#dqd^X^-ju`D`ZqXEC4hpUHaZf3CoT|JkFi{WtIZ^xu8% zpZ|XA{{2rp{_lUwjeq~2{`~iU$M?Vg=Y9S6|JbL0|HJqF`(Jwf-~aSu|NbZK`ujg) z%g_HJ$+!Nq`5gE!9=7klMgHmkwgu<^+hiR6uk1bhKd)~3e+DjnNIOH&bHe+_=O6m}f6p6GyYb)u4`6#BcEj3{Ah&?*Kk@0`|L$x5{(G$X_g|vv=YOWS zNB3!N31G zXa4<9Ir#5?$i~0_9jAT#uUK^VKY#dv|4df%{xj;f{AWD_<3g}?tN9sc(}{@B0&bDu)u9^_|a{N(5V|4U!~ z`yX}U-+z;p|Nd)E`14y-t+LkP4DafCS9NY+b#JE zZhO~X|M!3Y`+xsI_Fn(?@Be`h|Ng%R_b2}S-}L(5|KhX%{+A#B_upgXpZ{)izx)rL z{o=oO+x7nnp_~8nT2J^dV&DE>#n&%mnjpMl%#KZA(te+JpG|I+S#|9Pz^{%5h74sP!YCLjH8H|gbnui0P! z2QU5oKkLZ9{~g!<{m(xA?|;VefB!R&{`()d{onu66aW7EZu$E^YwzFxp(}p;S1h~t zpUrpge`dRt|G6Az|7Xyy{m-JA`=3cZ0+RQ+%^~$3i^_jSKGXk<;?Cf<9kXiEe-^`v z|I8+>|JmIZ{5L2*{oki6{+~g}`9Fhn(0@jiN%& zf9}6~_MZO?yx#vA*!2H1fZ82QqW`%Roc?Xm7xU-_j~y^bhJf4Opm7vd$*BL9VXOW# zaQXaaFEEKiL3uJDCYiWP)h&LppyNc-?8aGgL2V-HuJ9kI^k>oGbm;MXHZQ3&ux(LpG6*2 z$GL*rbqrjl|HbsO{|m?`Li&Z^J~GH|P+t>-LE~1S{-)@EMlO~AqDGnjb)0+ut2#~n zFJV~upMl>2QeTJ#g3Cz;$>9HtvJwBejkEqUNJoIn6gG{_{|w@R{~2Zc|1*fV{TDDz zhO}`&`HW5LKd)xMe+CxS{|rn*{}~uS_JR68pne~ypAPDqg2W(h0QZGhH2<@SyZvVp zu>a4%1sZoV|Iff@4IYyM=TlL)|4cGI{~3hcA#M|L`OhHW2=*_dpv`|3pSJ(fre*(` zcnqL%EBT*+Nf2r`$j>0VLH#cf2Dt&`9*{e@|1&TOK>FUG{K29CP8Xmt5e5z&@K_!L zsNKSC{GWlx1cE_qkQk@le+CY1a6gVi%Jo0Ldh~x5K`n@%QTo0hI~o5oGJ^WPL<}Yj zV%%=07nb+*UsCvkMMjpHWjBKE> z1I7Og%yJMjL1u&U5UB48j#rS~pfm(>4+w+Yhzp~+8#ERHG6Q55Xsm`=64D+ItJ3)4X#;!qb0gs!3!W86AbU%Q^Ko}$s z3Ukmn6==Lj2om1lF(XiYs`{T1l<#>R|1*jB{TJ1a{Le1t@Sj!04pR3DI)cY0LHStS zKJ9-}#l-(fr9J<(tm6MO3A=#XqwI=t|5KXx{g=0E`p+!t0U1*QnaQE|pPAR_zqW1k ze@1p?NFNr|hX>_t(AX7d+z`bNAoqg&0ir>119dwi$nAXpnc2nvvv7jSe%b#FY@jh1 z_5Y0A`u`>M!vEVwPW>NMx#Pd4SNne!NuU3mvOfPAL_Gd8h}Ni z;_7by!R0Ei<$nf#yZ>xbUjL;nvj5B2)c-e+p8cOgHTgfcddh!h#pM4C3TgkP?Q8xE zTebgB={@m3zkkdB^!oY#b=)ieGlBXS%31#voLl}27?l3!P|NtwC>#5qPb2m}qqzTn zMVq|;B3kkP8TcLlGw@pdXA|@O@0~XDKO>+0e+D*CdXj>Se}c#8Kz=~V*Nhzh!Q*Bi ze}c*s(6|ws`hON)!~crbG5-Zr{r(GTM*e4!^atA|XH)c_M?2@gs72X-F73Sk>>7Fh z*|duPGpLsQ7qjp9ujs$%fBx*-|BGiH`k&Fd;=hhh%YP<~%KuCnRsSXITK=A#xawn##iT{~&JO4B4b^hlv>-x{A-Tq%Cc=dntv~B+xHERDeX%_rv(<=PWsh;;=)F}Ty zt8B=BCWWm3<+E=5kE=WI-z{bFf3twz|JqI!{{^*!{xb_I|7T$0`Om-zN;hoav3+)a z?f)7UY5xN==Khbb-1OfqX~BOPtNQ;eYDNErEPMWkHJ${Ix3Q>K{CCRR_1~fN=zlJg ziT@dO+WvEz_x)!!ocN!?aK?XDi$(wSb5H({pKxpIy}EKLdw6q|E`!ub?&$csw6eJ{tUI;&=Pcqnz+x)urXX zPyUAg*?s5#7tg%=zkkD*|NeDn{);-!`mYS9JM*Er*VHE;hHEqL}{Eo}3DW-ZV-Nb!Fe$4USFD-Qo>)u{N-AR79gLBRb#1Fyq>22Rue z44}G#Mdm*PvnXUv08~eS#__@RlIDK~PUHU!ymtQ?guTH1X>R?h|JtF;{ufTW`#*L1 z|2yXH`7dhS_n$#B?mvTw&wmC%pZ}t|8UF=UBmOgT>i>s~FM`TdPhp5B|@kpZ}jpz3jiB?acp~Q*ZrutUvdk&v(s#5x?#Kvll=Auab4~ zKa=yW|B?wO{~OfZ{~tZ?&41Z~$NwW%fB2tq@Z6R113NF zZ&GvrzjDrv|MAlv|L1hy^`F&t^?zQ+mH&Aim;U#tJoR7JbJ2eeT~MDq_CKR!^nWRv zrvJ9dEB_k=wEySV^!?8On(yIOf%FYPZ4OpY{iOAufzt>wKPl(}u4@=%lK&gVEdS4I z*887Pr}ICT?V|sw)9(DYZM^iK(|6Z@v$CuI{ii?w&l`UJzhK;z|K<}v{4d-2{l8rK zoBz?rejr-(udk{{b^U{O5?i^IxOv!GBiY)BhRW_y1>h-t}KI_0)fZtP}sm1J?c* zaGLg?)2QXYjMt3+A~tRRnPn6HGs}kkXHyFL&m!#xY3qZ=h(Y}TP}>jG4&(x*du#9* z2BSp4e@=~@|GJUO{<~G4{vSK>-hT<-&HouKm;7gSUH4xg_~`$mUv;VWH z#s6o5jd6hbO`v&T(0m%RBDjCeBxv;?+)onm{Ldf}@}E&5`9H6D<9{Z-p8vegEB-rF zUHGpUb?`r%=dS-8A;3@N+ z6aQIUw*F_dTJfLVZtj0(qu&2ay7m8A^-BLUDy9EtQOf==?b!04MKbU|189s0G;a$^ z_slB)8MsXTix?$ABawsrPuz8r(OFm zUhwe0c)_dx-iv?!cb@m>fBg2p|25|R{l6GAUiRYO|Eha`|5v{G^*`^)xBtc0e*d5Q z;Ii{ zkN#Klo%f$xGYdSf2A*eO(FU^xwUYj`DW?AyG;jVd>oxbkV(`lUvY~7L3;V79&*8G_ zKbPOG|02of|BGke`OlyI{J&KF*Z-a?{{9bJ|L=eD-hcmdPW=1d_vqjMsZaj>zxoX{ z5ApB+g`a=__kI8K|Lw1T|K~mb_rLPOzyFJ0{rkV{#lQdMm;e25x%lsY&-s7Oa8t%CnNIvJ31 z2-GKK)rF)d&^#Gv97ZPoKcjNte-^#=|2+2d{!4~!`LC39;J;AR{{LK|r~ixRJ^1gp z@XLSi#lQY*F8cf5dHcWrxyS$g&pY|=|E5>}{@?lW@Bj0k|G;DCZ+`vzfA!a2@O;kh z5C8t({{HX(-Jk#duYU9I|Dm`4{#RZ4_g{VC-~TN6pZ+sOKK{=fb?ZNW^7a2V4fph%^AA~`EIQ!+_|He!I{zq^6_uq5D zpa14Ppa09`KmE@ZeEC0v>%spFwj2I4m@oX#U@+l7gI4{229<*U44`p7sgVB+Vjlk) z_-+3)a2Wh&;x_s(VjlNj!YS{+xNG@;DWB&5GC@861>C#;^Lfnt&+oV9zfk0%|5DjE z{_D5A_#e9L+yB0EfBtWL`0xL@kDxiHfB!%Hg3Nz`<~u-c1GyDMzXi{K{Qv(JG&lAe zG9UBc$N&G6pZxouaq8dyklp|O+pPHaUv=7_|9n*+{xiki`_Jfi{y&53esEir(Q@H` z7R$;1Ic+-sbJ^7X=dvjL&ta7OpIHGkR&NVl4+b7H28~UNxc_I63i{8ekno>Lqu@V_ zNz;FBx0(ONqPG24E4=jIu>Qe+&na*Jhc5sAKj+}z|DgG}4KM%wzyJN;|Cc`@b4Q@{ zS0Hy|xF0&N_TcBg|9d|D```QM-~Xzs|Ndv5`1e0@@4x>}>;C>Xn(_U=V(qK{!pXP( zbBCP#&*l!Q%NG4-G@AIIQMd6wgIXbEJrii`7t}`swJ||;0BFn`6c?a0B@qM}A6Cx& zSzU(jRmf5o_c|BXwp{C8@9^xtdR+y6cbzWxtd_xpeB?!Vx4@Hj7n zEF}LjYS;f4^I!R2E^6<8$*5!hrBW{Zm&&>IU!m&hf5qmH|7{ok{$G6jA2<)E9D&SJ z?fwWUdtmOxpnv`U|K!KN|2tp*`(JYY-+#L;|NdL7`S;&q>EHiGGk^Y9Z-4t=BKsz| zo@8-a|DO>w=U~wLpFyh@QlHAjgVz#($A&@cl{i3kupXrRgqA-{ib?;O)QbKyXg2;= ziCF(%B5eDA4!;&ag5C)|u5DmhhdGV{?|NTGt>EHiFPyhYz2hGdx{rBHy&fovKJ>UOpb-eqp(fs(o zYURcMa+&-73k9$K&t*UTKdWK$e@3;!|BRsgD<1ZrLBtO{<^rB0Wzqi602&(+bOw(* z@fa5W*A8FuUol|ifA`8$|II7T{pa-D@n0qF^ndf3JO5QHp8Qv&w4xl2%f2F+!H=9NM7+wXt<`+xciq>h_>@8AE53;+IS zAN}_~dC%Yf5vzay51#k=zxRaa|8+~Q{+Eb5{9i0|>wn3RmH(yu=lz%PnEYSDrTf3Q zUHg9}r=I^3R#o7&kPIwZ{~17YY@jiEQLq1uvI+nB&AR>vG@kvx=;F=vAO4%q|NFn__`mJ?98$J^`1K#0&mVmM_kY*BfBz$o|NCEk>fitT1ONWJt^D)fbdmkJ z>$W}qA2a{i|Kb%-{<~J6`p;`S`#-Z*-G5f~!vE|Vx&H->YyVruulcVRGVebVX#N8< zZVZ|)2jyo_z5wM9`LzGcx?TShyRZC@?7s1z$7Ab%CWp=cm2=L5*Tu3&-uN$5`}V*6 z@<0Ehw*UK|u;<_Z%G3Y;&$;*S|IrVSvS{_|fB(B4{QE!Q*1!MDpZ@!Q?eo9?TVMbC zzv%J5|4SbK`#=BQzyIEw{{9bH_4|Lt&L98vT3-Iws(<)jv+Blw;iRMgMWXlqwKfmRsY>f z_x%@kUGks7c*cJw+a>>*oj3mH4m-+unz|K6Mb{^y;B)LV6@|NhTA@b|y% zyzl>IDxUopO}+A;+56CcUjJSHC8Kxz7Y<(apUGn4e-`7W|7?0i|Cv=2!Rebp*bO{Z z%q(I9S+B+nTBD^8NlStb{~2Zd{xix&{AbrK`Y-M^<3F=O+kZC4ssCB+=lGCt zcHe)+vg`k&m%jgR-1_dn=j`wQQ`Y_dpMU7z|Cs&%{?}dj_rLYZzyA>jAnoMp6aW54 z9r*X(dfvbPB}f1LkKXz3zwOe$|0B2l{U5RN7q~w3oc8iRU&87C?B3h|Guo{9&u%~e zKd1fF|BO1IdN=<+qhj)ZMwy8J45D8D89?jIxGf;#vEX@T1@L-UMnTK}%(CwP86-Tx z{sj4#)v)+Kn?>V)Cd20cYz|ZZb9gTMFBY@!zgN$_|LVo}|GQ3k{a?TN&3~6UpZ`mA zef^)h^UwdflYjraZ2bG*7qre}{lEWVTmJsnU-0+8MAPs8VJm(0spy?|&wh=>Lo=NszTMn$`aq^m_jDIIZ|^Qg-$~pU!_G~PW|_u z_5Od zLf5D1{s*s71I-IFN&IJEQU1@!rT3pvz~Vonfc<|CrGWo@rg8t76+{0s$bkC7DgPN% z%l`A(%=m9re)PYP=j#7Vwu}BVJFfiCXush!+{{m-gd^IySd>VH;He-@PPLFGKS&7k?80W@C71R8@A1JAn& z=tceK*Ny)#X<7JR%rf&otE|_52GIPDMBsl0*~I_C)?NQiGPeE~cAEX4QNQ~?qsfH- zjHWaHGdnE(FOqoRzij&X|K5`y{ufL={h!}b3mVk_XW#>^{WXTH2?6C>@Ho2Ye+DLr|Lh_j|Go3J|2Ill@n1kQ6*6Z4+9v>7 zZ!8KL3o80=lC=6iziIn_29=Wk44ReTJk4s`|DVHk&VQk(P5%vYPW-oTzw)2WXUl&M z&+Y#e^UnPjjyv*SD)YpDq42~189n#>7tg)`u0y!o*8gWPpZlNDcq+J$#Gu>qpHZjo zzo2LTe_^Ld|24vv{^v94`tMhM&~Et8pjQ5$K{@X~qe{kq4zmKt`VR5P|E!9I|D!8T z{;!+f%Kzd9RsY$=1O78|Y5!+nkpQn>XXaP_&&+T2U)UrLJdVY{s`j4&G#ce+Fq#|H>CU4#*(j|6ku{;s3y#qyP2Y=ls`j>iI8hnDd`i%KSeA z7id0-`#)qag7AMv4rxf7fYKFc?GvZze_7kg{{k90{~7o^|1$`>{bvx3_%ChS_uo2U z)qiHt_z-A4vO&ZDEnSAVj z>ykJBd99cK=e3*jU({y$e+kcN|Ak$e{xitN{%4R31J^qYpz%LYy5+O|&&2Qb-!gpB ze=(hg{|xNT{~1_~AnU?F`<}r2kwEj?pg9-7(nGJXfB=0`ac7w<9|jz zum9ZgY5(PHC;rdty7XV%toc6!s2wXE@n6=V`@eVb?*9x*6_9XL$^OrvlJ}p_q2)iP zdF_7&)$;!g`i=j&tf&1CZ@%)MLABvOgKFM?M%AqUjPgnUg`La)vnYV-R?wOc(0nRn z{*Hmm>_4lp&wp*NN&h*7KzoGf{%2qS%_o8813>F-B>yw8 zDEw#UHUBTHnf71TbJBm?_%;6{%1{3H&f58(K{Oh&zC_gLKd)x?e?I-P{|wS`;C2&u z{6aqAKd%*NJ!j^B2KnUwjB1(xnbnK`t4A#U&!CX;pFuhbT(>Yt1pa4J4gW7>o$(*M z))Q2>g4P~_#%I~}|FcVa{uedM`_IU33`rZHH8SA2Y><1|pnDZS?g!0Bg4W4E{lF;l zpMg#GKZl~re?hIZ|3Zpc|2f2x|7*F=_^;yB@t=X$6|$EJR9A!AQzD?U*84w$IB4!V z;6G>`506R4e-`Cv$eK*h8c>O_|IEtqkntNapZ|=~pgkAv;I(mr7ODR^Ky?kbCA3ej z^`C)N{Xe^$&3`s=3-DSt@cJIG+d*yxxgF$g(A+O*?i!SiK;j@jfZ|2~GPeQB3(T7T z8CdlGGq4)}7tl-j&n_DTnac#Vxk2N=eAeK3VQ{+x)P5Cl1JA{?DhK^%lmyModqUP) zf!gLEInX$nkP~FR5GcF_ZU6IX$Ng7w?fTCl5&WMK6!xs3H6e2U88~Di)JTA|AY2%fcNUK>q6Qgpf(3+oh2yVL3IYG z{Q>Gr2-ri`27qV~AA~`2AT=PhAU#~h|Ct2M{|g$#|2K|U_+Q?s@jsU;XrGombZ!N- z#|PvNP`HBJhwOIH+&c_|`~$+EI02~v`3JPt0JLrfv<6M+KLfJ}#7xjy4$!^~Q2ouS z3~7h3Y5Zqk*MzKb;smXQ0=1>|A?wJw^!_t|FehlAl@3G=NG&_4oL2eIz^e40kwfc0 wyNv69WDG5`Po diff --git a/spacedust/static/index.html b/spacedust/static/index.html deleted file mode 100644 index 78c4c8d..0000000 --- a/spacedust/static/index.html +++ /dev/null @@ -1,54 +0,0 @@ - - - - - Spacedust documentation - - - - - -
-

- TODO: pdsls jetstream link - Launch 🛸 UFOs app: Explore lexicons -

- -
- - - - - - - - diff --git a/ufos/Cargo.toml b/ufos/Cargo.toml deleted file mode 100644 index ec22d3b..0000000 --- a/ufos/Cargo.toml +++ /dev/null @@ -1,38 +0,0 @@ -[package] -name = "ufos" -version = "0.1.0" -edition = "2021" - -[dependencies] -anyhow = "1.0.97" -async-trait = "0.1.88" -base64 = "0.22.1" -bincode = { version = "2.0.1", features = ["serde"] } -cardinality-estimator-safe = { version = "4.0.2", features = ["with_serde", "with_digest"] } -chrono = { version = "0.4.41", features = ["serde"] } -clap = { version = "4.5.31", features = ["derive"] } -dropshot = "0.16.0" -env_logger = "0.11.7" -fjall = { version = "2.8.0", features = ["lz4"] } -getrandom = "0.3.3" -http = "1.3.1" -jetstream = { path = "../jetstream", features = ["metrics"] } -log = "0.4.26" -lsm-tree = "2.6.6" -metrics = "0.24.2" -metrics-exporter-prometheus = { version = "0.17.0", default-features = false, features = ["http-listener"] } -schemars = { version = "0.8.22", features = ["raw_value", "chrono"] } -semver = "1.0.26" -serde = "1.0.219" -serde_json = "1.0.140" -serde_qs = "1.0.0-rc.3" -sha2 = "0.10.9" -thiserror = "2.0.12" -tokio = { version = "1.44.2", features = ["full", "sync", "time"] } -tokio-util = "0.7.15" - -[target.'cfg(not(target_env = "msvc"))'.dependencies] -tikv-jemallocator = "0.6.0" - -[dev-dependencies] -tempfile = "3.19.1" diff --git a/ufos/fuzz/.gitignore b/ufos/fuzz/.gitignore deleted file mode 100644 index 1a45eee..0000000 --- a/ufos/fuzz/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -target -corpus -artifacts -coverage diff --git a/ufos/fuzz/Cargo.toml b/ufos/fuzz/Cargo.toml deleted file mode 100644 index 63483a7..0000000 --- a/ufos/fuzz/Cargo.toml +++ /dev/null @@ -1,39 +0,0 @@ -[package] -name = "ufos-fuzz" -version = "0.0.0" -publish = false -edition = "2021" - -[package.metadata] -cargo-fuzz = true - -[dependencies] -libfuzzer-sys = "0.4" -ufos = { path = ".." } -jetstream = { path = "../../jetstream" } -bincode = { version = "2.0.1", features = ["serde"] } -cardinality-estimator = { version = "1.0.2", features = ["with_serde"] } - -[target.'cfg(not(target_env = "msvc"))'.dependencies] -tikv-jemallocator = "0.6.0" - -[[bin]] -name = "counts_value" -path = "fuzz_targets/counts_value.rs" -test = false -doc = false -bench = false - -[[bin]] -name = "estimated_dids_value" -path = "fuzz_targets/estimated_dids_value.rs" -test = false -doc = false -bench = false - -[[bin]] -name = "cardinality_estimator" -path = "fuzz_targets/cardinality_estimator.rs" -test = false -doc = false -bench = false diff --git a/ufos/fuzz/fuzz_targets/cardinality_estimator.rs b/ufos/fuzz/fuzz_targets/cardinality_estimator.rs deleted file mode 100644 index 95219fc..0000000 --- a/ufos/fuzz/fuzz_targets/cardinality_estimator.rs +++ /dev/null @@ -1,20 +0,0 @@ -#![no_main] - -use bincode::config::{Configuration, LittleEndian, Varint, Limit, standard}; -use bincode::serde::decode_from_slice; -use cardinality_estimator::CardinalityEstimator; -use libfuzzer_sys::fuzz_target; - -type C = Configuration>; -static BINCODE_CONF: C = standard().with_limit::<1048576>(); - -fuzz_target!(|data: &[u8]| { - if let Ok((mut estimator, _n)) = decode_from_slice::, C>( - data, - BINCODE_CONF, - ) { - // crash happens *much* faster if we just do kinda anything with the estimator - estimator.insert(&"asdf".to_string()); - assert!(estimator.estimate() > 0); - } -}); diff --git a/ufos/fuzz/fuzz_targets/counts_value.rs b/ufos/fuzz/fuzz_targets/counts_value.rs deleted file mode 100644 index 2fb43d1..0000000 --- a/ufos/fuzz/fuzz_targets/counts_value.rs +++ /dev/null @@ -1,25 +0,0 @@ -#![no_main] - -// use jetstream::exports::Did; -use ufos::db_types::DbBytes; -use ufos::store_types::CountsValue; -use libfuzzer_sys::fuzz_target; - -#[cfg(not(target_env = "msvc"))] -use tikv_jemallocator::Jemalloc; - -#[cfg(not(target_env = "msvc"))] -#[global_allocator] -static GLOBAL: Jemalloc = Jemalloc; - -fuzz_target!(|data: &[u8]| { - if let Ok((counts_value, n)) = CountsValue::from_db_bytes(data) { - assert!(n <= data.len()); - let serialized = counts_value.to_db_bytes().unwrap(); - assert_eq!(serialized.len(), n); - let (and_back, n_again) = CountsValue::from_db_bytes(&serialized).unwrap(); - assert_eq!(n_again, n); - assert_eq!(and_back.counts(), counts_value.counts()); - assert_eq!(and_back.dids().estimate(), counts_value.dids().estimate()); - } -}); diff --git a/ufos/fuzz/fuzz_targets/estimated_dids_value.rs b/ufos/fuzz/fuzz_targets/estimated_dids_value.rs deleted file mode 100644 index 182af02..0000000 --- a/ufos/fuzz/fuzz_targets/estimated_dids_value.rs +++ /dev/null @@ -1,24 +0,0 @@ -#![no_main] - -// use jetstream::exports::Did; -use ufos::db_types::DbBytes; -use ufos::store_types::EstimatedDidsValue; -use libfuzzer_sys::fuzz_target; - -#[cfg(not(target_env = "msvc"))] -use tikv_jemallocator::Jemalloc; - -#[cfg(not(target_env = "msvc"))] -#[global_allocator] -static GLOBAL: Jemalloc = Jemalloc; - -fuzz_target!(|data: &[u8]| { - if let Ok((counts_value, n)) = EstimatedDidsValue::from_db_bytes(data) { - assert!(n <= data.len()); - let serialized = counts_value.to_db_bytes().unwrap(); - assert_eq!(serialized.len(), n); - let (and_back, n_again) = EstimatedDidsValue::from_db_bytes(&serialized).unwrap(); - assert_eq!(n_again, n); - assert_eq!(and_back.0.estimate(), counts_value.0.estimate()); - } -}); diff --git a/ufos/readme.md b/ufos/readme.md deleted file mode 100644 index 8334bb2..0000000 --- a/ufos/readme.md +++ /dev/null @@ -1,152 +0,0 @@ -# UFOs - -_work in progress_ - - - ----- - -cross-compile for raspi 1: - -set up `cross` - -```bash -# build and deploy - -cross build --release --target arm-unknown-linux-gnueabihf && scp ../target/arm-unknown-linux-gnueabihf/release/ufos angel-hair.local:ufos -``` - -for bonilla (rp4) - -```bash -cross build --release --target armv7-unknown-linux-gnueabihf && scp ../target/armv7-unknown-linux-gnueabihf/release/ufos pi@bonilla.local:ufos -``` - -glibc will cause problems when switching between (`GLIBC_2.25` message). clean up (next build will be slowww) - -```bash -cargo clean -``` - -for bonilla but 64-bit? (rp4) -```bash -cross build --release --target aarch64-unknown-linux-gnu && scp ../target/aarch64-unknown-linux-gnu/release/ufos pi@bonilla.local:ufos -# ^^ fails due to linker? - -cross build --release --target aarch64-unknown-linux-musl && scp ../target/aarch64-unknown-linux-musl/release/ufos pi@bonilla.local:ufos -# seems to work - -rsync -avhP ufos-bff-rl/ pi@bonilla:/mnt/ufos-db/ - -RUST_LOG=info ./ufos --jetstream us-west-2 --data /mnt/ufos-db/ -``` - -nginx forward proxy for websocket (run this on another host): - -```nginx - - -worker_processes 2; - -pid nginx.pid; - -events { - worker_connections 2000; - - # use [ kqueue | epoll | /dev/poll | select | poll ]; - # use kqueue; -} - -http { - default_type application/octet-stream; - - - log_format main '$remote_addr - $remote_user [$time_local] ' - '"$request" $status $bytes_sent ' - '"$http_referer" "$http_user_agent" ' - '"$gzip_ratio"'; - - log_format download '$remote_addr - $remote_user [$time_local] ' - '"$request" $status $bytes_sent ' - '"$http_referer" "$http_user_agent" ' - '"$http_range" "$sent_http_content_range"'; - - client_header_timeout 3m; - client_body_timeout 3m; - send_timeout 3m; - - client_header_buffer_size 1k; - large_client_header_buffers 4 4k; - - gzip on; - gzip_min_length 1100; - gzip_buffers 4 8k; - gzip_types text/plain; - - output_buffers 1 32k; - postpone_output 1460; - - sendfile on; - tcp_nopush on; - tcp_nodelay on; - send_lowat 12000; - - keepalive_timeout 75 20; - - upstream websocket { - server jetstream2.us-west.bsky.network:443; - } - - server { - listen 8080; - - access_log /dev/null; - - location / { - proxy_pass https://websocket; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - } - } - -} -``` - -running - -```bash -RUST_LOG=info ./ufos --jetstream ws://192.168.1.139:8080/subscribe --jetstream-force --jetstream-no-zstd --data /mnt/ufos-data-blah/ -``` - -try without info-level logs for better perf - -running on bonilla - -```bash -./ufos --jetstream us-west-2 --jetstream-force --data /mnt/ufos-data-no-compression-2/ -``` - -(reusing data dir from angel-hair) - - -ipv6 is having some trouble. but also maybe there's a deadlock somewhere - -```bash -sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1 -``` - - ---- - -## fuzzing - -got bit by https://github.com/cloudflare/cardinality-estimator/pull/12, so now we have a fuzz target. - -install cargo-fuzz and then - -```bash -RUSTFLAGS="-Z sanitizer=address" cargo +nightly fuzz run cardinality_estimator -``` - -to fuzz the counts value things diff --git a/ufos/src/consumer.rs b/ufos/src/consumer.rs deleted file mode 100644 index 475293d..0000000 --- a/ufos/src/consumer.rs +++ /dev/null @@ -1,241 +0,0 @@ -use crate::store_types::SketchSecretPrefix; -use jetstream::{ - events::{Cursor, EventKind, JetstreamEvent}, - exports::{Did, Nsid}, - DefaultJetstreamEndpoints, JetstreamCompression, JetstreamConfig, JetstreamConnector, - JetstreamReceiver, -}; -use metrics::{ - counter, describe_counter, describe_gauge, describe_histogram, gauge, histogram, Unit, -}; -use std::mem; -use std::time::Duration; -use tokio::sync::mpsc::{channel, Receiver, Sender}; -use tokio::time::{timeout, Interval}; - -use crate::error::{BatchInsertError, FirehoseEventError}; -use crate::{DeleteAccount, EventBatch, UFOsCommit}; - -pub const MAX_BATCHED_RECORDS: usize = 128; // *non-blocking* limit. drops oldest batched record per collection once reached. -pub const MAX_ACCOUNT_REMOVES: usize = 1024; // hard limit, extremely unlikely to reach, but just in case -pub const MAX_BATCHED_COLLECTIONS: usize = 64; // hard limit, MAX_BATCHED_RECORDS applies per-collection -pub const MIN_BATCH_SPAN_SECS: f64 = 2.; // breathe -pub const MAX_BATCH_SPAN_SECS: f64 = 60.; // hard limit, pause consumer if we're unable to send by now -pub const SEND_TIMEOUT_S: f64 = 150.; // if the channel is blocked longer than this, something is probably up -pub const BATCH_QUEUE_SIZE: usize = 64; // used to be 1, but sometimes inserts are just really slow???????? - -pub type LimitedBatch = EventBatch; - -#[derive(Debug, Default)] -struct CurrentBatch { - initial_cursor: Option, - batch: LimitedBatch, -} - -#[derive(Debug)] -pub struct Batcher { - jetstream_receiver: JetstreamReceiver, - batch_sender: Sender, - current_batch: CurrentBatch, - sketch_secret: SketchSecretPrefix, - rate_limit: Interval, -} - -pub async fn consume( - jetstream_endpoint: &str, - cursor: Option, - no_compress: bool, - sketch_secret: SketchSecretPrefix, -) -> anyhow::Result> { - let endpoint = DefaultJetstreamEndpoints::endpoint_or_shortcut(jetstream_endpoint); - if endpoint == jetstream_endpoint { - log::info!("connecting to jetstream at {endpoint}"); - } else { - log::info!("connecting to jetstream at {jetstream_endpoint} => {endpoint}"); - } - let config: JetstreamConfig = JetstreamConfig { - endpoint, - compression: if no_compress { - JetstreamCompression::None - } else { - JetstreamCompression::Zstd - }, - replay_on_reconnect: true, - channel_size: 1024, // buffer up to ~1s of jetstream events - ..Default::default() - }; - let jetstream_receiver = JetstreamConnector::new(config)? - .connect_cursor(cursor) - .await?; - let (batch_sender, batch_reciever) = channel::(BATCH_QUEUE_SIZE); - let mut batcher = Batcher::new(jetstream_receiver, batch_sender, sketch_secret); - tokio::task::spawn(async move { - let r = batcher.run().await; - log::warn!("batcher ended: {r:?}"); - }); - Ok(batch_reciever) -} - -impl Batcher { - pub fn new( - jetstream_receiver: JetstreamReceiver, - batch_sender: Sender, - sketch_secret: SketchSecretPrefix, - ) -> Self { - describe_counter!( - "batcher_batches_sent", - Unit::Count, - "how many batches of events were sent from Batcher to storage" - ); - describe_gauge!( - "batcher_batch_age", - Unit::Microseconds, - "how old the last-sent batch was" - ); - describe_gauge!( - "batcher_send_queue_capacity", - Unit::Count, - "how many spaces are available for batches in the send queue" - ); - describe_histogram!( - "batcher_total_collections", - Unit::Count, - "how many collections are in this batch" - ); - let mut rate_limit = tokio::time::interval(std::time::Duration::from_millis(3)); - rate_limit.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - Self { - jetstream_receiver, - batch_sender, - current_batch: Default::default(), - sketch_secret, - rate_limit, - } - } - - pub async fn run(&mut self) -> anyhow::Result<()> { - // TODO: report errors *from here* probably, since this gets shipped off into a spawned task that might just vanish - loop { - match timeout(Duration::from_secs_f64(30.), self.jetstream_receiver.recv()).await { - Err(_elapsed) => self.no_events_step().await?, - Ok(Some(event)) => self.handle_event(event).await?, - Ok(None) => anyhow::bail!("channel closed"), - } - } - } - - async fn no_events_step(&mut self) -> anyhow::Result<()> { - let empty = self.current_batch.batch.is_empty(); - log::info!("no events received, stepping batcher (empty? {empty})"); - if !empty { - self.send_current_batch_now(true, "no events step").await?; - } - Ok(()) - } - - async fn handle_event(&mut self, event: JetstreamEvent) -> anyhow::Result<()> { - if let Some(earliest) = &self.current_batch.initial_cursor { - if event.cursor.duration_since(earliest)? > Duration::from_secs_f64(MAX_BATCH_SPAN_SECS) - { - self.send_current_batch_now(false, "time since event") - .await?; - } - } else { - self.current_batch.initial_cursor = Some(event.cursor); - } - - match event.kind { - EventKind::Commit => { - let commit = event - .commit - .ok_or(FirehoseEventError::CommitEventMissingCommit)?; - let (commit, nsid) = UFOsCommit::from_commit_info(commit, event.did, event.cursor)?; - self.handle_commit(commit, nsid).await?; - } - EventKind::Account => { - let account = event - .account - .ok_or(FirehoseEventError::AccountEventMissingAccount)?; - if !account.active { - self.handle_delete_account(event.did, event.cursor).await?; - } - } - _ => {} - } - - // if the queue is empty and we have enough, send immediately. otherewise, let the current batch fill up. - if let Some(earliest) = &self.current_batch.initial_cursor { - if event.cursor.duration_since(earliest)?.as_secs_f64() > MIN_BATCH_SPAN_SECS - && self.batch_sender.capacity() == BATCH_QUEUE_SIZE - { - self.send_current_batch_now(true, "available queue").await?; - } - } - Ok(()) - } - - async fn handle_commit(&mut self, commit: UFOsCommit, collection: Nsid) -> anyhow::Result<()> { - let optimistic_res = self.current_batch.batch.insert_commit_by_nsid( - &collection, - commit, - MAX_BATCHED_COLLECTIONS, - &self.sketch_secret, - ); - - if let Err(BatchInsertError::BatchFull(commit)) = optimistic_res { - self.send_current_batch_now(false, "handle commit").await?; - self.current_batch.batch.insert_commit_by_nsid( - &collection, - commit, - MAX_BATCHED_COLLECTIONS, - &self.sketch_secret, - )?; - } else { - optimistic_res?; - } - - Ok(()) - } - - async fn handle_delete_account(&mut self, did: Did, cursor: Cursor) -> anyhow::Result<()> { - if self.current_batch.batch.account_removes.len() >= MAX_ACCOUNT_REMOVES { - self.send_current_batch_now(false, "delete account").await?; - } - self.current_batch - .batch - .account_removes - .push(DeleteAccount { did, cursor }); - Ok(()) - } - - // holds up all consumer progress until it can send to the channel - // use this when the current batch is too full to add more to it - async fn send_current_batch_now(&mut self, small: bool, referrer: &str) -> anyhow::Result<()> { - let size_label = if small { "small" } else { "full" }; - let queue_cap = self.batch_sender.capacity(); - - if let Some(cursor) = self.current_batch.initial_cursor { - gauge!("batcher_batch_age", "size" => size_label).set(cursor.elapsed_micros_f64()); - } - histogram!("batcher_total_collections", "size" => size_label) - .record(self.current_batch.batch.total_collections() as f64); - gauge!("batcher_send_queue_capacity").set(queue_cap as f64); - - let beginning = match self.current_batch.initial_cursor.map(|c| c.elapsed()) { - None => "unknown".to_string(), - Some(Ok(t)) => format!("{t:?}"), - Some(Err(e)) => format!("+{:?}", e.duration()), - }; - log::trace!( - "sending batch now from {beginning}, {size_label}, queue capacity: {queue_cap}, referrer: {referrer}", - ); - let current = mem::take(&mut self.current_batch); - self.rate_limit.tick().await; - self.batch_sender - .send_timeout(current.batch, Duration::from_secs_f64(SEND_TIMEOUT_S)) - .await?; - counter!("batcher_batches_sent", "size" => size_label, "referrer" => referrer.to_string()) - .increment(1); - Ok(()) - } -} diff --git a/ufos/src/db_types.rs b/ufos/src/db_types.rs deleted file mode 100644 index d2f0d75..0000000 --- a/ufos/src/db_types.rs +++ /dev/null @@ -1,570 +0,0 @@ -use crate::{Cursor, Did, Nsid, RecordKey}; -use bincode::{ - config::{standard, Config}, - de::Decode as BincodeDecode, - decode_from_slice, - enc::Encode as BincodeEncode, - encode_to_vec, - error::{DecodeError, EncodeError}, -}; -use lsm_tree::range::prefix_to_range; -use std::fmt; -use std::marker::PhantomData; -use std::ops::{Bound, Range}; -use thiserror::Error; - -#[non_exhaustive] -#[derive(Error, Debug)] -pub enum EncodingError { - #[error("failed to parse Atrium string type: {0}")] - BadAtriumStringType(&'static str), - #[error("Not enough NSID segments for a usable prefix")] - NotEnoughNsidSegments, - #[error("failed to bincode-encode: {0}")] - BincodeEncodeFailed(#[from] EncodeError), - #[error("failed to bincode-decode: {0}")] - BincodeDecodeFailed(#[from] DecodeError), - #[error("decode missing suffix bytes")] - DecodeMissingSuffix, - #[error("decode ran out of bytes")] - DecodeNotEnoughBytes, - #[error("string contained a null byte, which is not allowed, which is annoying, sorry")] - StringContainedNull, - #[error("string was not terminated with null byte")] - UnterminatedString, - #[error("could not convert from utf8: {0}")] - NotUtf8(#[from] std::str::Utf8Error), - #[error("could not convert from utf8: {0}")] - NotUtf8String(#[from] std::string::FromUtf8Error), - #[error("could not get array from slice: {0}")] - BadSlice(#[from] std::array::TryFromSliceError), - #[error("wrong static prefix. expected {1:?}, found {0:?}")] - WrongStaticPrefix(String, String), // found, expected - #[error("failed to deserialize json")] - JsonError(#[from] serde_json::Error), - #[error("unexpected extra bytes ({0} bytes) left after decoding")] - DecodeTooManyBytes(usize), - #[error("expected exclusive bound from lsm_tree (likely bug)")] - BadRangeBound, - #[error("expected a truncated u64 for mod {0}, found remainder: {1}")] - InvalidTruncated(u64, u64), -} - -pub type EncodingResult = Result; - -fn bincode_conf() -> impl Config { - standard() - .with_big_endian() - .with_fixed_int_encoding() - .with_limit::<{ 2_usize.pow(20) }>() // 1MB -} - -pub trait DbBytes { - fn to_db_bytes(&self) -> EncodingResult>; - fn from_db_bytes(bytes: &[u8]) -> Result<(Self, usize), EncodingError> - where - Self: Sized; - fn as_prefix_range_end(&self) -> EncodingResult> { - let bytes = self.to_db_bytes()?; - let (_, Bound::Excluded(range_end)) = prefix_to_range(&bytes) else { - return Err(EncodingError::BadRangeBound); - }; - Ok(range_end.to_vec()) - } -} - -pub trait SubPrefixBytes { - fn sub_prefix(input: T) -> EncodingResult>; -} - -#[derive(PartialEq)] -pub struct DbConcat { - pub prefix: P, - pub suffix: S, -} - -impl - DbConcat -{ - pub fn from_pair(prefix: P, suffix: S) -> Self { - Self { prefix, suffix } - } - pub fn from_prefix_to_db_bytes(prefix: &P) -> EncodingResult> { - prefix.to_db_bytes() - } - pub fn to_prefix_db_bytes(&self) -> EncodingResult> { - self.prefix.to_db_bytes() - } - pub fn prefix_range_end(prefix: &P) -> EncodingResult> { - prefix.as_prefix_range_end() - } - pub fn range_end(&self) -> EncodingResult> { - Self::prefix_range_end(&self.prefix) - } - pub fn range(&self) -> Result>, EncodingError> { - let prefix_bytes = self.prefix.to_db_bytes()?; - let (Bound::Included(start), Bound::Excluded(end)) = prefix_to_range(&prefix_bytes) else { - return Err(EncodingError::BadRangeBound); - }; - Ok(start.to_vec()..end.to_vec()) - } - pub fn range_to_prefix_end(&self) -> Result>, EncodingError> { - Ok(self.to_db_bytes()?..self.range_end()?) - } -} - -impl Default for DbConcat { - fn default() -> Self { - Self { - prefix: Default::default(), - suffix: Default::default(), - } - } -} - -impl fmt::Debug for DbConcat { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "DbConcat<{:?} || {:?}>", self.prefix, self.suffix) - } -} - -impl DbBytes for DbConcat { - fn to_db_bytes(&self) -> EncodingResult> { - let mut combined = self.prefix.to_db_bytes()?; - combined.append(&mut self.suffix.to_db_bytes()?); - Ok(combined) - } - fn from_db_bytes(bytes: &[u8]) -> Result<(Self, usize), EncodingError> - where - Self: Sized, - { - let (prefix, eaten) = P::from_db_bytes(bytes)?; - assert!( - eaten <= bytes.len(), - "eaten({}) < len({})", - eaten, - bytes.len() - ); - let Some(suffix_bytes) = bytes.get(eaten..) else { - return Err(EncodingError::DecodeMissingSuffix); - }; - if suffix_bytes.is_empty() { - return Err(EncodingError::DecodeMissingSuffix); - }; - let (suffix, also_eaten) = S::from_db_bytes(suffix_bytes)?; - assert!( - also_eaten <= suffix_bytes.len(), - "also eaten({}) < suffix len({})", - also_eaten, - suffix_bytes.len() - ); - Ok((Self { prefix, suffix }, eaten + also_eaten)) - } -} - -#[derive(Debug, Default, PartialEq)] -pub struct DbEmpty(()); -impl DbBytes for DbEmpty { - fn to_db_bytes(&self) -> EncodingResult> { - Ok(vec![]) - } - fn from_db_bytes(_: &[u8]) -> Result<(Self, usize), EncodingError> { - Ok((Self(()), 0)) - } -} - -pub trait StaticStr { - fn static_str() -> &'static str; -} - -#[derive(PartialEq)] -pub struct DbStaticStr { - marker: PhantomData, -} -impl Default for DbStaticStr { - fn default() -> Self { - Self { - marker: PhantomData, - } - } -} -impl fmt::Debug for DbStaticStr { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "DbStaticStr({:?})", S::static_str()) - } -} -impl DbBytes for DbStaticStr { - fn to_db_bytes(&self) -> EncodingResult> { - S::static_str().to_string().to_db_bytes() - } - fn from_db_bytes(bytes: &[u8]) -> Result<(Self, usize), EncodingError> { - let (prefix, eaten) = String::from_db_bytes(bytes)?; - if prefix != S::static_str() { - return Err(EncodingError::WrongStaticPrefix( - prefix, - S::static_str().to_string(), - )); - } - Ok(( - Self { - marker: PhantomData, - }, - eaten, - )) - } -} - -/// marker trait: impl on a type to indicate that that DbBytes should use bincode on it -pub trait UseBincodePlz {} - -impl DbBytes for T -where - T: BincodeEncode + BincodeDecode<()> + UseBincodePlz + Sized + std::fmt::Debug, -{ - fn to_db_bytes(&self) -> EncodingResult> { - Ok(encode_to_vec(self, bincode_conf())?) - } - fn from_db_bytes(bytes: &[u8]) -> Result<(Self, usize), EncodingError> { - Ok(decode_from_slice(bytes, bincode_conf())?) - } -} - -/// helper trait: impl on a type to get helpers to implement DbBytes -pub trait SerdeBytes: serde::Serialize + for<'a> serde::Deserialize<'a> { - fn to_bytes(&self) -> EncodingResult> - where - Self: std::fmt::Debug, - { - Ok(bincode::serde::encode_to_vec(self, bincode_conf())?) - } - fn from_bytes(bytes: &[u8]) -> Result<(Self, usize), EncodingError> { - Ok(bincode::serde::decode_from_slice(bytes, bincode_conf())?) - } -} - -////// - -impl UseBincodePlz for [u8; N] {} - -// bare bytes (NOT prefix-encoded!) -impl DbBytes for Vec { - fn to_db_bytes(&self) -> EncodingResult> { - Ok(self.to_vec()) - } - // greedy, consumes ALL remaining bytes - fn from_db_bytes(bytes: &[u8]) -> Result<(Self, usize), EncodingError> { - Ok((bytes.to_owned(), bytes.len())) - } -} - -/// Lexicographic-sort-friendly null-terminating serialization for String -/// -/// Null bytes technically can appear within utf-8 strings. Currently we will just bail in that case. -/// -/// In the future, null bytes could be escaped, or maybe this becomes SLIP-encoded. Either should be -/// backwards-compatible I think. -/// -/// TODO: wrap in another type. it's actually probably not desirable to serialize strings this way -/// *except* where needed as a prefix. -impl DbBytes for String { - fn to_db_bytes(&self) -> EncodingResult> { - let mut v = self.as_bytes().to_vec(); - if v.contains(&0x00) { - return Err(EncodingError::StringContainedNull); - } - v.push(0x00); - Ok(v) - } - fn from_db_bytes(bytes: &[u8]) -> Result<(Self, usize), EncodingError> { - for (i, byte) in bytes.iter().enumerate() { - if *byte == 0x00 { - let (string_bytes, _) = bytes.split_at(i); - let s = std::str::from_utf8(string_bytes)?; - return Ok((s.to_string(), i + 1)); // +1 for the null byte - } - } - Err(EncodingError::UnterminatedString) - } -} - -impl SubPrefixBytes<&str> for String { - fn sub_prefix(input: &str) -> EncodingResult> { - let v = input.as_bytes(); - if v.contains(&0x00) { - return Err(EncodingError::StringContainedNull); - } - // NO null terminator!! - Ok(v.to_vec()) - } -} - -impl DbBytes for Did { - fn from_db_bytes(bytes: &[u8]) -> Result<(Self, usize), EncodingError> { - let (s, n) = decode_from_slice(bytes, bincode_conf())?; - let me = Self::new(s).map_err(EncodingError::BadAtriumStringType)?; - Ok((me, n)) - } - fn to_db_bytes(&self) -> EncodingResult> { - Ok(encode_to_vec(self.as_ref(), bincode_conf())?) - } -} - -impl DbBytes for Nsid { - fn from_db_bytes(bytes: &[u8]) -> Result<(Self, usize), EncodingError> { - let (s, n) = String::from_db_bytes(bytes)?; // null-terminated DbBytes impl!! - let me = Self::new(s).map_err(EncodingError::BadAtriumStringType)?; - Ok((me, n)) - } - fn to_db_bytes(&self) -> EncodingResult> { - String::to_db_bytes(&self.to_string()) // null-terminated DbBytes impl!!!! - } -} -impl SubPrefixBytes<&str> for Nsid { - fn sub_prefix(input: &str) -> EncodingResult> { - String::sub_prefix(input) - } -} - -impl DbBytes for RecordKey { - fn from_db_bytes(bytes: &[u8]) -> Result<(Self, usize), EncodingError> { - let (s, n) = decode_from_slice(bytes, bincode_conf())?; - let me = Self::new(s).map_err(EncodingError::BadAtriumStringType)?; - Ok((me, n)) - } - fn to_db_bytes(&self) -> EncodingResult> { - Ok(encode_to_vec(self.as_ref(), bincode_conf())?) - } -} - -impl DbBytes for Cursor { - fn to_db_bytes(&self) -> EncodingResult> { - Ok(self.to_raw_u64().to_be_bytes().to_vec()) - } - fn from_db_bytes(bytes: &[u8]) -> Result<(Self, usize), EncodingError> { - if bytes.len() < 8 { - return Err(EncodingError::DecodeNotEnoughBytes); - } - let bytes8 = TryInto::<[u8; 8]>::try_into(&bytes[..8])?; - let cursor = Cursor::from_raw_u64(u64::from_be_bytes(bytes8)); - Ok((cursor, 8)) - } -} - -impl DbBytes for serde_json::Value { - fn to_db_bytes(&self) -> EncodingResult> { - self.to_string().to_db_bytes() - } - fn from_db_bytes(bytes: &[u8]) -> Result<(Self, usize), EncodingError> { - let (s, n) = String::from_db_bytes(bytes)?; - let v = s.parse()?; - Ok((v, n)) - } -} - -pub fn db_complete(bytes: &[u8]) -> Result { - let (t, n) = T::from_db_bytes(bytes)?; - if n < bytes.len() { - return Err(EncodingError::DecodeTooManyBytes(bytes.len() - n)); - } - Ok(t) -} - -#[cfg(test)] -mod test { - use super::{ - Cursor, DbBytes, DbConcat, DbEmpty, DbStaticStr, EncodingResult, Nsid, StaticStr, - SubPrefixBytes, - }; - - #[test] - fn test_db_empty() -> EncodingResult<()> { - let original = DbEmpty::default(); - let serialized = original.to_db_bytes()?; - assert_eq!(serialized.len(), 0); - let (restored, bytes_consumed) = DbEmpty::from_db_bytes(&serialized)?; - assert_eq!(restored, original); - assert_eq!(bytes_consumed, 0); - Ok(()) - } - - #[test] - fn test_string_roundtrip() -> EncodingResult<()> { - for (case, desc) in [ - ("", "empty string"), - ("a", "basic string"), - ("asdf asdf asdf even µnicode", "unicode string"), - ] { - let serialized = case.to_string().to_db_bytes()?; - let (restored, bytes_consumed) = String::from_db_bytes(&serialized)?; - assert_eq!(&restored, case, "string round-trip: {desc}"); - assert_eq!( - bytes_consumed, - serialized.len(), - "exact bytes consumed for round-trip: {desc}" - ); - } - Ok(()) - } - - #[test] - fn test_string_serialized_lexicographic_sort() -> EncodingResult<()> { - let aa = "aa".to_string().to_db_bytes()?; - let b = "b".to_string().to_db_bytes()?; - assert!(b > aa); - Ok(()) - } - - #[test] - fn test_nullstring_can_prefix() -> EncodingResult<()> { - for (s, pre, is_pre, desc) in [ - ("", "", true, "empty strings"), - ("", "a", false, "longer prefix"), - ("a", "", true, "empty prefix matches"), - ("a", "a", true, "whole string matches"), - ("a", "b", false, "entirely different"), - ("ab", "a", true, "prefix matches"), - ("ab", "b", false, "shorter and entirely different"), - ] { - let serialized = s.to_string().to_db_bytes()?; - let prefixed = String::sub_prefix(pre)?; - assert_eq!(serialized.starts_with(&prefixed), is_pre, "{desc}"); - } - Ok(()) - } - - #[test] - fn test_nsid_can_prefix() -> EncodingResult<()> { - for (s, pre, is_pre, desc) in [ - ("ab.cd.ef", "", true, "empty prefix"), - ("ab.cd.ef", "a", true, "tiny prefix"), - ("ab.cd.ef", "abc", false, "bad prefix"), - ("ab.cd.ef", "ab", true, "segment prefix"), - ("ab.cd.ef", "ab.cd", true, "multi-segment prefix"), - ("ab.cd.ef", "ab.cd.ef", true, "full match"), - ("ab.cd.ef", "ab.cd.ef.g", false, "prefix longer"), - ] { - let serialized = Nsid::new(s.to_string()).unwrap().to_db_bytes()?; - let prefixed = Nsid::sub_prefix(pre)?; - assert_eq!(serialized.starts_with(&prefixed), is_pre, "{desc}"); - } - Ok(()) - } - - #[test] - fn test_string_cursor_prefix_roundtrip() -> EncodingResult<()> { - type TwoThings = DbConcat; - for (lazy_prefix, tired_suffix, desc) in [ - ("", 0, "empty string and cursor"), - ("aaa", 0, "zero-cursor"), - ("", 1234, "empty string"), - ("aaaaa", 789, "string and cursor"), - ] { - let original = TwoThings { - prefix: lazy_prefix.to_string(), - suffix: Cursor::from_raw_u64(tired_suffix), - }; - let serialized = original.to_db_bytes()?; - let (restored, bytes_consumed) = TwoThings::from_db_bytes(&serialized)?; - assert_eq!(restored, original, "round-trip: {desc}"); - assert_eq!( - bytes_consumed, - serialized.len(), - "exact bytes consumed for round-trip: {desc}" - ); - } - Ok(()) - } - - #[test] - fn test_cursor_string_prefix_roundtrip() -> EncodingResult<()> { - type TwoThings = DbConcat; - for (tired_prefix, sad_suffix, desc) in [ - (0, "", "empty string and cursor"), - (0, "aaa", "zero-cursor"), - (1234, "", "empty string"), - (789, "aaaaa", "string and cursor"), - ] { - let original = TwoThings { - prefix: Cursor::from_raw_u64(tired_prefix), - suffix: sad_suffix.to_string(), - }; - let serialized = original.to_db_bytes()?; - let (restored, bytes_consumed) = TwoThings::from_db_bytes(&serialized)?; - assert_eq!(restored, original, "round-trip: {desc}"); - assert_eq!( - bytes_consumed, - serialized.len(), - "exact bytes consumed for round-trip: {desc}" - ); - } - Ok(()) - } - - #[test] - fn test_static_str() -> EncodingResult<()> { - #[derive(Debug, PartialEq)] - struct AStaticStr {} - impl StaticStr for AStaticStr { - fn static_str() -> &'static str { - "a static str" - } - } - type ADbStaticStr = DbStaticStr; - - let original = ADbStaticStr::default(); - let serialized = original.to_db_bytes()?; - let (restored, bytes_consumed) = ADbStaticStr::from_db_bytes(&serialized)?; - assert_eq!(restored, original); - assert_eq!(bytes_consumed, serialized.len()); - assert!(serialized.starts_with("a static str".as_bytes())); - - Ok(()) - } - - #[test] - fn test_static_str_empty() -> EncodingResult<()> { - #[derive(Debug, PartialEq)] - struct AnEmptyStr {} - impl StaticStr for AnEmptyStr { - fn static_str() -> &'static str { - "" - } - } - type ADbEmptyStr = DbStaticStr; - let original = ADbEmptyStr::default(); - let serialized = original.to_db_bytes()?; - let (restored, bytes_consumed) = ADbEmptyStr::from_db_bytes(&serialized)?; - assert_eq!(restored, original); - assert_eq!(bytes_consumed, serialized.len()); - assert_eq!(serialized, &[0x00]); - - Ok(()) - } - - #[test] - fn test_static_prefix() -> EncodingResult<()> { - #[derive(Debug, PartialEq)] - struct AStaticPrefix {} - impl StaticStr for AStaticPrefix { - fn static_str() -> &'static str { - "a static prefix" - } - } - type ADbStaticPrefix = DbStaticStr; - - type PrefixedCursor = DbConcat; - - let original = PrefixedCursor { - prefix: Default::default(), - suffix: Cursor::from_raw_u64(123), - }; - let serialized = original.to_db_bytes()?; - let (restored, bytes_consumed) = PrefixedCursor::from_db_bytes(&serialized)?; - assert_eq!(restored, original); - assert_eq!(bytes_consumed, serialized.len()); - assert_eq!(restored.suffix.to_raw_u64(), 123); - assert!(serialized.starts_with("a static prefix".as_bytes())); - - Ok(()) - } -} diff --git a/ufos/src/error.rs b/ufos/src/error.rs deleted file mode 100644 index 012b61d..0000000 --- a/ufos/src/error.rs +++ /dev/null @@ -1,45 +0,0 @@ -use crate::db_types::EncodingError; -use crate::UFOsCommit; -use thiserror::Error; - -#[derive(Debug, Error)] -pub enum FirehoseEventError { - #[error("Create/Update commit operation missing record data")] - CruMissingRecord, - #[error("Account event missing account info")] - AccountEventMissingAccount, - #[error("Commit event missing commit info")] - CommitEventMissingCommit, -} - -#[derive(Debug, Error)] -pub enum BatchInsertError { - #[error("Batch is full and no creates are left to be truncated")] - BatchFull(UFOsCommit), - #[error("Bug: tried to index beyond batch limit: {0}")] - BatchOverflow(usize), - #[error("Bug: non-terminating head advancement??")] - BatchForever, -} - -#[derive(Debug, Error)] -pub enum StorageError { - #[error("Failed to initialize: {0}")] - InitError(String), - #[error("DB seems to be in a bad state: {0}")] - BadStateError(String), - #[error("Fjall error")] - FjallError(#[from] fjall::Error), - #[error("LSM-tree error (from fjall)")] - FjallLsmError(#[from] fjall::LsmError), - #[error("Bytes encoding error")] - EncodingError(#[from] EncodingError), - #[error("If you ever see this, there's a bug in the code. The error was stolen")] - Stolen, - #[error("Failed to join tokio task: {0}")] - JoinError(#[from] tokio::task::JoinError), - #[error("Background task already started")] - BackgroundAlreadyStarted, - #[error("Batch sender exited")] - BatchSenderExited, -} diff --git a/ufos/src/file_consumer.rs b/ufos/src/file_consumer.rs deleted file mode 100644 index 375ba0a..0000000 --- a/ufos/src/file_consumer.rs +++ /dev/null @@ -1,78 +0,0 @@ -use crate::consumer::{Batcher, LimitedBatch, BATCH_QUEUE_SIZE}; -use crate::store_types::SketchSecretPrefix; -use crate::Cursor; -use anyhow::Result; -use jetstream::{error::JetstreamEventError, events::JetstreamEvent}; -use std::path::PathBuf; -use tokio::{ - fs::File, - io::{AsyncBufReadExt, BufReader}, - sync::mpsc::{channel, Receiver, Sender}, -}; - -async fn read_jsonl(f: File, sender: Sender, cursor: Option) -> Result<()> { - let mut lines = BufReader::new(f).lines(); - if let Some(db_cursor) = cursor { - log::info!("jsonl fixture: skipping events before cursor {db_cursor:?}"); - let mut bad_lines = 0; - let mut skipped = 0; - while let Some(line) = lines.next_line().await? { - let Ok(event) = serde_json::from_str::(&line) else { - bad_lines += 1; - continue; - }; - if event.cursor < db_cursor { - skipped += 1; - continue; - } - if event.cursor == db_cursor { - log::info!("jsonl fixture: found existing db cursor! skipped {skipped} old events and failed parsing {bad_lines} lines"); - break; - } - anyhow::bail!("jsonl fixture: did not find existing db cursor, found event cursor {:?} which is newer. bailing.", event.cursor); - } - } else { - log::info!("jsonl fixture: no cursor provided, sending every event"); - } - - log::info!("jsonl fixture: now sending events"); - while let Some(line) = lines.next_line().await? { - match serde_json::from_str::(&line) { - Ok(event) => match sender.send(event).await { - Ok(_) => {} - Err(e) => { - log::warn!("All receivers for the jsonl fixture have been dropped, bye: {e:?}"); - return Err(JetstreamEventError::ReceiverClosedError.into()); - } - }, - Err(parse_err) => { - log::warn!("failed to parse event: {parse_err:?} from event:\n{line}"); - continue; - } - } - } - log::info!("reached end of jsonl file, looping on noop to keep server alive."); - loop { - tokio::time::sleep(std::time::Duration::from_secs_f64(10.)).await; - } -} - -pub async fn consume( - p: PathBuf, - sketch_secret: SketchSecretPrefix, - cursor: Option, -) -> Result> { - let f = File::open(p).await?; - let (jsonl_sender, jsonl_receiver) = channel::(16); - let (batch_sender, batch_reciever) = channel::(BATCH_QUEUE_SIZE); - let mut batcher = Batcher::new(jsonl_receiver, batch_sender, sketch_secret); - tokio::task::spawn(async move { - let r = read_jsonl(f, jsonl_sender, cursor).await; - log::warn!("read_jsonl finished: {r:?}"); - }); - tokio::task::spawn(async move { - let r = batcher.run().await; - log::warn!("batcher finished: {r:?}"); - }); - Ok(batch_reciever) -} diff --git a/ufos/src/index_html.rs b/ufos/src/index_html.rs deleted file mode 100644 index 051b0fd..0000000 --- a/ufos/src/index_html.rs +++ /dev/null @@ -1,54 +0,0 @@ -pub const INDEX_HTML: &str = r#" - - - - UFOs API documentation - - - - - -
-

- Launch 🛸 UFOs app: Explore lexicons -

- -
- - - - - - - - -"#; diff --git a/ufos/src/lib.rs b/ufos/src/lib.rs deleted file mode 100644 index fbc5f3a..0000000 --- a/ufos/src/lib.rs +++ /dev/null @@ -1,617 +0,0 @@ -pub mod consumer; -pub mod db_types; -pub mod error; -pub mod file_consumer; -pub mod index_html; -pub mod server; -pub mod storage; -pub mod storage_fjall; -pub mod store_types; - -use crate::db_types::{EncodingError, EncodingResult}; -use crate::error::BatchInsertError; -use crate::store_types::{CountsValue, SketchSecretPrefix}; -use cardinality_estimator_safe::{Element, Sketch}; -use error::FirehoseEventError; -use jetstream::events::{CommitEvent, CommitOp, Cursor}; -use jetstream::exports::{Did, Nsid, RecordKey}; -use schemars::JsonSchema; -use serde::Serialize; -use serde_json::value::RawValue; -use sha2::Sha256; -use std::collections::HashMap; -use std::time::Duration; - -fn did_element(sketch_secret: &SketchSecretPrefix, did: &Did) -> Element<14> { - Element::from_digest_with_prefix::(sketch_secret, did.as_bytes()) -} - -pub fn nice_duration(dt: Duration) -> String { - let secs = dt.as_secs_f64(); - if secs < 1. { - return format!("{:.0}ms", secs * 1000.); - } - if secs < 60. { - return format!("{secs:.02}s"); - } - let mins = (secs / 60.).floor(); - let rsecs = secs - (mins * 60.); - if mins < 60. { - return format!("{mins:.0}m{rsecs:.0}s"); - } - let hrs = (mins / 60.).floor(); - let rmins = mins - (hrs * 60.); - if hrs < 24. { - return format!("{hrs:.0}h{rmins:.0}m{rsecs:.0}s"); - } - let days = (hrs / 24.).floor(); - let rhrs = hrs - (days * 24.); - format!("{days:.0}d{rhrs:.0}h{rmins:.0}m{rsecs:.0}s") -} - -#[derive(Debug, Default, Clone)] -pub struct CollectionCommits { - pub creates: usize, - pub updates: usize, - pub deletes: usize, - pub dids_estimate: Sketch<14>, - pub commits: Vec, - head: usize, -} - -impl CollectionCommits { - fn advance_head(&mut self) { - self.head += 1; - if self.head > LIMIT { - self.head = 0; - } - } - /// lossy-ish commit insertion - /// - /// - new commits are *always* added to the batch or else rejected as full. - /// - when LIMIT is reached, new commits can displace existing `creates`. - /// `update`s and `delete`s are *never* displaced. - /// - if all batched `creates` have been displaced, the batch is full. - /// - /// in general it's rare for commits to be displaced except for very high- - /// volume collections such as `app.bsky.feed.like`. - /// - /// it could be nice in the future to retain all batched commits and just - /// drop new `creates` after a limit instead. - pub fn truncating_insert( - &mut self, - commit: UFOsCommit, - sketch_secret: &SketchSecretPrefix, - ) -> Result<(), BatchInsertError> { - if (self.updates + self.deletes) == LIMIT { - // nothing can be displaced (only `create`s may be displaced) - return Err(BatchInsertError::BatchFull(commit)); - } - - // every kind of commit counts as "user activity" - self.dids_estimate - .insert(did_element(sketch_secret, &commit.did)); - - match commit.action { - CommitAction::Put(PutAction { - is_update: false, .. - }) => { - self.creates += 1; - } - CommitAction::Put(PutAction { - is_update: true, .. - }) => { - self.updates += 1; - } - CommitAction::Cut => { - self.deletes += 1; - } - } - - if self.commits.len() < LIMIT { - // normal insert: there's space left to put a new commit at the end - self.commits.push(commit); - } else { - // displacement insert: find an old `create` we can displace - let head_started_at = self.head; - loop { - let candidate = self - .commits - .get_mut(self.head) - .ok_or(BatchInsertError::BatchOverflow(self.head))?; - if candidate.action.is_create() { - *candidate = commit; - break; - } - self.advance_head(); - if self.head == head_started_at { - return Err(BatchInsertError::BatchForever); - } - } - } - - Ok(()) - } -} - -#[derive(Debug, Clone)] -pub struct DeleteAccount { - pub did: Did, - pub cursor: Cursor, -} - -#[derive(Debug, Clone)] -pub enum CommitAction { - Put(PutAction), - Cut, -} -impl CommitAction { - pub fn is_create(&self) -> bool { - match self { - CommitAction::Put(PutAction { is_update, .. }) => !is_update, - CommitAction::Cut => false, - } - } -} - -#[derive(Debug, Clone)] -pub struct PutAction { - record: Box, - is_update: bool, -} - -#[derive(Debug, Clone)] -pub struct UFOsCommit { - cursor: Cursor, - did: Did, - rkey: RecordKey, - rev: String, - action: CommitAction, -} - -#[derive(Debug, Clone, Serialize)] -pub struct UFOsRecord { - pub cursor: Cursor, - pub did: Did, - pub collection: Nsid, - pub rkey: RecordKey, - pub rev: String, - // TODO: cid? - pub record: Box, - pub is_update: bool, -} - -impl UFOsCommit { - pub fn from_commit_info( - commit: CommitEvent, - did: Did, - cursor: Cursor, - ) -> Result<(Self, Nsid), FirehoseEventError> { - let action = match commit.operation { - CommitOp::Delete => CommitAction::Cut, - cru => CommitAction::Put(PutAction { - record: commit.record.ok_or(FirehoseEventError::CruMissingRecord)?, - is_update: cru == CommitOp::Update, - }), - }; - let batched = Self { - cursor, - did, - rkey: commit.rkey, - rev: commit.rev, - action, - }; - Ok((batched, commit.collection)) - } -} - -#[derive(Debug, Default, Clone)] -pub struct EventBatch { - pub commits_by_nsid: HashMap>, - pub account_removes: Vec, -} - -impl EventBatch { - pub fn insert_commit_by_nsid( - &mut self, - collection: &Nsid, - commit: UFOsCommit, - max_collections: usize, - sketch_secret: &SketchSecretPrefix, - ) -> Result<(), BatchInsertError> { - let map = &mut self.commits_by_nsid; - if !map.contains_key(collection) && map.len() >= max_collections { - return Err(BatchInsertError::BatchFull(commit)); - } - map.entry(collection.clone()) - .or_default() - .truncating_insert(commit, sketch_secret)?; - Ok(()) - } - pub fn total_collections(&self) -> usize { - self.commits_by_nsid.len() - } - pub fn account_removes(&self) -> usize { - self.account_removes.len() - } - pub fn estimate_dids(&self) -> usize { - let mut estimator = Sketch::<14>::default(); - for commits in self.commits_by_nsid.values() { - estimator.merge(&commits.dids_estimate); - } - estimator.estimate() - } - pub fn latest_cursor(&self) -> Option { - let mut oldest = Cursor::from_start(); - for commits in self.commits_by_nsid.values() { - for commit in &commits.commits { - if commit.cursor > oldest { - oldest = commit.cursor; - } - } - } - if let Some(del) = self.account_removes.last() { - if del.cursor > oldest { - oldest = del.cursor; - } - } - if oldest > Cursor::from_start() { - Some(oldest) - } else { - None - } - } - pub fn is_empty(&self) -> bool { - self.commits_by_nsid.is_empty() && self.account_removes.is_empty() - } -} - -#[derive(Debug, Serialize, JsonSchema)] -#[serde(rename_all = "camelCase")] -pub enum ConsumerInfo { - Jetstream { - endpoint: String, - started_at: u64, - latest_cursor: Option, - rollup_cursor: Option, - }, -} - -#[derive(Debug, PartialEq, Serialize, JsonSchema)] -pub struct NsidCount { - nsid: String, - creates: u64, - updates: u64, - deletes: u64, - dids_estimate: u64, -} -impl NsidCount { - pub fn new(nsid: &Nsid, counts: &CountsValue) -> Self { - let crud = counts.counts(); - Self { - nsid: nsid.to_string(), - creates: crud.creates, - updates: crud.updates, - deletes: crud.deletes, - dids_estimate: counts.dids().estimate() as u64, - } - } -} - -#[derive(Debug, PartialEq, Serialize, JsonSchema)] -pub struct PrefixCount { - prefix: String, - creates: u64, - updates: u64, - deletes: u64, - dids_estimate: u64, -} -impl PrefixCount { - pub fn new(prefix: &str, counts: &CountsValue) -> Self { - let crud = counts.counts(); - Self { - prefix: prefix.to_string(), - creates: crud.creates, - updates: crud.updates, - deletes: crud.deletes, - dids_estimate: counts.dids().estimate() as u64, - } - } -} - -#[derive(Debug, PartialEq, Serialize, JsonSchema)] -#[serde(tag = "type", rename_all = "camelCase")] -pub enum PrefixChild { - Collection(NsidCount), - Prefix(PrefixCount), -} - -#[derive(Debug, Serialize, JsonSchema)] -pub struct NsidPrefix(String); -impl NsidPrefix { - /// Input must not include a trailing dot. - pub fn new(pre: &str) -> EncodingResult { - // it's a valid prefix if appending `.name` makes it a valid NSID - Nsid::new(format!("{pre}.name")).map_err(EncodingError::BadAtriumStringType)?; - // hack (shouldn't really be here): reject prefixes that aren't at least 2 segments long - if !pre.contains('.') { - return Err(EncodingError::NotEnoughNsidSegments); - } - Ok(Self(pre.to_string())) - } - pub fn is_group_of(&self, other: &Nsid) -> bool { - assert!( - other.as_str().starts_with(&self.0), - "must be a prefix of other" - ); - self.0 == other.domain_authority() - } - /// The prefix as initialized (no trailing dot) - pub fn as_str(&self) -> &str { - self.0.as_str() - } - /// The prefix with a trailing `.` appended to avoid matching a longer segment - pub fn terminated(&self) -> String { - format!("{}.", self.0) - } -} - -#[derive(Debug, Serialize, JsonSchema)] -pub struct JustCount { - creates: u64, - updates: u64, - deletes: u64, - dids_estimate: u64, -} - -#[derive(Debug)] -pub enum OrderCollectionsBy { - Lexi { cursor: Option> }, - RecordsCreated, - DidsEstimate, -} -impl Default for OrderCollectionsBy { - fn default() -> Self { - Self::Lexi { cursor: None } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_truncating_insert_truncates() -> anyhow::Result<()> { - let mut commits: CollectionCommits<2> = Default::default(); - - commits.truncating_insert( - UFOsCommit { - cursor: Cursor::from_raw_u64(100), - did: Did::new("did:plc:whatever".to_string()).unwrap(), - rkey: RecordKey::new("rkey-asdf-a".to_string()).unwrap(), - rev: "rev-asdf".to_string(), - action: CommitAction::Put(PutAction { - record: RawValue::from_string("{}".to_string())?, - is_update: false, - }), - }, - &[0u8; 16], - )?; - - commits.truncating_insert( - UFOsCommit { - cursor: Cursor::from_raw_u64(101), - did: Did::new("did:plc:whatever".to_string()).unwrap(), - rkey: RecordKey::new("rkey-asdf-b".to_string()).unwrap(), - rev: "rev-asdg".to_string(), - action: CommitAction::Put(PutAction { - record: RawValue::from_string("{}".to_string())?, - is_update: false, - }), - }, - &[0u8; 16], - )?; - - commits.truncating_insert( - UFOsCommit { - cursor: Cursor::from_raw_u64(102), - did: Did::new("did:plc:whatever".to_string()).unwrap(), - rkey: RecordKey::new("rkey-asdf-c".to_string()).unwrap(), - rev: "rev-asdh".to_string(), - action: CommitAction::Put(PutAction { - record: RawValue::from_string("{}".to_string())?, - is_update: false, - }), - }, - &[0u8; 16], - )?; - - assert_eq!(commits.creates, 3); - assert_eq!(commits.dids_estimate.estimate(), 1); - assert_eq!(commits.commits.len(), 2); - - let mut found_first = false; - let mut found_last = false; - for commit in commits.commits { - match commit.rev.as_ref() { - "rev-asdf" => { - found_first = true; - } - "rev-asdh" => { - found_last = true; - } - _ => {} - } - } - assert!(!found_first); - assert!(found_last); - - Ok(()) - } - - #[test] - fn test_truncating_insert_counts_updates() -> anyhow::Result<()> { - let mut commits: CollectionCommits<2> = Default::default(); - - commits.truncating_insert( - UFOsCommit { - cursor: Cursor::from_raw_u64(100), - did: Did::new("did:plc:whatever".to_string()).unwrap(), - rkey: RecordKey::new("rkey-asdf-a".to_string()).unwrap(), - rev: "rev-asdf".to_string(), - action: CommitAction::Put(PutAction { - record: RawValue::from_string("{}".to_string())?, - is_update: true, - }), - }, - &[0u8; 16], - )?; - - assert_eq!(commits.creates, 0); - assert_eq!(commits.updates, 1); - assert_eq!(commits.deletes, 0); - assert_eq!(commits.dids_estimate.estimate(), 1); - assert_eq!(commits.commits.len(), 1); - Ok(()) - } - - #[test] - fn test_truncating_insert_does_not_truncate_deletes() -> anyhow::Result<()> { - let mut commits: CollectionCommits<2> = Default::default(); - - commits.truncating_insert( - UFOsCommit { - cursor: Cursor::from_raw_u64(100), - did: Did::new("did:plc:whatever".to_string()).unwrap(), - rkey: RecordKey::new("rkey-asdf-a".to_string()).unwrap(), - rev: "rev-asdf".to_string(), - action: CommitAction::Cut, - }, - &[0u8; 16], - )?; - - commits.truncating_insert( - UFOsCommit { - cursor: Cursor::from_raw_u64(101), - did: Did::new("did:plc:whatever".to_string()).unwrap(), - rkey: RecordKey::new("rkey-asdf-b".to_string()).unwrap(), - rev: "rev-asdg".to_string(), - action: CommitAction::Put(PutAction { - record: RawValue::from_string("{}".to_string())?, - is_update: false, - }), - }, - &[0u8; 16], - )?; - - commits.truncating_insert( - UFOsCommit { - cursor: Cursor::from_raw_u64(102), - did: Did::new("did:plc:whatever".to_string()).unwrap(), - rkey: RecordKey::new("rkey-asdf-c".to_string()).unwrap(), - rev: "rev-asdh".to_string(), - action: CommitAction::Put(PutAction { - record: RawValue::from_string("{}".to_string())?, - is_update: false, - }), - }, - &[0u8; 16], - )?; - - assert_eq!(commits.creates, 2); - assert_eq!(commits.deletes, 1); - assert_eq!(commits.dids_estimate.estimate(), 1); - assert_eq!(commits.commits.len(), 2); - - let mut found_first = false; - let mut found_last = false; - let mut found_delete = false; - for commit in commits.commits { - match commit.rev.as_ref() { - "rev-asdg" => { - found_first = true; - } - "rev-asdh" => { - found_last = true; - } - _ => {} - } - if let CommitAction::Cut = commit.action { - found_delete = true; - } - } - assert!(!found_first); - assert!(found_last); - assert!(found_delete); - - Ok(()) - } - - #[test] - fn test_truncating_insert_maxes_out_deletes() -> anyhow::Result<()> { - let mut commits: CollectionCommits<2> = Default::default(); - - commits - .truncating_insert( - UFOsCommit { - cursor: Cursor::from_raw_u64(100), - did: Did::new("did:plc:whatever".to_string()).unwrap(), - rkey: RecordKey::new("rkey-asdf-a".to_string()).unwrap(), - rev: "rev-asdf".to_string(), - action: CommitAction::Cut, - }, - &[0u8; 16], - ) - .unwrap(); - - // this create will just be discarded - commits - .truncating_insert( - UFOsCommit { - cursor: Cursor::from_raw_u64(80), - did: Did::new("did:plc:whatever".to_string()).unwrap(), - rkey: RecordKey::new("rkey-asdf-zzz".to_string()).unwrap(), - rev: "rev-asdzzz".to_string(), - action: CommitAction::Put(PutAction { - record: RawValue::from_string("{}".to_string())?, - is_update: false, - }), - }, - &[0u8; 16], - ) - .unwrap(); - - commits - .truncating_insert( - UFOsCommit { - cursor: Cursor::from_raw_u64(101), - did: Did::new("did:plc:whatever".to_string()).unwrap(), - rkey: RecordKey::new("rkey-asdf-b".to_string()).unwrap(), - rev: "rev-asdg".to_string(), - action: CommitAction::Cut, - }, - &[0u8; 16], - ) - .unwrap(); - - let res = commits.truncating_insert( - UFOsCommit { - cursor: Cursor::from_raw_u64(102), - did: Did::new("did:plc:whatever".to_string()).unwrap(), - rkey: RecordKey::new("rkey-asdf-c".to_string()).unwrap(), - rev: "rev-asdh".to_string(), - action: CommitAction::Cut, - }, - &[0u8; 16], - ); - - assert!(res.is_err()); - let overflowed = match res { - Err(BatchInsertError::BatchFull(c)) => c, - e => panic!("expected overflow but a different error happened: {e:?}"), - }; - assert_eq!(overflowed.rev, "rev-asdh"); - - Ok(()) - } -} diff --git a/ufos/src/main.rs b/ufos/src/main.rs deleted file mode 100644 index e0c2936..0000000 --- a/ufos/src/main.rs +++ /dev/null @@ -1,260 +0,0 @@ -use clap::Parser; -use jetstream::events::Cursor; -use metrics::{describe_gauge, gauge, Unit}; -use metrics_exporter_prometheus::PrometheusBuilder; -use std::path::PathBuf; -use std::time::{Duration, SystemTime}; -use ufos::consumer; -use ufos::file_consumer; -use ufos::server; -use ufos::storage::{StorageWhatever, StoreBackground, StoreReader, StoreWriter}; -use ufos::storage_fjall::FjallStorage; -use ufos::store_types::SketchSecretPrefix; -use ufos::{nice_duration, ConsumerInfo}; - -#[cfg(not(target_env = "msvc"))] -use tikv_jemallocator::Jemalloc; - -#[cfg(not(target_env = "msvc"))] -#[global_allocator] -static GLOBAL: Jemalloc = Jemalloc; - -/// Aggregate links in the at-mosphere -#[derive(Parser, Debug, Clone)] -#[command(version, about, long_about = None)] -struct Args { - /// Jetstream server to connect to (exclusive with --fixture). Provide either a wss:// URL, or a shorhand value: - /// 'us-east-1', 'us-east-2', 'us-west-1', or 'us-west-2' - #[arg(long)] - jetstream: String, - /// allow changing jetstream endpoints - #[arg(long, action)] - jetstream_force: bool, - /// don't request zstd-compressed jetstream events - /// - /// reduces CPU at the expense of more ingress bandwidth - #[arg(long, action)] - jetstream_no_zstd: bool, - /// Location to store persist data to disk - #[arg(long)] - data: PathBuf, - /// DEBUG: don't start the jetstream consumer or its write loop - #[arg(long, action)] - pause_writer: bool, - /// Adjust runtime settings like background task intervals for efficient backfill - #[arg(long, action)] - backfill: bool, - /// DEBUG: force the rw loop to fall behind by pausing it - /// todo: restore this - #[arg(long, action)] - pause_rw: bool, - /// reset the rollup cursor, scrape through missed things in the past (backfill) - #[arg(long, action)] - reroll: bool, - /// DEBUG: interpret jetstream as a file fixture - #[arg(long, action)] - jetstream_fixture: bool, -} - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - env_logger::init(); - - let args = Args::parse(); - let jetstream = args.jetstream.clone(); - let (read_store, write_store, cursor, sketch_secret) = FjallStorage::init( - args.data.clone(), - jetstream, - args.jetstream_force, - Default::default(), - )?; - go(args, read_store, write_store, cursor, sketch_secret).await?; - Ok(()) -} - -async fn go( - args: Args, - read_store: impl StoreReader + 'static + Clone, - mut write_store: impl StoreWriter + 'static, - cursor: Option, - sketch_secret: SketchSecretPrefix, -) -> anyhow::Result<()> { - println!("starting server with storage..."); - let serving = server::serve(read_store.clone()); - - if args.pause_writer { - log::info!("not starting jetstream or the write loop."); - serving.await.map_err(|e| anyhow::anyhow!(e))?; - return Ok(()); - } - - let batches = if args.jetstream_fixture { - log::info!("starting with jestream file fixture: {:?}", args.jetstream); - file_consumer::consume(args.jetstream.into(), sketch_secret, cursor).await? - } else { - log::info!( - "starting consumer with cursor: {cursor:?} from {:?} ago", - cursor.map(|c| c.elapsed()) - ); - consumer::consume(&args.jetstream, cursor, false, sketch_secret).await? - }; - - let rolling = write_store - .background_tasks(args.reroll)? - .run(args.backfill); - let consuming = write_store.receive_batches(batches); - - let stating = do_update_stuff(read_store); - - install_metrics_server()?; - - tokio::select! { - z = serving => log::warn!("serve task ended: {z:?}"), - z = rolling => log::warn!("rollup task ended: {z:?}"), - z = consuming => log::warn!("consuming task ended: {z:?}"), - z = stating => log::warn!("status task ended: {z:?}"), - }; - - println!("bye!"); - - Ok(()) -} - -fn install_metrics_server() -> anyhow::Result<()> { - log::info!("installing metrics server..."); - let host = [0, 0, 0, 0]; - let port = 8765; - PrometheusBuilder::new() - .set_quantiles(&[0.5, 0.9, 0.99, 1.0])? - .set_bucket_duration(Duration::from_secs(60))? - .set_bucket_count(std::num::NonZero::new(10).unwrap()) // count * duration = 10 mins. stuff doesn't happen that fast here. - .set_enable_unit_suffix(false) // this seemed buggy for constellation (sometimes wouldn't engage) - .with_http_listener((host, port)) - .install()?; - log::info!( - "metrics server installed! listening on http://{}.{}.{}.{}:{port}", - host[0], - host[1], - host[2], - host[3] - ); - Ok(()) -} - -async fn do_update_stuff(read_store: impl StoreReader) { - describe_gauge!( - "persisted_cursor_age", - Unit::Microseconds, - "microseconds between our clock and the latest persisted event's cursor" - ); - describe_gauge!( - "rollup_cursor_age", - Unit::Microseconds, - "microseconds between our clock and the latest rollup cursor" - ); - let started_at = std::time::SystemTime::now(); - let mut first_cursor = None; - let mut first_rollup = None; - let mut last_at = std::time::SystemTime::now(); - let mut last_cursor = None; - let mut last_rollup = None; - let mut interval = tokio::time::interval(std::time::Duration::from_secs(4)); - interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - loop { - interval.tick().await; - match read_store.get_consumer_info().await { - Err(e) => log::warn!("failed to get jetstream consumer info: {e:?}"), - Ok(ConsumerInfo::Jetstream { - latest_cursor, - rollup_cursor, - .. - }) => { - let now = std::time::SystemTime::now(); - let latest_cursor = latest_cursor.map(Cursor::from_raw_u64); - let rollup_cursor = rollup_cursor.map(Cursor::from_raw_u64); - backfill_info( - latest_cursor, - rollup_cursor, - last_cursor, - last_rollup, - last_at, - first_cursor, - first_rollup, - started_at, - now, - ); - first_cursor = first_cursor.or(latest_cursor); - first_rollup = first_rollup.or(rollup_cursor); - last_cursor = latest_cursor; - last_rollup = rollup_cursor; - last_at = now; - } - } - } -} - -#[allow(clippy::too_many_arguments)] -fn backfill_info( - latest_cursor: Option, - rollup_cursor: Option, - last_cursor: Option, - last_rollup: Option, - last_at: SystemTime, - first_cursor: Option, - first_rollup: Option, - started_at: SystemTime, - now: SystemTime, -) { - if let Some(cursor) = latest_cursor { - gauge!("persisted_cursor_age").set(cursor.elapsed_micros_f64()); - } - if let Some(cursor) = rollup_cursor { - gauge!("rollup_cursor_age").set(cursor.elapsed_micros_f64()); - } - - let nice_dt_two_maybes = |earlier: Option, later: Option| match (earlier, later) - { - (Some(earlier), Some(later)) => match later.duration_since(&earlier) { - Ok(dt) => nice_duration(dt), - Err(e) => { - let rev_dt = e.duration(); - format!("+{}", nice_duration(rev_dt)) - } - }, - _ => "unknown".to_string(), - }; - - let rate = |mlatest: Option, msince: Option, real: Duration| { - mlatest - .zip(msince) - .map(|(latest, since)| { - latest - .duration_since(&since) - .unwrap_or(Duration::from_millis(1)) - }) - .map(|dtc| format!("{:.2}", dtc.as_secs_f64() / real.as_secs_f64())) - .unwrap_or("??".into()) - }; - - let dt_real = now - .duration_since(last_at) - .unwrap_or(Duration::from_millis(1)); - - let dt_real_total = now - .duration_since(started_at) - .unwrap_or(Duration::from_millis(1)); - - let cursor_rate = rate(latest_cursor, last_cursor, dt_real); - let cursor_avg = rate(latest_cursor, first_cursor, dt_real_total); - - let rollup_rate = rate(rollup_cursor, last_rollup, dt_real); - let rollup_avg = rate(rollup_cursor, first_rollup, dt_real_total); - - log::trace!( - "cursor: {} behind (→{}, {cursor_rate}x, {cursor_avg}x avg). rollup: {} behind (→{}, {rollup_rate}x, {rollup_avg}x avg).", - latest_cursor.map(|c| c.elapsed().map(nice_duration).unwrap_or("++".to_string())).unwrap_or("?".to_string()), - nice_dt_two_maybes(last_cursor, latest_cursor), - rollup_cursor.map(|c| c.elapsed().map(nice_duration).unwrap_or("++".to_string())).unwrap_or("?".to_string()), - nice_dt_two_maybes(last_rollup, rollup_cursor), - ); -} diff --git a/ufos/src/server/collections_query.rs b/ufos/src/server/collections_query.rs deleted file mode 100644 index 1af927a..0000000 --- a/ufos/src/server/collections_query.rs +++ /dev/null @@ -1,72 +0,0 @@ -use crate::Nsid; -use async_trait::async_trait; -use dropshot::{ - ApiEndpointBodyContentType, ExtractorMetadata, HttpError, Query, RequestContext, ServerContext, - SharedExtractor, -}; -use schemars::JsonSchema; -use serde::Deserialize; -use std::collections::HashSet; - -/// The real type that gets deserialized -#[derive(Debug, Deserialize, JsonSchema)] -pub struct MultiCollectionQuery { - pub collection: Vec, -} - -/// The fake corresponding type for docs that dropshot won't freak out about a -/// vec for -#[derive(Deserialize, JsonSchema)] -#[allow(dead_code)] -struct MultiCollectionQueryForDocs { - /// One or more collection [NSID](https://atproto.com/specs/nsid)s - /// - /// Pass this parameter multiple times to specify multiple collections, like - /// `collection=app.bsky.feed.like&collection=app.bsky.feed.post` - collection: String, -} - -impl TryFrom for HashSet { - type Error = HttpError; - fn try_from(mcq: MultiCollectionQuery) -> Result { - let mut out = HashSet::with_capacity(mcq.collection.len()); - for c in mcq.collection { - let nsid = Nsid::new(c).map_err(|e| { - HttpError::for_bad_request( - None, - format!("failed to convert collection to an NSID: {e:?}"), - ) - })?; - out.insert(nsid); - } - Ok(out) - } -} - -// The `SharedExtractor` implementation for Query describes how to -// construct an instance of `Query` from an HTTP request: namely, by -// parsing the query string to an instance of `QueryType`. -#[async_trait] -impl SharedExtractor for MultiCollectionQuery { - async fn from_request( - ctx: &RequestContext, - ) -> Result { - let raw_query = ctx.request.uri().query().unwrap_or(""); - let q = serde_qs::from_str(raw_query).map_err(|e| { - HttpError::for_bad_request(None, format!("unable to parse query string: {e}")) - })?; - Ok(q) - } - - fn metadata(body_content_type: ApiEndpointBodyContentType) -> ExtractorMetadata { - // HACK: query type switcheroo: passing MultiCollectionQuery to - // `metadata` would "helpfully" panic because dropshot believes we can - // only have scalar types in a query. - // - // so instead we have a fake second type whose only job is to look the - // same as MultiCollectionQuery exept that it has `String` instead of - // `Vec`, which dropshot will accept, and generate ~close-enough - // docs for. - as SharedExtractor>::metadata(body_content_type) - } -} diff --git a/ufos/src/server/cors.rs b/ufos/src/server/cors.rs deleted file mode 100644 index 61a8815..0000000 --- a/ufos/src/server/cors.rs +++ /dev/null @@ -1,23 +0,0 @@ -use dropshot::{HttpError, HttpResponseHeaders, HttpResponseOk}; -use schemars::JsonSchema; -use serde::Serialize; - -pub type OkCorsResponse = Result>, HttpError>; - -/// Helper for constructing Ok responses: return OkCors(T).into() -/// (not happy with this yet) -pub struct OkCors(pub T); - -impl From> for OkCorsResponse -where - T: Serialize + JsonSchema + Send + Sync, -{ - fn from(ok: OkCors) -> OkCorsResponse { - let mut res = HttpResponseHeaders::new_unnamed(HttpResponseOk(ok.0)); - res.headers_mut() - .insert("access-control-allow-origin", "*".parse().unwrap()); - Ok(res) - } -} - -// TODO: cors for HttpError diff --git a/ufos/src/server/mod.rs b/ufos/src/server/mod.rs deleted file mode 100644 index 2a054e7..0000000 --- a/ufos/src/server/mod.rs +++ /dev/null @@ -1,767 +0,0 @@ -mod collections_query; -mod cors; - -use crate::index_html::INDEX_HTML; -use crate::storage::StoreReader; -use crate::store_types::{HourTruncatedCursor, WeekTruncatedCursor}; -use crate::{ - ConsumerInfo, Cursor, JustCount, Nsid, NsidCount, NsidPrefix, OrderCollectionsBy, PrefixChild, - UFOsRecord, -}; -use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; -use chrono::{DateTime, Utc}; -use collections_query::MultiCollectionQuery; -use cors::{OkCors, OkCorsResponse}; -use dropshot::endpoint; -use dropshot::ApiDescription; -use dropshot::Body; -use dropshot::ConfigDropshot; -use dropshot::ConfigLogging; -use dropshot::ConfigLoggingLevel; -use dropshot::HttpError; -use dropshot::HttpResponse; -use dropshot::Query; -use dropshot::RequestContext; -use dropshot::ServerBuilder; -use dropshot::ServerContext; -use http::{ - header::{ORIGIN, USER_AGENT}, - Response, StatusCode, -}; -use metrics::{counter, describe_counter, describe_histogram, histogram, Unit}; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; -use std::collections::{HashMap, HashSet}; -use std::future::Future; -use std::sync::Arc; -use std::time::Instant; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; - -fn describe_metrics() { - describe_counter!( - "server_requests_total", - Unit::Count, - "total requests handled" - ); - describe_histogram!( - "server_handler_latency", - Unit::Microseconds, - "time to respond to a request in microseconds, excluding dropshot overhead" - ); -} - -async fn instrument_handler(ctx: &RequestContext, handler: H) -> Result -where - R: HttpResponse, - H: Future>, - T: ServerContext, -{ - let start = Instant::now(); - let result = handler.await; - let latency = start.elapsed(); - let status_code = match &result { - Ok(response) => response.status_code(), - Err(ref e) => e.status_code.as_status(), - } - .as_str() // just the number (.to_string()'s Display does eg `200 OK`) - .to_string(); - let endpoint = ctx.endpoint.operation_id.clone(); - let headers = ctx.request.headers(); - let origin = headers - .get(ORIGIN) - .and_then(|v| v.to_str().ok()) - .unwrap_or("") - .to_string(); - let ua = headers - .get(USER_AGENT) - .and_then(|v| v.to_str().ok()) - .map(|ua| { - if ua.starts_with("Mozilla/5.0 ") { - "browser" - } else { - ua - } - }) - .unwrap_or("") - .to_string(); - counter!("server_requests_total", - "endpoint" => endpoint.clone(), - "origin" => origin, - "ua" => ua, - "status_code" => status_code, - ) - .increment(1); - histogram!("server_handler_latency", "endpoint" => endpoint).record(latency.as_micros() as f64); - result -} - -struct Context { - pub spec: Arc, - storage: Box, -} - -fn dt_to_cursor(dt: DateTime) -> Result { - let t = dt.timestamp_micros(); - if t < 0 { - Err(HttpError::for_bad_request(None, "timestamp too old".into())) - } else { - let t = t as u64; - let t_now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_micros() as u64; - const ONE_HOUR: u64 = 60 * 60 * 1_000_000; - if t > t_now && (t - t_now > 2 * ONE_HOUR) { - Err(HttpError::for_bad_request(None, "future timestamp".into())) - } else { - Ok(HourTruncatedCursor::truncate_raw_u64(t)) - } - } -} - -/// Serve index page as html -#[endpoint { - method = GET, - path = "/", - /* - * not useful to have this in openapi - */ - unpublished = true, -}] -async fn index(ctx: RequestContext) -> Result, HttpError> { - instrument_handler(&ctx, async { - Ok(Response::builder() - .status(StatusCode::OK) - .header(http::header::CONTENT_TYPE, "text/html") - .body(INDEX_HTML.into())?) - }) - .await -} - -/// Meta: get the openapi spec for this api -#[endpoint { - method = GET, - path = "/openapi", - /* - * not useful to have this in openapi - */ - unpublished = true, -}] -async fn get_openapi(ctx: RequestContext) -> OkCorsResponse { - instrument_handler(&ctx, async { - let spec = (*ctx.context().spec).clone(); - OkCors(spec).into() - }) - .await -} - -#[derive(Debug, Serialize, JsonSchema)] -struct MetaInfo { - storage_name: String, - storage: serde_json::Value, - consumer: ConsumerInfo, -} -/// UFOs meta-info -#[endpoint { - method = GET, - path = "/meta" -}] -async fn get_meta_info(ctx: RequestContext) -> OkCorsResponse { - let Context { storage, .. } = ctx.context(); - let failed_to_get = - |what| move |e| HttpError::for_internal_error(format!("failed to get {what}: {e:?}")); - - instrument_handler(&ctx, async { - let storage_info = storage - .get_storage_stats() - .await - .map_err(failed_to_get("storage info"))?; - - let consumer = storage - .get_consumer_info() - .await - .map_err(failed_to_get("consumer info"))?; - - OkCors(MetaInfo { - storage_name: storage.name(), - storage: storage_info, - consumer, - }) - .into() - }) - .await -} - -// TODO: replace with normal (🙃) multi-qs value somehow -fn to_multiple_nsids(s: &str) -> Result, String> { - let mut out = HashSet::new(); - for collection in s.split(',') { - let Ok(nsid) = Nsid::new(collection.to_string()) else { - return Err(format!("collection {collection:?} was not a valid NSID")); - }; - out.insert(nsid); - } - Ok(out) -} - -#[derive(Debug, Deserialize, JsonSchema)] -struct RecordsCollectionsQuery { - collection: Option, // JsonSchema not implemented for Nsid :( -} -#[derive(Debug, Serialize, JsonSchema)] -struct ApiRecord { - did: String, - collection: String, - rkey: String, - record: Box, - time_us: u64, -} -impl From for ApiRecord { - fn from(ufo: UFOsRecord) -> Self { - Self { - did: ufo.did.to_string(), - collection: ufo.collection.to_string(), - rkey: ufo.rkey.to_string(), - record: ufo.record, - time_us: ufo.cursor.to_raw_u64(), - } - } -} -/// Record samples -/// -/// Get most recent records seen in the firehose, by collection NSID -/// -/// Multiple collections are supported. They will be delivered in one big array with no -/// specified order. -#[endpoint { - method = GET, - path = "/records", -}] -async fn get_records_by_collections( - ctx: RequestContext, - collection_query: Query, -) -> OkCorsResponse> { - let Context { storage, .. } = ctx.context(); - instrument_handler(&ctx, async { - let mut limit = 42; - let query = collection_query.into_inner(); - let collections = if let Some(provided_collection) = query.collection { - to_multiple_nsids(&provided_collection) - .map_err(|reason| HttpError::for_bad_request(None, reason))? - } else { - limit = 12; - let min_time_ago = SystemTime::now() - Duration::from_secs(86_400 * 3); // we want at least 3 days of data - let since: WeekTruncatedCursor = Cursor::at(min_time_ago).into(); - let (collections, _) = storage - .get_collections( - 1000, - Default::default(), - Some(since.try_as().unwrap()), - None, - ) - .await - .map_err(|e| HttpError::for_internal_error(e.to_string()))?; - collections - .into_iter() - .map(|c| Nsid::new(c.nsid).unwrap()) - .collect() - }; - - let records = storage - .get_records_by_collections(collections, limit, true) - .await - .map_err(|e| HttpError::for_internal_error(e.to_string()))? - .into_iter() - .map(|r| r.into()) - .collect(); - - OkCors(records).into() - }) - .await -} - -#[derive(Debug, Deserialize, JsonSchema)] -struct CollectionsStatsQuery { - /// Limit stats to those seen after this UTC datetime - /// - /// default: 1 week ago - since: Option>, - /// Limit stats to those seen before this UTC datetime - /// - /// default: now - until: Option>, -} -/// Collection stats -/// -/// Get record statistics for collections during a specific time period. -/// -/// Note: the statistics are "rolled up" into hourly buckets in the background, -/// so the data here can be as stale as that background task is behind. See the -/// meta info endpoint to find out how up-to-date the rollup currently is. (In -/// general it sholud be pretty close to live) -#[endpoint { - method = GET, - path = "/collections/stats" -}] -async fn get_collection_stats( - ctx: RequestContext, - collections_query: MultiCollectionQuery, - query: Query, -) -> OkCorsResponse> { - let Context { storage, .. } = ctx.context(); - - instrument_handler(&ctx, async { - let q = query.into_inner(); - let collections: HashSet = collections_query.try_into()?; - - let since = q.since.map(dt_to_cursor).transpose()?.unwrap_or_else(|| { - let week_ago_secs = 7 * 86_400; - let week_ago = SystemTime::now() - Duration::from_secs(week_ago_secs); - Cursor::at(week_ago).into() - }); - - let until = q.until.map(dt_to_cursor).transpose()?; - - let mut seen_by_collection = HashMap::with_capacity(collections.len()); - - for collection in &collections { - let counts = storage - .get_collection_counts(collection, since, until) - .await - .map_err(|e| HttpError::for_internal_error(format!("boooo: {e:?}")))?; - - seen_by_collection.insert(collection.to_string(), counts); - } - - OkCors(seen_by_collection).into() - }) - .await -} - -#[derive(Debug, Serialize, JsonSchema)] -struct CollectionsResponse { - /// Each known collection and its associated statistics - /// - /// The order is unspecified. - collections: Vec, - /// Include in a follow-up request to get the next page of results, if more are available - cursor: Option, -} -#[derive(Debug, Deserialize, JsonSchema)] -#[serde(rename_all = "kebab-case")] -pub enum CollectionsQueryOrder { - RecordsCreated, - DidsEstimate, -} -impl From<&CollectionsQueryOrder> for OrderCollectionsBy { - fn from(q: &CollectionsQueryOrder) -> Self { - match q { - CollectionsQueryOrder::RecordsCreated => OrderCollectionsBy::RecordsCreated, - CollectionsQueryOrder::DidsEstimate => OrderCollectionsBy::DidsEstimate, - } - } -} -#[derive(Debug, Deserialize, JsonSchema)] -struct CollectionsQuery { - /// The maximum number of collections to return in one request. - /// - /// Default: `100` normally, `32` if `order` is specified. - #[schemars(range(min = 1, max = 200))] - limit: Option, - /// Get a paginated response with more collections. - /// - /// Always omit the cursor for the first request. If more collections than the limit are available, the response will contain a non-null `cursor` to include with the next request. - /// - /// `cursor` is mutually exclusive with `order`. - cursor: Option, - /// Limit collections and statistics to those seen after this UTC datetime - since: Option>, - /// Limit collections and statistics to those seen before this UTC datetime - until: Option>, - /// Get a limited, sorted list - /// - /// Mutually exclusive with `cursor` -- sorted results cannot be paged. - order: Option, -} - -/// List collections -/// -/// With statistics. -/// -/// ## To fetch a full list: -/// -/// Omit the `order` parameter and page through the results using the `cursor`. There have been a lot of collections seen in the ATmosphere, well over 400 at time of writing, so you *will* need to make a series of paginaged requests with `cursor`s to get them all. -/// -/// The set of collections across multiple requests is not guaranteed to be a perfectly consistent snapshot: -/// -/// - all collection NSIDs observed before the first request will be included in the results -/// -/// - *new* NSIDs observed in the firehose *while paging* might be included or excluded from the final set -/// -/// - no duplicate NSIDs will occur in the combined results -/// -/// In practice this is close enough for most use-cases to not worry about. -/// -/// ## To fetch the top collection NSIDs: -/// -/// Specify the `order` parameter (must be either `records-created` or `did-estimate`). Note that ordered results cannot be paged. -/// -/// All statistics are bucketed hourly, so the most granular effecitve time boundary for `since` and `until` is one hour. -#[endpoint { - method = GET, - path = "/collections" -}] -async fn get_collections( - ctx: RequestContext, - query: Query, -) -> OkCorsResponse { - let Context { storage, .. } = ctx.context(); - let q = query.into_inner(); - - instrument_handler(&ctx, async { - if q.cursor.is_some() && q.order.is_some() { - let msg = - "`cursor` is mutually exclusive with `order`. ordered results cannot be paged."; - return Err(HttpError::for_bad_request(None, msg.to_string())); - } - - let order = if let Some(ref o) = q.order { - o.into() - } else { - let cursor = q - .cursor - .and_then(|c| if c.is_empty() { None } else { Some(c) }) - .map(|c| URL_SAFE_NO_PAD.decode(&c)) - .transpose() - .map_err(|e| HttpError::for_bad_request(None, format!("invalid cursor: {e:?}")))?; - OrderCollectionsBy::Lexi { cursor } - }; - - let limit = match (q.limit, q.order) { - (Some(limit), _) => limit, - (None, Some(_)) => 32, - (None, None) => 100, - }; - - if !(1..=200).contains(&limit) { - let msg = format!("limit not in 1..=200: {limit}"); - return Err(HttpError::for_bad_request(None, msg)); - } - - let since = q.since.map(dt_to_cursor).transpose()?; - let until = q.until.map(dt_to_cursor).transpose()?; - - let (collections, next_cursor) = storage - .get_collections(limit, order, since, until) - .await - .map_err(|e| HttpError::for_internal_error(format!("oh shoot: {e:?}")))?; - - let next_cursor = next_cursor.map(|c| URL_SAFE_NO_PAD.encode(c)); - - OkCors(CollectionsResponse { - collections, - cursor: next_cursor, - }) - .into() - }) - .await -} - -#[derive(Debug, Serialize, JsonSchema)] -struct PrefixResponse { - /// Note that total may not include counts beyond the current page (TODO) - total: JustCount, - children: Vec, - /// Include in a follow-up request to get the next page of results, if more are available - cursor: Option, -} -#[derive(Debug, Deserialize, JsonSchema)] -struct PrefixQuery { - /// - /// The final segment of a collection NSID is the `name`, and everything before it is called its `group`. eg: - /// - /// - `app.bsky.feed.post` and `app.bsky.feed.like` are both in the _lexicon group_ "`app.bsky.feed`". - /// - prefix: String, - /// The maximum number of collections to return in one request. - /// - /// The number of items actually returned may be less than the limit. If paginating, this does **not** indicate that no - /// more items are available! Check if the `cursor` in the response is `null` to determine the end of items. - /// - /// Default: `100` normally, `32` if `order` is specified. - #[schemars(range(min = 1, max = 200))] - limit: Option, - /// Get a paginated response with more collections. - /// - /// Always omit the cursor for the first request. If more collections than the limit are available, the response will contain a non-null `cursor` to include with the next request. - /// - /// `cursor` is mutually exclusive with `order`. - cursor: Option, - /// Limit collections and statistics to those seen after this UTC datetime - /// - /// Default: all-time - since: Option>, - /// Limit collections and statistics to those seen before this UTC datetime - /// - /// Default: now - until: Option>, - /// Get a limited, sorted list - /// - /// Mutually exclusive with `cursor` -- sorted results cannot be paged. - order: Option, -} -/// Prefix-filter collections list -/// -/// This endpoint enumerates all collection NSIDs for a lexicon group. -/// -/// ## To fetch a full list: -/// -/// Omit the `order` parameter and page through the results using the `cursor`. There have been a lot of collections seen in the ATmosphere, well over 400 at time of writing, so you *will* need to make a series of paginaged requests with `cursor`s to get them all. -/// -/// The set of collections across multiple requests is not guaranteed to be a perfectly consistent snapshot: -/// -/// - all collection NSIDs observed before the first request will be included in the results -/// -/// - *new* NSIDs observed in the firehose *while paging* might be included or excluded from the final set -/// -/// - no duplicate NSIDs will occur in the combined results -/// -/// In practice this is close enough for most use-cases to not worry about. -/// -/// ## To fetch the top collection NSIDs: -/// -/// Specify the `order` parameter (must be either `records-created` or `did-estimate`). Note that ordered results cannot be paged. -/// -/// All statistics are bucketed hourly, so the most granular effecitve time boundary for `since` and `until` is one hour. -#[endpoint { - method = GET, - path = "/prefix" -}] -async fn get_prefix( - ctx: RequestContext, - query: Query, -) -> OkCorsResponse { - let Context { storage, .. } = ctx.context(); - let q = query.into_inner(); - - instrument_handler(&ctx, async { - let prefix = NsidPrefix::new(&q.prefix).map_err(|e| { - HttpError::for_bad_request( - None, - format!("{:?} was not a valid NSID prefix: {e:?}", q.prefix), - ) - })?; - - if q.cursor.is_some() && q.order.is_some() { - let msg = - "`cursor` is mutually exclusive with `order`. ordered results cannot be paged."; - return Err(HttpError::for_bad_request(None, msg.to_string())); - } - - let order = if let Some(ref o) = q.order { - o.into() - } else { - let cursor = q - .cursor - .and_then(|c| if c.is_empty() { None } else { Some(c) }) - .map(|c| URL_SAFE_NO_PAD.decode(&c)) - .transpose() - .map_err(|e| HttpError::for_bad_request(None, format!("invalid cursor: {e:?}")))?; - OrderCollectionsBy::Lexi { cursor } - }; - - let limit = match (q.limit, q.order) { - (Some(limit), _) => limit, - (None, Some(_)) => 32, - (None, None) => 100, - }; - - if !(1..=200).contains(&limit) { - let msg = format!("limit not in 1..=200: {limit}"); - return Err(HttpError::for_bad_request(None, msg)); - } - - let since = q.since.map(dt_to_cursor).transpose()?; - let until = q.until.map(dt_to_cursor).transpose()?; - - let (total, children, next_cursor) = storage - .get_prefix(prefix, limit, order, since, until) - .await - .map_err(|e| HttpError::for_internal_error(format!("oh shoot: {e:?}")))?; - - let next_cursor = next_cursor.map(|c| URL_SAFE_NO_PAD.encode(c)); - - OkCors(PrefixResponse { - total, - children, - cursor: next_cursor, - }) - .into() - }) - .await -} - -#[derive(Debug, Deserialize, JsonSchema)] -struct CollectionTimeseriesQuery { - collection: String, // JsonSchema not implemented for Nsid :( - /// Limit collections and statistics to those seen after this UTC datetime - /// - /// default: 1 week ago - since: Option>, - /// Limit collections and statistics to those seen before this UTC datetime - /// - /// default: now - until: Option>, - /// time steps between data, in seconds - /// - /// the step will be rounded down to the nearest hour - /// - /// default: 86400 (24hrs) - #[schemars(range(min = 3600))] - step: Option, - // todo: rolling averages -} -#[derive(Debug, Serialize, JsonSchema)] -struct CollectionTimeseriesResponse { - range: Vec>, - series: HashMap>, -} -/// Collection timeseries stats -#[endpoint { - method = GET, - path = "/timeseries" -}] -async fn get_timeseries( - ctx: RequestContext, - query: Query, -) -> OkCorsResponse { - let Context { storage, .. } = ctx.context(); - let q = query.into_inner(); - - instrument_handler(&ctx, async { - let since = q.since.map(dt_to_cursor).transpose()?.unwrap_or_else(|| { - let week_ago_secs = 7 * 86_400; - let week_ago = SystemTime::now() - Duration::from_secs(week_ago_secs); - Cursor::at(week_ago).into() - }); - - let until = q.until.map(dt_to_cursor).transpose()?; - - let step = if let Some(secs) = q.step { - if secs < 3600 { - let msg = format!("step is too small: {secs}"); - Err(HttpError::for_bad_request(None, msg))?; - } - (secs / 3600) * 3600 // trucate to hour - } else { - 86_400 - }; - - let nsid = Nsid::new(q.collection).map_err(|e| { - HttpError::for_bad_request(None, format!("collection was not a valid NSID: {e:?}")) - })?; - - let (range_cursors, series) = storage - .get_timeseries(vec![nsid], since, until, step) - .await - .map_err(|e| HttpError::for_internal_error(format!("oh shoot: {e:?}")))?; - - let range = range_cursors - .into_iter() - .map(|c| DateTime::::from_timestamp_micros(c.to_raw_u64() as i64).unwrap()) - .collect(); - - let series = series - .into_iter() - .map(|(k, v)| (k.to_string(), v.iter().map(Into::into).collect())) - .collect(); - - OkCors(CollectionTimeseriesResponse { range, series }).into() - }) - .await -} - -#[derive(Debug, Deserialize, JsonSchema)] -struct SearchQuery { - /// Query - /// - /// at least two alphanumeric (+hyphen) characters must be present - q: String, -} -#[derive(Debug, Serialize, JsonSchema)] -struct SearchResponse { - matches: Vec, -} -/// Search lexicons -#[endpoint { - method = GET, - path = "/search" -}] -async fn search_collections( - ctx: RequestContext, - query: Query, -) -> OkCorsResponse { - let Context { storage, .. } = ctx.context(); - let q = query.into_inner(); - instrument_handler(&ctx, async { - // TODO: query validation - // TODO: also handle multi-space stuff (ufos-app tries to on client) - let terms: Vec = q.q.split(' ').map(Into::into).collect(); - let matches = storage - .search_collections(terms) - .await - .map_err(|e| HttpError::for_internal_error(format!("oh ugh: {e:?}")))?; - OkCors(SearchResponse { matches }).into() - }) - .await -} - -pub async fn serve(storage: impl StoreReader + 'static) -> Result<(), String> { - describe_metrics(); - let log = ConfigLogging::StderrTerminal { - level: ConfigLoggingLevel::Warn, - } - .to_logger("server") - .map_err(|e| e.to_string())?; - - let mut api = ApiDescription::new(); - - api.register(index).unwrap(); - api.register(get_openapi).unwrap(); - api.register(get_meta_info).unwrap(); - api.register(get_records_by_collections).unwrap(); - api.register(get_collection_stats).unwrap(); - api.register(get_collections).unwrap(); - api.register(get_prefix).unwrap(); - api.register(get_timeseries).unwrap(); - api.register(search_collections).unwrap(); - - let context = Context { - spec: Arc::new( - api.openapi( - "UFOs API: Every lexicon in the ATmosphere", - env!("CARGO_PKG_VERSION") - .parse() - .inspect_err(|e| { - log::warn!("failed to parse cargo package version for openapi: {e:?}") - }) - .unwrap_or(semver::Version::new(0, 0, 1)), - ) - .description("Samples and statistics of atproto records by their collection NSID") - .contact_name("part of @microcosm.blue") - .contact_url("https://microcosm.blue") - .json() - .map_err(|e| e.to_string())?, - ), - storage: Box::new(storage), - }; - - ServerBuilder::new(api, context, log) - .config(ConfigDropshot { - bind_address: "0.0.0.0:9999".parse().unwrap(), - ..Default::default() - }) - .start() - .map_err(|error| format!("failed to start server: {error}"))? - .await -} diff --git a/ufos/src/storage.rs b/ufos/src/storage.rs deleted file mode 100644 index 222da1b..0000000 --- a/ufos/src/storage.rs +++ /dev/null @@ -1,151 +0,0 @@ -use crate::store_types::{CountsValue, HourTruncatedCursor, SketchSecretPrefix}; -use crate::{ - error::StorageError, ConsumerInfo, Cursor, EventBatch, JustCount, NsidCount, NsidPrefix, - OrderCollectionsBy, PrefixChild, UFOsRecord, -}; -use async_trait::async_trait; -use jetstream::exports::{Did, Nsid}; -use metrics::{describe_histogram, histogram, Unit}; -use std::collections::{HashMap, HashSet}; -use std::path::Path; -use std::time::{Duration, Instant}; -use tokio::sync::mpsc::Receiver; -use tokio_util::sync::CancellationToken; - -pub type StorageResult = Result; - -pub trait StorageWhatever, B: StoreBackground, C> { - fn init( - path: impl AsRef, - endpoint: String, - force_endpoint: bool, - config: C, - ) -> StorageResult<(R, W, Option, SketchSecretPrefix)> - where - Self: Sized; -} - -#[async_trait] -pub trait StoreWriter: Clone + Send + Sync -where - Self: 'static, -{ - fn background_tasks(&mut self, reroll: bool) -> StorageResult; - - async fn receive_batches( - self, - mut batches: Receiver>, - ) -> StorageResult<()> { - describe_histogram!( - "storage_slow_batches", - Unit::Microseconds, - "batches that took more than 3s to insert" - ); - while let Some(event_batch) = batches.recv().await { - let token = CancellationToken::new(); - let cancelled = token.clone(); - tokio::spawn(async move { - let started = Instant::now(); - let mut concerned = false; - loop { - tokio::select! { - _ = tokio::time::sleep(Duration::from_secs(3)) => { - if !concerned { - log::warn!("taking a long time to insert an event batch..."); - } - concerned = true; - } - _ = cancelled.cancelled() => { - if concerned { - log::warn!("finally inserted slow event batch (or failed) after {:?}", started.elapsed()); - histogram!("storage_slow_batches").record(started.elapsed().as_micros() as f64); - } - break - } - } - } - }); - tokio::task::spawn_blocking({ - let mut me = self.clone(); - move || { - let _guard = token.drop_guard(); - me.insert_batch(event_batch) - } - }) - .await??; - } - - Err(StorageError::BatchSenderExited) - } - - fn insert_batch( - &mut self, - event_batch: EventBatch, - ) -> StorageResult<()>; - - fn step_rollup(&mut self) -> StorageResult<(usize, HashSet)>; - - fn trim_collection( - &mut self, - collection: &Nsid, - limit: usize, - full_scan: bool, - ) -> StorageResult<(usize, usize, bool)>; - - fn delete_account(&mut self, did: &Did) -> StorageResult; -} - -#[async_trait] -pub trait StoreBackground: Send + Sync { - async fn run(mut self, backfill: bool) -> StorageResult<()>; -} - -#[async_trait] -pub trait StoreReader: Send + Sync { - fn name(&self) -> String; - - async fn get_storage_stats(&self) -> StorageResult; - - async fn get_consumer_info(&self) -> StorageResult; - - async fn get_collections( - &self, - limit: usize, - order: OrderCollectionsBy, - since: Option, - until: Option, - ) -> StorageResult<(Vec, Option>)>; - - async fn get_prefix( - &self, - prefix: NsidPrefix, - limit: usize, - order: OrderCollectionsBy, - since: Option, - until: Option, - ) -> StorageResult<(JustCount, Vec, Option>)>; - - async fn get_timeseries( - &self, - collections: Vec, - since: HourTruncatedCursor, - until: Option, - step: u64, - ) -> StorageResult<(Vec, HashMap>)>; - - async fn get_collection_counts( - &self, - collection: &Nsid, - since: HourTruncatedCursor, - until: Option, - ) -> StorageResult; - - async fn get_records_by_collections( - &self, - collections: HashSet, - limit: usize, - expand_each_collection: bool, - ) -> StorageResult>; - - async fn search_collections(&self, terms: Vec) -> StorageResult>; -} diff --git a/ufos/src/storage_fjall.rs b/ufos/src/storage_fjall.rs deleted file mode 100644 index 4e88844..0000000 --- a/ufos/src/storage_fjall.rs +++ /dev/null @@ -1,2845 +0,0 @@ -use crate::db_types::{ - db_complete, DbBytes, DbStaticStr, EncodingResult, StaticStr, SubPrefixBytes, -}; -use crate::error::StorageError; -use crate::storage::{StorageResult, StorageWhatever, StoreBackground, StoreReader, StoreWriter}; -use crate::store_types::{ - AllTimeDidsKey, AllTimeRecordsKey, AllTimeRollupKey, CommitCounts, CountsValue, CursorBucket, - DeleteAccountQueueKey, DeleteAccountQueueVal, HourTruncatedCursor, HourlyDidsKey, - HourlyRecordsKey, HourlyRollupKey, HourlyRollupStaticPrefix, JetstreamCursorKey, - JetstreamCursorValue, JetstreamEndpointKey, JetstreamEndpointValue, LiveCountsKey, - NewRollupCursorKey, NewRollupCursorValue, NsidRecordFeedKey, NsidRecordFeedVal, - RecordLocationKey, RecordLocationMeta, RecordLocationVal, RecordRawValue, SketchSecretKey, - SketchSecretPrefix, TakeoffKey, TakeoffValue, TrimCollectionCursorKey, WeekTruncatedCursor, - WeeklyDidsKey, WeeklyRecordsKey, WeeklyRollupKey, WithCollection, WithRank, HOUR_IN_MICROS, - WEEK_IN_MICROS, -}; -use crate::{ - nice_duration, CommitAction, ConsumerInfo, Did, EncodingError, EventBatch, JustCount, Nsid, - NsidCount, NsidPrefix, OrderCollectionsBy, PrefixChild, PrefixCount, UFOsRecord, -}; -use async_trait::async_trait; -use fjall::{ - Batch as FjallBatch, Config, Keyspace, PartitionCreateOptions, PartitionHandle, Snapshot, -}; -use jetstream::events::Cursor; -use metrics::{counter, describe_counter, describe_histogram, histogram, Unit}; -use std::collections::{HashMap, HashSet}; -use std::iter::Peekable; -use std::ops::Bound; -use std::path::Path; -use std::sync::{ - atomic::{AtomicBool, Ordering}, - Arc, -}; -use std::time::{Duration, Instant, SystemTime}; - -const MAX_BATCHED_ACCOUNT_DELETE_RECORDS: usize = 1024; -const MAX_BATCHED_ROLLUP_COUNTS: usize = 256; - -/// -/// new data format, roughly: -/// -/// Partition: 'global' -/// -/// - Global sequence counter (is the jetstream cursor -- monotonic with many gaps) -/// - key: "js_cursor" (literal) -/// - val: u64 -/// -/// - Jetstream server endpoint (persisted because the cursor can't be used on another instance without data loss) -/// - key: "js_endpoint" (literal) -/// - val: string (URL of the instance) -/// -/// - Launch date -/// - key: "takeoff" (literal) -/// - val: u64 (micros timestamp, not from jetstream for now so not precise) -/// -/// - Cardinality estimator secret -/// - key: "sketch_secret" (literal) -/// - val: [u8; 16] -/// -/// - Rollup cursor (bg work: roll stats into hourlies, delete accounts, old record deletes) -/// - key: "rollup_cursor" (literal) -/// - val: u64 (tracks behind js_cursor) -/// -/// - Feed trim cursor (bg work: delete oldest excess records) -/// - key: "trim_cursor" || nullstr (nsid) -/// - val: u64 (earliest previously-removed feed entry jetstream cursor) -/// -/// Partition: 'feed' -/// -/// - Per-collection list of record references ordered by jetstream cursor -/// - key: nullstr || u64 (collection nsid null-terminated, jetstream cursor) -/// - val: nullstr || nullstr || nullstr (did, rkey, rev. rev is mostly a sanity-check for now.) -/// -/// -/// Partition: 'records' -/// -/// - Actual records by their atproto location -/// - key: nullstr || nullstr || nullstr (did, collection, rkey) -/// - val: u64 || bool || nullstr || rawval (js_cursor, is_update, rev, actual record) -/// -/// -/// Partition: 'rollups' -/// -/// - Live (batched) records counts and dids estimate per collection -/// - key: "live_counts" || u64 || nullstr (js_cursor, nsid) -/// - val: u64 || HLL (count (not cursor), estimator) -/// -/// -/// - Hourly total record counts and dids estimate per collection -/// - key: "hourly_counts" || u64 || nullstr (hour, nsid) -/// - val: u64 || HLL (count (not cursor), estimator) -/// -/// - Hourly record count ranking -/// - key: "hourly_rank_records" || u64 || u64 || nullstr (hour, count, nsid) -/// - val: [empty] -/// -/// - Hourly did estimate ranking -/// - key: "hourly_rank_dids" || u64 || u64 || nullstr (hour, dids estimate, nsid) -/// - val: [empty] -/// -/// -/// - Weekly total record counts and dids estimate per collection -/// - key: "weekly_counts" || u64 || nullstr (week, nsid) -/// - val: u64 || HLL (count (not cursor), estimator) -/// -/// - Weekly record count ranking -/// - key: "weekly_rank_records" || u64 || u64 || nullstr (week, count, nsid) -/// - val: [empty] -/// -/// - Weekly did estimate ranking -/// - key: "weekly_rank_dids" || u64 || u64 || nullstr (week, dids estimate, nsid) -/// - val: [empty] -/// -/// -/// - All-time total record counts and dids estimate per collection -/// - key: "ever_counts" || nullstr (nsid) -/// - val: u64 || HLL (count (not cursor), estimator) -/// -/// - All-time total record record count ranking -/// - key: "ever_rank_records" || u64 || nullstr (count, nsid) -/// - val: [empty] -/// -/// - All-time did estimate ranking -/// - key: "ever_rank_dids" || u64 || nullstr (dids estimate, nsid) -/// - val: [empty] -/// -/// -/// Partition: 'queues' -/// -/// - Delete account queue -/// - key: "delete_acount" || u64 (js_cursor) -/// - val: nullstr (did) -/// -/// -/// TODO: moderation actions -/// TODO: account privacy preferences. Might wait for the protocol-level (PDS-level?) stuff to land. Will probably do lazy fetching + caching on read. -#[derive(Debug)] -pub struct FjallStorage {} - -#[derive(Debug, Default)] -pub struct FjallConfig { - /// drop the db when the storage is dropped - /// - /// this is only meant for tests - #[cfg(test)] - pub temp: bool, -} - -impl StorageWhatever for FjallStorage { - fn init( - path: impl AsRef, - endpoint: String, - force_endpoint: bool, - _config: FjallConfig, - ) -> StorageResult<(FjallReader, FjallWriter, Option, SketchSecretPrefix)> { - let keyspace = { - let config = Config::new(path); - - // #[cfg(not(test))] - // let config = config.fsync_ms(Some(4_000)); - - config.open()? - }; - - let global = keyspace.open_partition("global", PartitionCreateOptions::default())?; - let feeds = keyspace.open_partition("feeds", PartitionCreateOptions::default())?; - let records = keyspace.open_partition("records", PartitionCreateOptions::default())?; - let rollups = keyspace.open_partition("rollups", PartitionCreateOptions::default())?; - let queues = keyspace.open_partition("queues", PartitionCreateOptions::default())?; - - let js_cursor = get_static_neu::(&global)?; - - let sketch_secret = if js_cursor.is_some() { - let stored_endpoint = - get_static_neu::(&global)?; - let JetstreamEndpointValue(stored) = stored_endpoint.ok_or(StorageError::InitError( - "found cursor but missing js_endpoint, refusing to start.".to_string(), - ))?; - - let Some(stored_secret) = - get_static_neu::(&global)? - else { - return Err(StorageError::InitError( - "found cursor but missing sketch_secret, refusing to start.".to_string(), - )); - }; - - if stored != endpoint { - if force_endpoint { - log::warn!("forcing a jetstream switch from {stored:?} to {endpoint:?}"); - insert_static_neu::( - &global, - JetstreamEndpointValue(endpoint.to_string()), - )?; - } else { - return Err(StorageError::InitError(format!( - "stored js_endpoint {stored:?} differs from provided {endpoint:?}, refusing to start without --jetstream-force."))); - } - } - stored_secret - } else { - log::info!("initializing a fresh db!"); - init_static_neu::( - &global, - JetstreamEndpointValue(endpoint.to_string()), - )?; - - log::info!("generating new secret for cardinality sketches..."); - let mut sketch_secret: SketchSecretPrefix = [0u8; 16]; - getrandom::fill(&mut sketch_secret).map_err(|e| { - StorageError::InitError(format!( - "failed to get a random secret for cardinality sketches: {e:?}" - )) - })?; - init_static_neu::(&global, sketch_secret)?; - - init_static_neu::(&global, Cursor::at(SystemTime::now()))?; - init_static_neu::(&global, Cursor::from_start())?; - - sketch_secret - }; - - let reader = FjallReader { - keyspace: keyspace.clone(), - global: global.clone(), - feeds: feeds.clone(), - records: records.clone(), - rollups: rollups.clone(), - }; - let writer = FjallWriter { - bg_taken: Arc::new(AtomicBool::new(false)), - keyspace, - global, - feeds, - records, - rollups, - queues, - }; - Ok((reader, writer, js_cursor, sketch_secret)) - } -} - -type FjallRKV = fjall::Result<(fjall::Slice, fjall::Slice)>; - -#[derive(Clone)] -pub struct FjallReader { - keyspace: Keyspace, - global: PartitionHandle, - feeds: PartitionHandle, - records: PartitionHandle, - rollups: PartitionHandle, -} - -/// An iterator that knows how to skip over deleted/invalidated records -struct RecordIterator { - db_iter: Box>, - records: PartitionHandle, - limit: usize, - fetched: usize, -} -impl RecordIterator { - pub fn new( - feeds: &PartitionHandle, - records: PartitionHandle, - collection: &Nsid, - limit: usize, - ) -> StorageResult { - let prefix = NsidRecordFeedKey::from_prefix_to_db_bytes(collection)?; - let db_iter = feeds.prefix(prefix).rev(); - Ok(Self { - db_iter: Box::new(db_iter), - records, - limit, - fetched: 0, - }) - } - fn get_record(&self, db_next: FjallRKV) -> StorageResult> { - let (key_bytes, val_bytes) = db_next?; - let feed_key = db_complete::(&key_bytes)?; - let feed_val = db_complete::(&val_bytes)?; - let location_key: RecordLocationKey = (&feed_key, &feed_val).into(); - - let Some(location_val_bytes) = self.records.get(location_key.to_db_bytes()?)? else { - // record was deleted (hopefully) - return Ok(None); - }; - - let (meta, n) = RecordLocationMeta::from_db_bytes(&location_val_bytes)?; - - if meta.cursor() != feed_key.cursor() { - // older/different version - return Ok(None); - } - if meta.rev != feed_val.rev() { - // weird... - log::warn!("record lookup: cursor match but rev did not...? excluding."); - return Ok(None); - } - let Some(raw_value_bytes) = location_val_bytes.get(n..) else { - log::warn!( - "record lookup: found record but could not get bytes to decode the record??" - ); - return Ok(None); - }; - let rawval = db_complete::(raw_value_bytes)?; - Ok(Some(UFOsRecord { - collection: feed_key.collection().clone(), - cursor: feed_key.cursor(), - did: feed_val.did().clone(), - rkey: feed_val.rkey().clone(), - rev: meta.rev.to_string(), - record: rawval.try_into()?, - is_update: meta.is_update, - })) - } -} -impl Iterator for RecordIterator { - type Item = StorageResult>; - fn next(&mut self) -> Option { - if self.fetched == self.limit { - return Some(Ok(None)); - } - let record = loop { - let db_next = self.db_iter.next()?; // None short-circuits here - match self.get_record(db_next) { - Err(e) => return Some(Err(e)), - Ok(Some(record)) => break record, - Ok(None) => continue, - } - }; - self.fetched += 1; - Some(Ok(Some(record))) - } -} - -type GetCounts = Box StorageResult>; -type GetByterCounts = StorageResult<(Nsid, GetCounts)>; -type NsidCounter = Box>; -fn get_lexi_iter( - snapshot: &Snapshot, - start: Bound>, - end: Bound>, -) -> StorageResult { - Ok(Box::new(snapshot.range((start, end)).map(|kv| { - let (k_bytes, v_bytes) = kv?; - let key = db_complete::(&k_bytes)?; - let nsid = key.collection().clone(); - let get_counts: GetCounts = Box::new(move || Ok(db_complete::(&v_bytes)?)); - Ok((nsid, get_counts)) - }))) -} -type GetRollupKey = Arc EncodingResult>>; -fn get_lookup_iter( - snapshot: lsm_tree::Snapshot, - start: Bound>, - end: Bound>, - get_rollup_key: GetRollupKey, -) -> StorageResult { - Ok(Box::new(snapshot.range((start, end)).rev().map( - move |kv| { - let (k_bytes, _) = kv?; - let key = db_complete::(&k_bytes)?; - let nsid = key.collection().clone(); - let get_counts: GetCounts = Box::new({ - let nsid = nsid.clone(); - let snapshot = snapshot.clone(); - let get_rollup_key = get_rollup_key.clone(); - move || { - let db_count_bytes = snapshot.get(get_rollup_key(&nsid)?)?.expect( - "integrity: all-time rank rollup must have corresponding all-time count rollup", - ); - Ok(db_complete::(&db_count_bytes)?) - } - }); - Ok((nsid, get_counts)) - }, - ))) -} - -type CollectionSerieses = HashMap>; - -impl FjallReader { - fn get_storage_stats(&self) -> StorageResult { - let rollup_cursor = - get_static_neu::(&self.global)? - .map(|c| c.to_raw_u64()); - - Ok(serde_json::json!({ - "keyspace_disk_space": self.keyspace.disk_space(), - "keyspace_journal_count": self.keyspace.journal_count(), - "keyspace_sequence": self.keyspace.instant(), - "rollup_cursor": rollup_cursor, - })) - } - - fn get_consumer_info(&self) -> StorageResult { - let global = self.global.snapshot(); - - let endpoint = - get_snapshot_static_neu::(&global)? - .ok_or(StorageError::BadStateError( - "Could not find jetstream endpoint".to_string(), - ))? - .0; - - let started_at = get_snapshot_static_neu::(&global)? - .ok_or(StorageError::BadStateError( - "Could not find jetstream takeoff time".to_string(), - ))? - .to_raw_u64(); - - let latest_cursor = - get_snapshot_static_neu::(&global)? - .map(|c| c.to_raw_u64()); - - let rollup_cursor = - get_snapshot_static_neu::(&global)? - .map(|c| c.to_raw_u64()); - - Ok(ConsumerInfo::Jetstream { - endpoint, - started_at, - latest_cursor, - rollup_cursor, - }) - } - - fn get_earliest_hour(&self, rollups: Option<&Snapshot>) -> StorageResult { - let cursor = rollups - .unwrap_or(&self.rollups.snapshot()) - .prefix(HourlyRollupStaticPrefix::default().to_db_bytes()?) - .next() - .transpose()? - .map(|(key_bytes, _)| db_complete::(&key_bytes)) - .transpose()? - .map(|key| key.cursor()) - .unwrap_or_else(|| Cursor::from_start().into()); - Ok(cursor) - } - - fn get_lexi_collections( - &self, - snapshot: Snapshot, - limit: usize, - cursor: Option>, - buckets: Vec, - ) -> StorageResult<(Vec, Option>)> { - let cursor_nsid = cursor.as_deref().map(db_complete::).transpose()?; - let mut iters: Vec> = Vec::with_capacity(buckets.len()); - for bucket in &buckets { - let it: NsidCounter = match bucket { - CursorBucket::Hour(t) => { - let start = cursor_nsid - .as_ref() - .map(|nsid| HourlyRollupKey::after_nsid(*t, nsid)) - .unwrap_or_else(|| HourlyRollupKey::start(*t))?; - let end = HourlyRollupKey::end(*t)?; - get_lexi_iter::(&snapshot, start, end)? - } - CursorBucket::Week(t) => { - let start = cursor_nsid - .as_ref() - .map(|nsid| WeeklyRollupKey::after_nsid(*t, nsid)) - .unwrap_or_else(|| WeeklyRollupKey::start(*t))?; - let end = WeeklyRollupKey::end(*t)?; - get_lexi_iter::(&snapshot, start, end)? - } - CursorBucket::AllTime => { - let start = cursor_nsid - .as_ref() - .map(AllTimeRollupKey::after_nsid) - .unwrap_or_else(AllTimeRollupKey::start)?; - let end = AllTimeRollupKey::end()?; - get_lexi_iter::(&snapshot, start, end)? - } - }; - iters.push(it.peekable()); - } - - let mut out = Vec::new(); - let mut current_nsid = None; - for _ in 0..limit { - // double-scan the iters for each element: this could be eliminated but we're starting simple. - // first scan: find the lowest nsid - // second scan: take + merge, and advance all iters with lowest nsid - let mut lowest: Option = None; - for iter in &mut iters { - if let Some(bla) = iter.peek_mut() { - let (nsid, _) = match bla { - Ok(v) => v, - Err(e) => Err(std::mem::replace(e, StorageError::Stolen))?, - }; - lowest = match lowest { - Some(ref current) if nsid.as_str() > current.as_str() => lowest, - _ => Some(nsid.clone()), - }; - } - } - current_nsid = lowest.clone(); - let Some(nsid) = lowest else { break }; - - let mut merged = CountsValue::default(); - for iter in &mut iters { - // unwrap: potential fjall error was already checked & bailed over when peeking in the first loop - if let Some(Ok((_, get_counts))) = iter.next_if(|v| v.as_ref().unwrap().0 == nsid) { - let counts = get_counts()?; - merged.merge(&counts); - } - } - out.push(NsidCount::new(&nsid, &merged)); - } - - let next_cursor = current_nsid.map(|s| s.to_db_bytes()).transpose()?; - Ok((out, next_cursor)) - } - - fn get_ordered_collections( - &self, - snapshot: Snapshot, - limit: usize, - order: OrderCollectionsBy, - buckets: Vec, - ) -> StorageResult> { - let mut iters: Vec = Vec::with_capacity(buckets.len()); - - for bucket in buckets { - let it: NsidCounter = match (&order, bucket) { - (OrderCollectionsBy::RecordsCreated, CursorBucket::Hour(t)) => { - get_lookup_iter::( - snapshot.clone(), - HourlyRecordsKey::start(t)?, - HourlyRecordsKey::end(t)?, - Arc::new({ - move |collection| HourlyRollupKey::new(t, collection).to_db_bytes() - }), - )? - } - (OrderCollectionsBy::DidsEstimate, CursorBucket::Hour(t)) => { - get_lookup_iter::( - snapshot.clone(), - HourlyDidsKey::start(t)?, - HourlyDidsKey::end(t)?, - Arc::new({ - move |collection| HourlyRollupKey::new(t, collection).to_db_bytes() - }), - )? - } - (OrderCollectionsBy::RecordsCreated, CursorBucket::Week(t)) => { - get_lookup_iter::( - snapshot.clone(), - WeeklyRecordsKey::start(t)?, - WeeklyRecordsKey::end(t)?, - Arc::new({ - move |collection| WeeklyRollupKey::new(t, collection).to_db_bytes() - }), - )? - } - (OrderCollectionsBy::DidsEstimate, CursorBucket::Week(t)) => { - get_lookup_iter::( - snapshot.clone(), - WeeklyDidsKey::start(t)?, - WeeklyDidsKey::end(t)?, - Arc::new({ - move |collection| WeeklyRollupKey::new(t, collection).to_db_bytes() - }), - )? - } - (OrderCollectionsBy::RecordsCreated, CursorBucket::AllTime) => { - get_lookup_iter::( - snapshot.clone(), - AllTimeRecordsKey::start()?, - AllTimeRecordsKey::end()?, - Arc::new(|collection| AllTimeRollupKey::new(collection).to_db_bytes()), - )? - } - (OrderCollectionsBy::DidsEstimate, CursorBucket::AllTime) => { - get_lookup_iter::( - snapshot.clone(), - AllTimeDidsKey::start()?, - AllTimeDidsKey::end()?, - Arc::new(|collection| AllTimeRollupKey::new(collection).to_db_bytes()), - )? - } - (OrderCollectionsBy::Lexi { .. }, _) => unreachable!(), - }; - iters.push(it); - } - - // overfetch by taking a bit more than the limit - // merge by collection - // sort by requested order, take limit, discard all remaining - // - // this isn't guaranteed to be correct, but it will hopefully be close most of the time: - // - it's possible that some NSIDs might score low during some time-buckets, and miss being merged - // - overfetching hopefully helps a bit by catching nsids near the threshold more often, but. yeah. - // - // this thing is heavy, there's probably a better way - let mut ranked: HashMap = HashMap::with_capacity(limit * 2); - for iter in iters { - for pair in iter.take((limit as f64 * 1.3).ceil() as usize) { - let (nsid, get_counts) = pair?; - let counts = get_counts()?; - ranked.entry(nsid).or_default().merge(&counts); - } - } - let mut ranked: Vec<(Nsid, CountsValue)> = ranked.into_iter().collect(); - match order { - OrderCollectionsBy::RecordsCreated => ranked.sort_by_key(|(_, c)| c.counts().creates), - OrderCollectionsBy::DidsEstimate => ranked.sort_by_key(|(_, c)| c.dids().estimate()), - OrderCollectionsBy::Lexi { .. } => unreachable!(), - } - let counts = ranked - .into_iter() - .rev() - .take(limit) - .map(|(nsid, cv)| NsidCount::new(&nsid, &cv)) - .collect(); - Ok(counts) - } - - fn get_collections( - &self, - limit: usize, - order: OrderCollectionsBy, - since: Option, - until: Option, - ) -> StorageResult<(Vec, Option>)> { - let snapshot = self.rollups.snapshot(); - let buckets = if let (None, None) = (since, until) { - vec![CursorBucket::AllTime] - } else { - let mut lower = self.get_earliest_hour(Some(&snapshot))?; - if let Some(specified) = since { - if specified > lower { - lower = specified; - } - } - let upper = until.unwrap_or_else(|| Cursor::at(SystemTime::now()).into()); - CursorBucket::buckets_spanning(lower, upper) - }; - match order { - OrderCollectionsBy::Lexi { cursor } => { - self.get_lexi_collections(snapshot, limit, cursor, buckets) - } - _ => Ok(( - self.get_ordered_collections(snapshot, limit, order, buckets)?, - None, - )), - } - } - - fn get_lexi_prefix( - &self, - snapshot: Snapshot, - prefix: NsidPrefix, - limit: usize, - cursor: Option>, - buckets: Vec, - ) -> StorageResult<(JustCount, Vec, Option>)> { - // let prefix_sub_with_null = prefix.as_str().to_string().to_db_bytes()?; - let prefix_sub = String::sub_prefix(&prefix.terminated())?; // with trailing dot to ensure full segment match - let cursor_child = cursor - .as_deref() - .map(|encoded_bytes| { - let decoded: String = db_complete(encoded_bytes)?; - // TODO: write some tests for cursors, there's probably bugs here - let as_sub_prefix_with_null = decoded.to_db_bytes()?; - Ok::<_, EncodingError>(as_sub_prefix_with_null) - }) - .transpose()?; - let mut iters: Vec = Vec::with_capacity(buckets.len()); - for bucket in &buckets { - let it: NsidCounter = match bucket { - CursorBucket::Hour(t) => { - let start = cursor_child - .as_ref() - .map(|child| HourlyRollupKey::after_nsid_prefix(*t, child)) - .unwrap_or_else(|| HourlyRollupKey::after_nsid_prefix(*t, &prefix_sub))?; - let end = HourlyRollupKey::nsid_prefix_end(*t, &prefix_sub)?; - get_lexi_iter::(&snapshot, start, end)? - } - CursorBucket::Week(t) => { - let start = cursor_child - .as_ref() - .map(|child| WeeklyRollupKey::after_nsid_prefix(*t, child)) - .unwrap_or_else(|| WeeklyRollupKey::after_nsid_prefix(*t, &prefix_sub))?; - let end = WeeklyRollupKey::nsid_prefix_end(*t, &prefix_sub)?; - get_lexi_iter::(&snapshot, start, end)? - } - CursorBucket::AllTime => { - let start = cursor_child - .as_ref() - .map(|child| AllTimeRollupKey::after_nsid_prefix(child)) - .unwrap_or_else(|| AllTimeRollupKey::after_nsid_prefix(&prefix_sub))?; - let end = AllTimeRollupKey::nsid_prefix_end(&prefix_sub)?; - get_lexi_iter::(&snapshot, start, end)? - } - }; - iters.push(it); - } - - // with apologies - let mut iters: Vec<_> = iters - .into_iter() - .map(|it| { - it.map(|bla| { - bla.map(|(nsid, v)| { - let Some(child) = Child::from_prefix(&nsid, &prefix) else { - panic!("failed from_prefix: {nsid:?} {prefix:?} (bad iter bounds?)"); - }; - (child, v) - }) - }) - .peekable() - }) - .collect(); - - let mut items = Vec::new(); - let mut prefix_count = CountsValue::default(); - #[derive(Debug, Clone, PartialEq)] - enum Child { - FullNsid(Nsid), - ChildPrefix(String), - } - impl Child { - fn from_prefix(nsid: &Nsid, prefix: &NsidPrefix) -> Option { - if prefix.is_group_of(nsid) { - return Some(Child::FullNsid(nsid.clone())); - } - let suffix = nsid.as_str().strip_prefix(&format!("{}.", prefix.0))?; - let (segment, _) = suffix.split_once('.').unwrap(); - let child_prefix = format!("{}.{segment}", prefix.0); - Some(Child::ChildPrefix(child_prefix)) - } - fn is_before(&self, other: &Child) -> bool { - match (self, other) { - (Child::FullNsid(s), Child::ChildPrefix(o)) if s.as_str() == o => true, - (Child::ChildPrefix(s), Child::FullNsid(o)) if s == o.as_str() => false, - (Child::FullNsid(s), Child::FullNsid(o)) => s.as_str() < o.as_str(), - (Child::ChildPrefix(s), Child::ChildPrefix(o)) => s < o, - (Child::FullNsid(s), Child::ChildPrefix(o)) => s.to_string() < *o, - (Child::ChildPrefix(s), Child::FullNsid(o)) => *s < o.to_string(), - } - } - fn into_inner(self) -> String { - match self { - Child::FullNsid(s) => s.to_string(), - Child::ChildPrefix(s) => s, - } - } - } - let mut current_child: Option = None; - for _ in 0..limit { - // double-scan the iters for each element: this could be eliminated but we're starting simple. - // first scan: find the lowest nsid - // second scan: take + merge, and advance all iters with lowest nsid - let mut lowest: Option = None; - for iter in &mut iters { - if let Some(bla) = iter.peek_mut() { - let (child, _) = match bla { - Ok(v) => v, - Err(e) => Err(std::mem::replace(e, StorageError::Stolen))?, - }; - - lowest = match lowest { - Some(ref current) if current.is_before(child) => lowest, - _ => Some(child.clone()), - }; - } - } - current_child = lowest.clone(); - let Some(child) = lowest else { break }; - - let mut merged = CountsValue::default(); - for iter in &mut iters { - // unwrap: potential fjall error was already checked & bailed over when peeking in the first loop - while let Some(Ok((_, get_counts))) = - iter.next_if(|v| v.as_ref().unwrap().0 == child) - { - let counts = get_counts()?; - prefix_count.merge(&counts); - merged.merge(&counts); - } - } - items.push(match child { - Child::FullNsid(nsid) => PrefixChild::Collection(NsidCount::new(&nsid, &merged)), - Child::ChildPrefix(prefix) => { - PrefixChild::Prefix(PrefixCount::new(&prefix, &merged)) - } - }); - } - - // TODO: could serialize the prefix count (with sketch) into the cursor so that uniqs can actually count up? - // ....er the sketch is probably too big - // TODO: this is probably buggy on child-type boundaries bleh - let next_cursor = current_child - .map(|s| s.into_inner().to_db_bytes()) - .transpose()?; - - Ok(((&prefix_count).into(), items, next_cursor)) - } - - fn get_prefix( - &self, - prefix: NsidPrefix, - limit: usize, - order: OrderCollectionsBy, - since: Option, - until: Option, - ) -> StorageResult<(JustCount, Vec, Option>)> { - let snapshot = self.rollups.snapshot(); - let buckets = if let (None, None) = (since, until) { - vec![CursorBucket::AllTime] - } else { - let mut lower = self.get_earliest_hour(Some(&snapshot))?; - if let Some(specified) = since { - if specified > lower { - lower = specified; - } - } - let upper = until.unwrap_or_else(|| Cursor::at(SystemTime::now()).into()); - CursorBucket::buckets_spanning(lower, upper) - }; - match order { - OrderCollectionsBy::Lexi { cursor } => { - self.get_lexi_prefix(snapshot, prefix, limit, cursor, buckets) - } - _ => todo!(), - } - } - - /// - step: output series time step, in seconds - fn get_timeseries( - &self, - collections: Vec, - since: HourTruncatedCursor, - until: Option, - step: u64, - ) -> StorageResult<(Vec, CollectionSerieses)> { - if step > WEEK_IN_MICROS { - panic!("week-stepping is todo"); - } - let until = until.unwrap_or_else(|| Cursor::at(SystemTime::now()).into()); - let Ok(dt) = Cursor::from(until).duration_since(&Cursor::from(since)) else { - return Ok(( - // empty: until < since - vec![], - collections.into_iter().map(|c| (c, vec![])).collect(), - )); - }; - let n_hours = (dt.as_micros() as u64) / HOUR_IN_MICROS; - let mut counts_by_hour = Vec::with_capacity(n_hours as usize); - let snapshot = self.rollups.snapshot(); - for hour in (0..n_hours).map(|i| since.nth_next(i)) { - let mut counts = Vec::with_capacity(collections.len()); - for nsid in &collections { - let count = snapshot - .get(&HourlyRollupKey::new(hour, nsid).to_db_bytes()?)? - .as_deref() - .map(db_complete::) - .transpose()? - .unwrap_or_default(); - counts.push(count); - } - counts_by_hour.push((hour, counts)); - } - - let step_hours = step / (HOUR_IN_MICROS / 1_000_000); - let mut output_hours = Vec::with_capacity(step_hours as usize); - let mut output_series: CollectionSerieses = collections - .iter() - .map(|c| (c.clone(), Vec::with_capacity(step_hours as usize))) - .collect(); - - for chunk in counts_by_hour.chunks(step_hours as usize) { - output_hours.push(chunk[0].0); // always guaranteed to have at least one element in a chunks chunk - for (i, collection) in collections.iter().enumerate() { - let mut c = CountsValue::default(); - for (_, counts) in chunk { - c.merge(&counts[i]); - } - output_series - .get_mut(collection) - .expect("output series is initialized with all collections") - .push(c); - } - } - - Ok((output_hours, output_series)) - } - - fn get_collection_counts( - &self, - collection: &Nsid, - since: HourTruncatedCursor, - until: Option, - ) -> StorageResult { - // grab snapshots in case rollups happen while we're working - let rollups = self.rollups.snapshot(); - - let until = until.unwrap_or_else(|| Cursor::at(SystemTime::now()).into()); - let buckets = CursorBucket::buckets_spanning(since, until); - let mut total_counts = CountsValue::default(); - - for bucket in buckets { - let key = match bucket { - CursorBucket::Hour(t) => HourlyRollupKey::new(t, collection).to_db_bytes()?, - CursorBucket::Week(t) => WeeklyRollupKey::new(t, collection).to_db_bytes()?, - CursorBucket::AllTime => unreachable!(), // TODO: fall back on this if the time span spans the whole dataset? - }; - let count = rollups - .get(&key)? - .as_deref() - .map(db_complete::) - .transpose()? - .unwrap_or_default(); - total_counts.merge(&count); - } - - Ok((&total_counts).into()) - } - - fn get_records_by_collections( - &self, - collections: HashSet, - limit: usize, - expand_each_collection: bool, - ) -> StorageResult> { - if collections.is_empty() { - return Ok(vec![]); - } - let mut record_iterators = Vec::new(); - for collection in collections { - let iter = RecordIterator::new(&self.feeds, self.records.clone(), &collection, limit)?; - record_iterators.push(iter.peekable()); - } - let mut merged = Vec::new(); - loop { - let mut latest: Option<(Cursor, usize)> = None; // ugh - for (i, iter) in record_iterators.iter_mut().enumerate() { - let Some(it) = iter.peek_mut() else { - continue; - }; - let it = match it { - Ok(v) => v, - Err(e) => Err(std::mem::replace(e, StorageError::Stolen))?, - }; - let Some(rec) = it else { - if expand_each_collection { - continue; - } else { - break; - } - }; - if let Some((cursor, _)) = latest { - if rec.cursor > cursor { - latest = Some((rec.cursor, i)) - } - } else { - latest = Some((rec.cursor, i)); - } - } - let Some((_, idx)) = latest else { - break; - }; - // yeah yeah whateverrrrrrrrrrrrrrrr - merged.push(record_iterators[idx].next().unwrap().unwrap().unwrap()); - } - Ok(merged) - } - - fn search_collections(&self, terms: Vec) -> StorageResult> { - let start = AllTimeRollupKey::start()?; - let end = AllTimeRollupKey::end()?; - let mut matches = Vec::new(); - let limit = 16; // TODO: param - for kv in self.rollups.range((start, end)) { - let (key_bytes, val_bytes) = kv?; - let key = db_complete::(&key_bytes)?; - let nsid = key.collection(); - for term in &terms { - if nsid.contains(term) { - let counts = db_complete::(&val_bytes)?; - matches.push(NsidCount::new(nsid, &counts)); - break; - } - } - if matches.len() >= limit { - break; - } - } - // TODO: indicate incomplete results - Ok(matches) - } -} - -#[async_trait] -impl StoreReader for FjallReader { - fn name(&self) -> String { - "fjall storage v2".into() - } - async fn get_storage_stats(&self) -> StorageResult { - let s = self.clone(); - tokio::task::spawn_blocking(move || FjallReader::get_storage_stats(&s)).await? - } - async fn get_consumer_info(&self) -> StorageResult { - let s = self.clone(); - tokio::task::spawn_blocking(move || FjallReader::get_consumer_info(&s)).await? - } - async fn get_collections( - &self, - limit: usize, - order: OrderCollectionsBy, - since: Option, - until: Option, - ) -> StorageResult<(Vec, Option>)> { - let s = self.clone(); - tokio::task::spawn_blocking(move || { - FjallReader::get_collections(&s, limit, order, since, until) - }) - .await? - } - async fn get_prefix( - &self, - prefix: NsidPrefix, - limit: usize, - order: OrderCollectionsBy, - since: Option, - until: Option, - ) -> StorageResult<(JustCount, Vec, Option>)> { - let s = self.clone(); - tokio::task::spawn_blocking(move || { - FjallReader::get_prefix(&s, prefix, limit, order, since, until) - }) - .await? - } - async fn get_timeseries( - &self, - collections: Vec, - since: HourTruncatedCursor, - until: Option, - step: u64, - ) -> StorageResult<(Vec, CollectionSerieses)> { - let s = self.clone(); - tokio::task::spawn_blocking(move || { - FjallReader::get_timeseries(&s, collections, since, until, step) - }) - .await? - } - async fn get_collection_counts( - &self, - collection: &Nsid, - since: HourTruncatedCursor, - until: Option, - ) -> StorageResult { - let s = self.clone(); - let collection = collection.clone(); - tokio::task::spawn_blocking(move || { - FjallReader::get_collection_counts(&s, &collection, since, until) - }) - .await? - } - async fn get_records_by_collections( - &self, - collections: HashSet, - limit: usize, - expand_each_collection: bool, - ) -> StorageResult> { - let s = self.clone(); - tokio::task::spawn_blocking(move || { - FjallReader::get_records_by_collections(&s, collections, limit, expand_each_collection) - }) - .await? - } - async fn search_collections(&self, terms: Vec) -> StorageResult> { - let s = self.clone(); - tokio::task::spawn_blocking(move || FjallReader::search_collections(&s, terms)).await? - } -} - -#[derive(Clone)] -pub struct FjallWriter { - bg_taken: Arc, - keyspace: Keyspace, - global: PartitionHandle, - feeds: PartitionHandle, - records: PartitionHandle, - rollups: PartitionHandle, - queues: PartitionHandle, -} - -impl FjallWriter { - fn rollup_delete_account( - &mut self, - cursor: Cursor, - key_bytes: &[u8], - val_bytes: &[u8], - ) -> StorageResult { - let did = db_complete::(val_bytes)?; - self.delete_account(&did)?; - let mut batch = self.keyspace.batch(); - batch.remove(&self.queues, key_bytes); - insert_batch_static_neu::(&mut batch, &self.global, cursor)?; - batch.commit()?; - Ok(1) - } - - fn rollup_live_counts( - &mut self, - timelies: impl Iterator>, - cursor_exclusive_limit: Option, - rollup_limit: usize, - ) -> StorageResult<(usize, HashSet)> { - // current strategy is to buffer counts in mem before writing the rollups - // we *could* read+write every single batch to rollup.. but their merge is associative so - // ...so save the db some work up front? is this worth it? who knows... - - let mut dirty_nsids = HashSet::new(); - - #[derive(Eq, Hash, PartialEq)] - enum Rollup { - Hourly(HourTruncatedCursor), - Weekly(WeekTruncatedCursor), - AllTime, - } - - let mut batch = self.keyspace.batch(); - let mut cursors_advanced = 0; - let mut last_cursor = Cursor::from_start(); - let mut counts_by_rollup: HashMap<(Nsid, Rollup), CountsValue> = HashMap::new(); - - for (i, kv) in timelies.enumerate() { - if i >= rollup_limit { - break; - } - - let (key_bytes, val_bytes) = kv?; - let key = db_complete::(&key_bytes)?; - - if cursor_exclusive_limit - .map(|limit| key.cursor() > limit) - .unwrap_or(false) - { - break; - } - - dirty_nsids.insert(key.collection().clone()); - - batch.remove(&self.rollups, key_bytes); - let val = db_complete::(&val_bytes)?; - counts_by_rollup - .entry(( - key.collection().clone(), - Rollup::Hourly(key.cursor().into()), - )) - .or_default() - .merge(&val); - counts_by_rollup - .entry(( - key.collection().clone(), - Rollup::Weekly(key.cursor().into()), - )) - .or_default() - .merge(&val); - counts_by_rollup - .entry((key.collection().clone(), Rollup::AllTime)) - .or_default() - .merge(&val); - - cursors_advanced += 1; - last_cursor = key.cursor(); - } - - // go through each new rollup thing and merge it with whatever might already be in the db - for ((nsid, rollup), counts) in counts_by_rollup { - let rollup_key_bytes = match rollup { - Rollup::Hourly(hourly_cursor) => { - HourlyRollupKey::new(hourly_cursor, &nsid).to_db_bytes()? - } - Rollup::Weekly(weekly_cursor) => { - WeeklyRollupKey::new(weekly_cursor, &nsid).to_db_bytes()? - } - Rollup::AllTime => AllTimeRollupKey::new(&nsid).to_db_bytes()?, - }; - let mut rolled: CountsValue = self - .rollups - .get(&rollup_key_bytes)? - .as_deref() - .map(db_complete::) - .transpose()? - .unwrap_or_default(); - - // now that we have values, we can know the exising ranks - let before_creates_count = rolled.counts().creates; - let before_dids_estimate = rolled.dids().estimate() as u64; - - // update the rollup - rolled.merge(&counts); - - // new ranks - let new_creates_count = rolled.counts().creates; - let new_dids_estimate = rolled.dids().estimate() as u64; - - // update create-ranked secondary index if rank changed - if new_creates_count != before_creates_count { - let (old_k, new_k) = match rollup { - Rollup::Hourly(cursor) => ( - HourlyRecordsKey::new(cursor, before_creates_count.into(), &nsid) - .to_db_bytes()?, - HourlyRecordsKey::new(cursor, new_creates_count.into(), &nsid) - .to_db_bytes()?, - ), - Rollup::Weekly(cursor) => ( - WeeklyRecordsKey::new(cursor, before_creates_count.into(), &nsid) - .to_db_bytes()?, - WeeklyRecordsKey::new(cursor, new_creates_count.into(), &nsid) - .to_db_bytes()?, - ), - Rollup::AllTime => ( - AllTimeRecordsKey::new(before_creates_count.into(), &nsid).to_db_bytes()?, - AllTimeRecordsKey::new(new_creates_count.into(), &nsid).to_db_bytes()?, - ), - }; - batch.remove(&self.rollups, &old_k); // TODO: when fjall gets weak delete, this will hopefully work way better - batch.insert(&self.rollups, &new_k, ""); - } - - // update dids-ranked secondary index if rank changed - if new_dids_estimate != before_dids_estimate { - let (old_k, new_k) = match rollup { - Rollup::Hourly(cursor) => ( - HourlyDidsKey::new(cursor, before_dids_estimate.into(), &nsid) - .to_db_bytes()?, - HourlyDidsKey::new(cursor, new_dids_estimate.into(), &nsid) - .to_db_bytes()?, - ), - Rollup::Weekly(cursor) => ( - WeeklyDidsKey::new(cursor, before_dids_estimate.into(), &nsid) - .to_db_bytes()?, - WeeklyDidsKey::new(cursor, new_dids_estimate.into(), &nsid) - .to_db_bytes()?, - ), - Rollup::AllTime => ( - AllTimeDidsKey::new(before_dids_estimate.into(), &nsid).to_db_bytes()?, - AllTimeDidsKey::new(new_dids_estimate.into(), &nsid).to_db_bytes()?, - ), - }; - batch.remove(&self.rollups, &old_k); // TODO: when fjall gets weak delete, this will hopefully work way better - batch.insert(&self.rollups, &new_k, ""); - } - - // replace the main counts rollup - batch.insert(&self.rollups, &rollup_key_bytes, &rolled.to_db_bytes()?); - } - - insert_batch_static_neu::(&mut batch, &self.global, last_cursor)?; - - batch.commit()?; - Ok((cursors_advanced, dirty_nsids)) - } -} - -impl StoreWriter for FjallWriter { - fn background_tasks(&mut self, reroll: bool) -> StorageResult { - if self.bg_taken.swap(true, Ordering::SeqCst) { - return Err(StorageError::BackgroundAlreadyStarted); - } - describe_histogram!( - "storage_trim_dirty_nsids", - Unit::Count, - "number of NSIDs trimmed" - ); - describe_histogram!( - "storage_trim_duration", - Unit::Microseconds, - "how long it took to trim the dirty NSIDs" - ); - describe_counter!( - "storage_trim_removed", - Unit::Count, - "how many records were removed during trim" - ); - if reroll { - log::info!("reroll: resetting rollup cursor..."); - insert_static_neu::(&self.global, Cursor::from_start())?; - log::info!("reroll: clearing trim cursors..."); - let mut batch = self.keyspace.batch(); - for kv in self - .global - .prefix(TrimCollectionCursorKey::from_prefix_to_db_bytes( - &Default::default(), - )?) - { - let (k, _) = kv?; - batch.remove(&self.global, k); - } - let n = batch.len(); - batch.commit()?; - log::info!("reroll: cleared {n} trim cursors."); - } - Ok(FjallBackground(self.clone())) - } - - fn insert_batch( - &mut self, - event_batch: EventBatch, - ) -> StorageResult<()> { - if event_batch.is_empty() { - return Ok(()); - } - - let mut batch = self.keyspace.batch(); - - // would be nice not to have to iterate everything at once here - let latest = event_batch.latest_cursor().unwrap(); - - for (nsid, commits) in event_batch.commits_by_nsid { - for commit in commits.commits { - let location_key: RecordLocationKey = (&commit, &nsid).into(); - - match commit.action { - CommitAction::Cut => { - batch.remove(&self.records, &location_key.to_db_bytes()?); - } - CommitAction::Put(put_action) => { - let feed_key = NsidRecordFeedKey::from_pair(nsid.clone(), commit.cursor); - let feed_val: NsidRecordFeedVal = - (&commit.did, &commit.rkey, commit.rev.as_str()).into(); - batch.insert( - &self.feeds, - feed_key.to_db_bytes()?, - feed_val.to_db_bytes()?, - ); - - let location_val: RecordLocationVal = - (commit.cursor, commit.rev.as_str(), put_action).into(); - batch.insert( - &self.records, - &location_key.to_db_bytes()?, - &location_val.to_db_bytes()?, - ); - } - } - } - let live_counts_key: LiveCountsKey = (latest, &nsid).into(); - let counts_value = CountsValue::new( - CommitCounts { - creates: commits.creates as u64, - updates: commits.updates as u64, - deletes: commits.deletes as u64, - }, - commits.dids_estimate, - ); - batch.insert( - &self.rollups, - &live_counts_key.to_db_bytes()?, - &counts_value.to_db_bytes()?, - ); - } - - for remove in event_batch.account_removes { - let queue_key = DeleteAccountQueueKey::new(remove.cursor); - let queue_val: DeleteAccountQueueVal = remove.did; - batch.insert( - &self.queues, - &queue_key.to_db_bytes()?, - &queue_val.to_db_bytes()?, - ); - } - - batch.insert( - &self.global, - DbStaticStr::::default().to_db_bytes()?, - latest.to_db_bytes()?, - ); - - batch.commit()?; - Ok(()) - } - - fn step_rollup(&mut self) -> StorageResult<(usize, HashSet)> { - let mut dirty_nsids = HashSet::new(); - - let rollup_cursor = - get_static_neu::(&self.global)?.ok_or( - StorageError::BadStateError("Could not find current rollup cursor".to_string()), - )?; - - // timelies - let live_counts_range = LiveCountsKey::range_from_cursor(rollup_cursor)?; - let mut timely_iter = self.rollups.range(live_counts_range).peekable(); - - let timely_next = timely_iter - .peek_mut() - .map(|kv| -> StorageResult { - match kv { - Err(e) => Err(std::mem::replace(e, fjall::Error::Poisoned))?, - Ok((key_bytes, _)) => { - let key = db_complete::(key_bytes)?; - Ok(key) - } - } - }) - .transpose()?; - - // delete accounts - let delete_accounts_range = - DeleteAccountQueueKey::new(rollup_cursor).range_to_prefix_end()?; - - let next_delete = self - .queues - .range(delete_accounts_range) - .next() - .transpose()? - .map(|(key_bytes, val_bytes)| { - db_complete::(&key_bytes) - .map(|k| (k.suffix, key_bytes, val_bytes)) - }) - .transpose()?; - - let cursors_stepped = match (timely_next, next_delete) { - (Some(timely), Some((delete_cursor, delete_key_bytes, delete_val_bytes))) => { - if timely.cursor() < delete_cursor { - let (n, dirty) = self.rollup_live_counts( - timely_iter, - Some(delete_cursor), - MAX_BATCHED_ROLLUP_COUNTS, - )?; - dirty_nsids.extend(dirty); - n - } else { - self.rollup_delete_account(delete_cursor, &delete_key_bytes, &delete_val_bytes)? - } - } - (Some(_), None) => { - let (n, dirty) = - self.rollup_live_counts(timely_iter, None, MAX_BATCHED_ROLLUP_COUNTS)?; - dirty_nsids.extend(dirty); - n - } - (None, Some((delete_cursor, delete_key_bytes, delete_val_bytes))) => { - self.rollup_delete_account(delete_cursor, &delete_key_bytes, &delete_val_bytes)? - } - (None, None) => 0, - }; - - Ok((cursors_stepped, dirty_nsids)) - } - - fn trim_collection( - &mut self, - collection: &Nsid, - limit: usize, - full_scan: bool, - ) -> StorageResult<(usize, usize, bool)> { - let mut dangling_feed_keys_cleaned = 0; - let mut records_deleted = 0; - - let live_range = if full_scan { - let start = NsidRecordFeedKey::from_prefix_to_db_bytes(collection)?; - let end = NsidRecordFeedKey::prefix_range_end(collection)?; - start..end - } else { - let feed_trim_cursor_key = - TrimCollectionCursorKey::new(collection.clone()).to_db_bytes()?; - let trim_cursor = self - .global - .get(&feed_trim_cursor_key)? - .map(|value_bytes| db_complete(&value_bytes)) - .transpose()? - .unwrap_or(Cursor::from_start()); - NsidRecordFeedKey::from_pair(collection.clone(), trim_cursor).range_to_prefix_end()? - }; - - let mut live_records_found = 0; - let mut candidate_new_feed_lower_cursor = None; - let ended_early = false; - let mut current_cursor: Option = None; - for (i, kv) in self.feeds.range(live_range).rev().enumerate() { - if i > 0 && i % 500_000 == 0 { - log::info!( - "trim: at {i} for {:?} (now at {})", - collection.to_string(), - current_cursor - .map(|c| c - .elapsed() - .map(nice_duration) - .unwrap_or("[not past]".into())) - .unwrap_or("??".into()), - ); - } - let (key_bytes, val_bytes) = kv?; - let feed_key = db_complete::(&key_bytes)?; - let feed_val = db_complete::(&val_bytes)?; - let location_key: RecordLocationKey = (&feed_key, &feed_val).into(); - let location_key_bytes = location_key.to_db_bytes()?; - - let Some(location_val_bytes) = self.records.get(&location_key_bytes)? else { - // record was deleted (hopefully) - self.feeds.remove(&*key_bytes)?; - dangling_feed_keys_cleaned += 1; - continue; - }; - - let (meta, _) = RecordLocationMeta::from_db_bytes(&location_val_bytes)?; - current_cursor = Some(meta.cursor()); - - if meta.cursor() != feed_key.cursor() { - // older/different version - self.feeds.remove(&*key_bytes)?; - dangling_feed_keys_cleaned += 1; - continue; - } - if meta.rev != feed_val.rev() { - // weird... - log::warn!("record lookup: cursor match but rev did not...? removing."); - self.records.remove(&location_key_bytes)?; - self.feeds.remove(&*key_bytes)?; - dangling_feed_keys_cleaned += 1; - continue; - } - - live_records_found += 1; - if live_records_found <= limit { - continue; - } - if candidate_new_feed_lower_cursor.is_none() { - candidate_new_feed_lower_cursor = Some(feed_key.cursor()); - } - - self.feeds.remove(&location_key_bytes)?; - self.feeds.remove(key_bytes)?; - records_deleted += 1; - } - - if !ended_early { - if let Some(new_cursor) = candidate_new_feed_lower_cursor { - self.global.insert( - &TrimCollectionCursorKey::new(collection.clone()).to_db_bytes()?, - &new_cursor.to_db_bytes()?, - )?; - } - } - - log::trace!("trim_collection ({collection:?}) removed {dangling_feed_keys_cleaned} dangling feed entries and {records_deleted} records (ended early? {ended_early})"); - Ok((dangling_feed_keys_cleaned, records_deleted, ended_early)) - } - - fn delete_account(&mut self, did: &Did) -> Result { - let mut records_deleted = 0; - let mut batch = self.keyspace.batch(); - let prefix = RecordLocationKey::from_prefix_to_db_bytes(did)?; - for kv in self.records.prefix(prefix) { - let (key_bytes, _) = kv?; - batch.remove(&self.records, key_bytes); - records_deleted += 1; - if batch.len() >= MAX_BATCHED_ACCOUNT_DELETE_RECORDS { - batch.commit()?; - batch = self.keyspace.batch(); - } - } - batch.commit()?; - Ok(records_deleted) - } -} - -pub struct FjallBackground(FjallWriter); - -#[async_trait] -impl StoreBackground for FjallBackground { - async fn run(mut self, backfill: bool) -> StorageResult<()> { - let mut dirty_nsids = HashSet::new(); - - // backfill condition here is iffy -- longer is good when doing the main ingest and then collection trims - // shorter once those are done helps things catch up - // the best setting for non-backfill is non-obvious.. it can be pretty slow and still be fine - let mut rollup = - tokio::time::interval(Duration::from_micros(if backfill { 100 } else { 32_000 })); - rollup.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - - // backfill condition again iffy. collection trims should probably happen in their own phase. - let mut trim = tokio::time::interval(Duration::from_secs(if backfill { 18 } else { 9 })); - trim.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - - loop { - tokio::select! { - _ = rollup.tick() => { - let mut db = self.0.clone(); - let (n, dirty) = tokio::task::spawn_blocking(move || db.step_rollup()).await??; - if n == 0 { - rollup.reset_after(Duration::from_millis(1_200)); // we're caught up, take a break - } - dirty_nsids.extend(dirty); - log::trace!("rolled up {n} items ({} collections now dirty)", dirty_nsids.len()); - }, - _ = trim.tick() => { - let n = dirty_nsids.len(); - log::trace!("trimming {n} nsids: {dirty_nsids:?}"); - let t0 = Instant::now(); - let (mut total_danglers, mut total_deleted) = (0, 0); - let mut completed = HashSet::new(); - for collection in &dirty_nsids { - let mut db = self.0.clone(); - let c = collection.clone(); - let (danglers, deleted, ended_early) = tokio::task::spawn_blocking(move || db.trim_collection(&c, 512, false)).await??; - total_danglers += danglers; - total_deleted += deleted; - if !ended_early { - completed.insert(collection.clone()); - } - if total_deleted > 10_000_000 { - log::info!("trim stopped early, more than 10M records already deleted."); - break; - } - } - let dt = t0.elapsed(); - log::trace!("finished trimming {n} nsids in {dt:?}: {total_danglers} dangling and {total_deleted} total removed."); - histogram!("storage_trim_dirty_nsids").record(completed.len() as f64); - histogram!("storage_trim_duration").record(dt.as_micros() as f64); - counter!("storage_trim_removed", "dangling" => "true").increment(total_danglers as u64); - counter!("storage_trim_removed", "dangling" => "false").increment((total_deleted - total_danglers) as u64); - for c in completed { - dirty_nsids.remove(&c); - } - }, - }; - } - } -} - -/// Get a value from a fixed key -fn get_static_neu(global: &PartitionHandle) -> StorageResult> { - let key_bytes = DbStaticStr::::default().to_db_bytes()?; - let value = global - .get(&key_bytes)? - .map(|value_bytes| db_complete(&value_bytes)) - .transpose()?; - Ok(value) -} - -/// Get a value from a fixed key -fn get_snapshot_static_neu( - global: &fjall::Snapshot, -) -> StorageResult> { - let key_bytes = DbStaticStr::::default().to_db_bytes()?; - let value = global - .get(&key_bytes)? - .map(|value_bytes| db_complete(&value_bytes)) - .transpose()?; - Ok(value) -} - -/// Set a value to a fixed key -fn insert_static_neu( - global: &PartitionHandle, - value: impl DbBytes, -) -> StorageResult<()> { - let key_bytes = DbStaticStr::::default().to_db_bytes()?; - let value_bytes = value.to_db_bytes()?; - global.insert(&key_bytes, &value_bytes)?; - Ok(()) -} - -/// Set a value to a fixed key, erroring if the value already exists -/// -/// Intended for single-threaded init: not safe under concurrency, since there -/// is no transaction between checking if the already exists and writing it. -fn init_static_neu( - global: &PartitionHandle, - value: impl DbBytes, -) -> StorageResult<()> { - let key_bytes = DbStaticStr::::default().to_db_bytes()?; - if global.get(&key_bytes)?.is_some() { - return Err(StorageError::InitError(format!( - "init failed: value for key {key_bytes:?} already exists" - ))); - } - let value_bytes = value.to_db_bytes()?; - global.insert(&key_bytes, &value_bytes)?; - Ok(()) -} - -/// Set a value to a fixed key -fn insert_batch_static_neu( - batch: &mut FjallBatch, - global: &PartitionHandle, - value: impl DbBytes, -) -> StorageResult<()> { - let key_bytes = DbStaticStr::::default().to_db_bytes()?; - let value_bytes = value.to_db_bytes()?; - batch.insert(global, &key_bytes, &value_bytes); - Ok(()) -} - -#[derive(Debug, serde::Serialize, schemars::JsonSchema)] -pub struct StorageInfo { - pub keyspace_disk_space: u64, - pub keyspace_journal_count: usize, - pub keyspace_sequence: u64, - pub global_approximate_len: usize, -} - -////////// temp stuff to remove: - -#[cfg(test)] -mod tests { - use super::*; - use crate::{DeleteAccount, RecordKey, UFOsCommit}; - use jetstream::events::{CommitEvent, CommitOp}; - use jetstream::exports::Cid; - use serde_json::value::RawValue; - - fn fjall_db() -> (FjallReader, FjallWriter) { - let (read, write, _, _) = FjallStorage::init( - tempfile::tempdir().unwrap(), - "offline test (no real jetstream endpoint)".to_string(), - false, - FjallConfig { temp: true }, - ) - .unwrap(); - (read, write) - } - - const TEST_BATCH_LIMIT: usize = 16; - fn beginning() -> HourTruncatedCursor { - Cursor::from_start().into() - } - - #[derive(Debug, Default)] - struct TestBatch { - pub batch: EventBatch, - } - - impl TestBatch { - #[allow(clippy::too_many_arguments)] - pub fn create( - &mut self, - did: &str, - collection: &str, - rkey: &str, - record: &str, - rev: Option<&str>, - cid: Option, - cursor: u64, - ) -> Nsid { - let did = Did::new(did.to_string()).unwrap(); - let collection = Nsid::new(collection.to_string()).unwrap(); - let record = RawValue::from_string(record.to_string()).unwrap(); - let cid = cid.unwrap_or( - "bafyreidofvwoqvd2cnzbun6dkzgfucxh57tirf3ohhde7lsvh4fu3jehgy" - .parse() - .unwrap(), - ); - - let event = CommitEvent { - collection, - rkey: RecordKey::new(rkey.to_string()).unwrap(), - rev: rev.unwrap_or("asdf").to_string(), - operation: CommitOp::Create, - record: Some(record), - cid: Some(cid), - }; - - let (commit, collection) = - UFOsCommit::from_commit_info(event, did.clone(), Cursor::from_raw_u64(cursor)) - .unwrap(); - - self.batch - .commits_by_nsid - .entry(collection.clone()) - .or_default() - .truncating_insert(commit, &[0u8; 16]) - .unwrap(); - - collection - } - #[allow(clippy::too_many_arguments)] - pub fn update( - &mut self, - did: &str, - collection: &str, - rkey: &str, - record: &str, - rev: Option<&str>, - cid: Option, - cursor: u64, - ) -> Nsid { - let did = Did::new(did.to_string()).unwrap(); - let collection = Nsid::new(collection.to_string()).unwrap(); - let record = RawValue::from_string(record.to_string()).unwrap(); - let cid = cid.unwrap_or( - "bafyreidofvwoqvd2cnzbun6dkzgfucxh57tirf3ohhde7lsvh4fu3jehgy" - .parse() - .unwrap(), - ); - - let event = CommitEvent { - collection, - rkey: RecordKey::new(rkey.to_string()).unwrap(), - rev: rev.unwrap_or("asdf").to_string(), - operation: CommitOp::Update, - record: Some(record), - cid: Some(cid), - }; - - let (commit, collection) = - UFOsCommit::from_commit_info(event, did.clone(), Cursor::from_raw_u64(cursor)) - .unwrap(); - - self.batch - .commits_by_nsid - .entry(collection.clone()) - .or_default() - .truncating_insert(commit, &[0u8; 16]) - .unwrap(); - - collection - } - #[allow(clippy::too_many_arguments)] - pub fn delete( - &mut self, - did: &str, - collection: &str, - rkey: &str, - rev: Option<&str>, - cursor: u64, - ) -> Nsid { - let did = Did::new(did.to_string()).unwrap(); - let collection = Nsid::new(collection.to_string()).unwrap(); - let event = CommitEvent { - collection, - rkey: RecordKey::new(rkey.to_string()).unwrap(), - rev: rev.unwrap_or("asdf").to_string(), - operation: CommitOp::Delete, - record: None, - cid: None, - }; - - let (commit, collection) = - UFOsCommit::from_commit_info(event, did, Cursor::from_raw_u64(cursor)).unwrap(); - - self.batch - .commits_by_nsid - .entry(collection.clone()) - .or_default() - .truncating_insert(commit, &[0u8; 16]) - .unwrap(); - - collection - } - pub fn delete_account(&mut self, did: &str, cursor: u64) -> Did { - let did = Did::new(did.to_string()).unwrap(); - self.batch.account_removes.push(DeleteAccount { - did: did.clone(), - cursor: Cursor::from_raw_u64(cursor), - }); - did - } - } - - #[test] - fn test_hello() -> anyhow::Result<()> { - let (read, mut write) = fjall_db(); - write.insert_batch::(EventBatch::default())?; - let JustCount { - creates, - dids_estimate, - .. - } = read.get_collection_counts( - &Nsid::new("a.b.c".to_string()).unwrap(), - beginning(), - None, - )?; - assert_eq!(creates, 0); - assert_eq!(dids_estimate, 0); - Ok(()) - } - - #[test] - fn test_insert_one() -> anyhow::Result<()> { - let (read, mut write) = fjall_db(); - - let mut batch = TestBatch::default(); - let collection = batch.create( - "did:plc:inze6wrmsm7pjl7yta3oig77", - "a.b.c", - "asdf", - "{}", - Some("rev-z"), - None, - 100, - ); - write.insert_batch(batch.batch)?; - write.step_rollup()?; - - let JustCount { - creates, - dids_estimate, - .. - } = read.get_collection_counts(&collection, beginning(), None)?; - assert_eq!(creates, 1); - assert_eq!(dids_estimate, 1); - let JustCount { - creates, - dids_estimate, - .. - } = read.get_collection_counts( - &Nsid::new("d.e.f".to_string()).unwrap(), - beginning(), - None, - )?; - assert_eq!(creates, 0); - assert_eq!(dids_estimate, 0); - - let records = read.get_records_by_collections([collection].into(), 2, false)?; - assert_eq!(records.len(), 1); - let rec = &records[0]; - assert_eq!(rec.record.get(), "{}"); - assert!(!rec.is_update); - - let records = read.get_records_by_collections( - [Nsid::new("d.e.f".to_string()).unwrap()].into(), - 2, - false, - )?; - assert_eq!(records.len(), 0); - - Ok(()) - } - - #[test] - fn test_get_multi_collection() -> anyhow::Result<()> { - let (read, mut write) = fjall_db(); - - let mut batch = TestBatch::default(); - batch.create( - "did:plc:inze6wrmsm7pjl7yta3oig77", - "a.a.a", - "aaa", - r#""earliest""#, - Some("rev-a"), - None, - 100, - ); - batch.create( - "did:plc:inze6wrmsm7pjl7yta3oig77", - "a.a.b", - "aab", - r#""in between""#, - Some("rev-ab"), - None, - 101, - ); - batch.create( - "did:plc:inze6wrmsm7pjl7yta3oig77", - "a.a.a", - "aaa-2", - r#""last""#, - Some("rev-a-2"), - None, - 102, - ); - write.insert_batch(batch.batch)?; - - let records = read.get_records_by_collections( - HashSet::from([ - Nsid::new("a.a.a".to_string()).unwrap(), - Nsid::new("a.a.b".to_string()).unwrap(), - Nsid::new("a.a.c".to_string()).unwrap(), - ]), - 100, - false, - )?; - assert_eq!(records.len(), 3); - assert_eq!(records[0].record.get(), r#""last""#); - assert_eq!( - records[0].collection, - Nsid::new("a.a.a".to_string()).unwrap() - ); - assert_eq!(records[1].record.get(), r#""in between""#); - assert_eq!( - records[1].collection, - Nsid::new("a.a.b".to_string()).unwrap() - ); - assert_eq!(records[2].record.get(), r#""earliest""#); - assert_eq!( - records[2].collection, - Nsid::new("a.a.a".to_string()).unwrap() - ); - - Ok(()) - } - - #[test] - fn test_get_multi_collection_expanded() -> anyhow::Result<()> { - let (read, mut write) = fjall_db(); - - let mut batch = TestBatch::default(); - // insert some older ones in aab - for i in 1..=3 { - batch.create( - "did:plc:inze6wrmsm7pjl7yta3oig77", - "a.a.b", - &format!("aab-{i}"), - &format!(r#""b {i}""#), - Some(&format!("rev-b-{i}")), - None, - 100 + i, - ); - } - // and some newer ones in aaa - for i in 1..=3 { - batch.create( - "did:plc:inze6wrmsm7pjl7yta3oig77", - "a.a.a", - &format!("aaa-{i}"), - &format!(r#""a {i}""#), - Some(&format!("rev-a-{i}")), - None, - 200 + i, - ); - } - write.insert_batch(batch.batch)?; - - let records = read.get_records_by_collections( - HashSet::from([ - Nsid::new("a.a.a".to_string()).unwrap(), - Nsid::new("a.a.b".to_string()).unwrap(), - Nsid::new("a.a.c".to_string()).unwrap(), - ]), - 2, - true, - )?; - assert_eq!(records.len(), 4); - assert_eq!(records[0].record.get(), r#""a 3""#); - assert_eq!( - records[0].collection, - Nsid::new("a.a.a".to_string()).unwrap() - ); - - assert_eq!(records[3].record.get(), r#""b 2""#); - assert_eq!( - records[3].collection, - Nsid::new("a.a.b".to_string()).unwrap() - ); - - Ok(()) - } - - #[test] - fn test_update_one() -> anyhow::Result<()> { - let (read, mut write) = fjall_db(); - - let mut batch = TestBatch::default(); - let collection = batch.create( - "did:plc:inze6wrmsm7pjl7yta3oig77", - "a.b.c", - "rkey-asdf", - "{}", - Some("rev-a"), - None, - 100, - ); - write.insert_batch(batch.batch)?; - - let mut batch = TestBatch::default(); - batch.update( - "did:plc:inze6wrmsm7pjl7yta3oig77", - "a.b.c", - "rkey-asdf", - r#"{"ch": "ch-ch-ch-changes"}"#, - Some("rev-z"), - None, - 101, - ); - write.insert_batch(batch.batch)?; - write.step_rollup()?; - - let JustCount { - creates, - dids_estimate, - .. - } = read.get_collection_counts(&collection, beginning(), None)?; - assert_eq!(creates, 1); - assert_eq!(dids_estimate, 1); - - let records = read.get_records_by_collections([collection].into(), 2, false)?; - assert_eq!(records.len(), 1); - let rec = &records[0]; - assert_eq!(rec.record.get(), r#"{"ch": "ch-ch-ch-changes"}"#); - assert!(rec.is_update); - Ok(()) - } - - #[test] - fn test_delete_one() -> anyhow::Result<()> { - let (read, mut write) = fjall_db(); - - let mut batch = TestBatch::default(); - let collection = batch.create( - "did:plc:inze6wrmsm7pjl7yta3oig77", - "a.b.c", - "rkey-asdf", - "{}", - Some("rev-a"), - None, - 100, - ); - write.insert_batch(batch.batch)?; - - let mut batch = TestBatch::default(); - batch.delete( - "did:plc:inze6wrmsm7pjl7yta3oig77", - "a.b.c", - "rkey-asdf", - Some("rev-z"), - 101, - ); - write.insert_batch(batch.batch)?; - write.step_rollup()?; - - let JustCount { - creates, - dids_estimate, - .. - } = read.get_collection_counts(&collection, beginning(), None)?; - assert_eq!(creates, 1); - assert_eq!(dids_estimate, 1); - - let records = read.get_records_by_collections([collection].into(), 2, false)?; - assert_eq!(records.len(), 0); - - Ok(()) - } - - #[test] - fn test_collection_trim() -> anyhow::Result<()> { - let (read, mut write) = fjall_db(); - - let mut batch = TestBatch::default(); - batch.create( - "did:plc:inze6wrmsm7pjl7yta3oig77", - "a.a.a", - "rkey-aaa", - "{}", - Some("rev-aaa"), - None, - 10_000, - ); - let mut last_b_cursor; - for i in 1..=10 { - last_b_cursor = 11_000 + i; - batch.create( - &format!("did:plc:inze6wrmsm7pjl7yta3oig7{}", i % 3), - "a.a.b", - &format!("rkey-bbb-{i}"), - &format!(r#"{{"n": {i}}}"#), - Some(&format!("rev-bbb-{i}")), - None, - last_b_cursor, - ); - } - batch.create( - "did:plc:inze6wrmsm7pjl7yta3oig77", - "a.a.c", - "rkey-ccc", - "{}", - Some("rev-ccc"), - None, - 12_000, - ); - - write.insert_batch(batch.batch)?; - - let records = read.get_records_by_collections( - HashSet::from([Nsid::new("a.a.a".to_string()).unwrap()]), - 100, - false, - )?; - assert_eq!(records.len(), 1); - let records = read.get_records_by_collections( - HashSet::from([Nsid::new("a.a.b".to_string()).unwrap()]), - 100, - false, - )?; - assert_eq!(records.len(), 10); - let records = read.get_records_by_collections( - HashSet::from([Nsid::new("a.a.c".to_string()).unwrap()]), - 100, - false, - )?; - assert_eq!(records.len(), 1); - let records = read.get_records_by_collections( - HashSet::from([Nsid::new("a.a.d".to_string()).unwrap()]), - 100, - false, - )?; - assert_eq!(records.len(), 0); - - write.trim_collection(&Nsid::new("a.a.a".to_string()).unwrap(), 6, false)?; - write.trim_collection(&Nsid::new("a.a.b".to_string()).unwrap(), 6, false)?; - write.trim_collection(&Nsid::new("a.a.c".to_string()).unwrap(), 6, false)?; - write.trim_collection(&Nsid::new("a.a.d".to_string()).unwrap(), 6, false)?; - - let records = read.get_records_by_collections( - HashSet::from([Nsid::new("a.a.a".to_string()).unwrap()]), - 100, - false, - )?; - assert_eq!(records.len(), 1); - let records = read.get_records_by_collections( - HashSet::from([Nsid::new("a.a.b".to_string()).unwrap()]), - 100, - false, - )?; - assert_eq!(records.len(), 6); - let records = read.get_records_by_collections( - HashSet::from([Nsid::new("a.a.c".to_string()).unwrap()]), - 100, - false, - )?; - assert_eq!(records.len(), 1); - let records = read.get_records_by_collections( - HashSet::from([Nsid::new("a.a.d".to_string()).unwrap()]), - 100, - false, - )?; - assert_eq!(records.len(), 0); - - Ok(()) - } - - #[test] - fn test_delete_account() -> anyhow::Result<()> { - let (read, mut write) = fjall_db(); - - let mut batch = TestBatch::default(); - batch.create( - "did:plc:person-a", - "a.a.a", - "rkey-aaa", - "{}", - Some("rev-aaa"), - None, - 10_000, - ); - for i in 1..=2 { - batch.create( - "did:plc:person-b", - "a.a.a", - &format!("rkey-bbb-{i}"), - &format!(r#"{{"n": {i}}}"#), - Some(&format!("rev-bbb-{i}")), - None, - 11_000 + i, - ); - } - write.insert_batch(batch.batch)?; - - let records = read.get_records_by_collections( - HashSet::from([Nsid::new("a.a.a".to_string()).unwrap()]), - 100, - false, - )?; - assert_eq!(records.len(), 3); - - let records_deleted = - write.delete_account(&Did::new("did:plc:person-b".to_string()).unwrap())?; - assert_eq!(records_deleted, 2); - - let records = read.get_records_by_collections( - HashSet::from([Nsid::new("a.a.a".to_string()).unwrap()]), - 100, - false, - )?; - assert_eq!(records.len(), 1); - - Ok(()) - } - - #[test] - fn rollup_delete_account_removes_record() -> anyhow::Result<()> { - let (read, mut write) = fjall_db(); - - let mut batch = TestBatch::default(); - batch.create( - "did:plc:person-a", - "a.a.a", - "rkey-aaa", - "{}", - Some("rev-aaa"), - None, - 10_000, - ); - write.insert_batch(batch.batch)?; - - let mut batch = TestBatch::default(); - batch.delete_account("did:plc:person-a", 9_999); // queue it before the rollup - write.insert_batch(batch.batch)?; - - write.step_rollup()?; - - let records = read.get_records_by_collections( - [Nsid::new("a.a.a".to_string()).unwrap()].into(), - 1, - false, - )?; - assert_eq!(records.len(), 0); - - Ok(()) - } - - #[test] - fn rollup_delete_live_count_step() -> anyhow::Result<()> { - let (read, mut write) = fjall_db(); - - let mut batch = TestBatch::default(); - batch.create( - "did:plc:person-a", - "a.a.a", - "rkey-aaa", - "{}", - Some("rev-aaa"), - None, - 10_000, - ); - write.insert_batch(batch.batch)?; - - let (n, _) = write.step_rollup()?; - assert_eq!(n, 1); - - let mut batch = TestBatch::default(); - batch.delete_account("did:plc:person-a", 10_001); - write.insert_batch(batch.batch)?; - - let records = read.get_records_by_collections( - [Nsid::new("a.a.a".to_string()).unwrap()].into(), - 1, - false, - )?; - assert_eq!(records.len(), 1); - - let (n, _) = write.step_rollup()?; - assert_eq!(n, 1); - - let records = read.get_records_by_collections( - [Nsid::new("a.a.a".to_string()).unwrap()].into(), - 1, - false, - )?; - assert_eq!(records.len(), 0); - - let mut batch = TestBatch::default(); - batch.delete_account("did:plc:person-a", 9_999); - write.insert_batch(batch.batch)?; - - let (n, _) = write.step_rollup()?; - assert_eq!(n, 0); - - Ok(()) - } - - #[test] - fn rollup_multiple_count_batches() -> anyhow::Result<()> { - let (_read, mut write) = fjall_db(); - - let mut batch = TestBatch::default(); - batch.create( - "did:plc:person-a", - "a.a.a", - "rkey-aaa", - "{}", - Some("rev-aaa"), - None, - 10_000, - ); - write.insert_batch(batch.batch)?; - - let mut batch = TestBatch::default(); - batch.create( - "did:plc:person-a", - "a.a.a", - "rkey-aab", - "{}", - Some("rev-aab"), - None, - 10_001, - ); - write.insert_batch(batch.batch)?; - - let (n, _) = write.step_rollup()?; - assert_eq!(n, 2); - - let (n, _) = write.step_rollup()?; - assert_eq!(n, 0); - - Ok(()) - } - - #[test] - fn counts_before_and_after_rollup() -> anyhow::Result<()> { - let (read, mut write) = fjall_db(); - - let mut batch = TestBatch::default(); - batch.create( - "did:plc:person-a", - "a.a.a", - "rkey-aaa", - "{}", - Some("rev-aaa"), - None, - 10_000, - ); - batch.create( - "did:plc:person-b", - "a.a.a", - "rkey-bbb", - "{}", - Some("rev-bbb"), - None, - 10_001, - ); - write.insert_batch(batch.batch)?; - - let mut batch = TestBatch::default(); - batch.delete_account("did:plc:person-a", 11_000); - write.insert_batch(batch.batch)?; - - let mut batch = TestBatch::default(); - batch.create( - "did:plc:person-a", - "a.a.a", - "rkey-aac", - "{}", - Some("rev-aac"), - None, - 12_000, - ); - write.insert_batch(batch.batch)?; - - // before any rollup - let JustCount { - creates, - dids_estimate, - .. - } = read.get_collection_counts( - &Nsid::new("a.a.a".to_string()).unwrap(), - beginning(), - None, - )?; - assert_eq!(creates, 0); - assert_eq!(dids_estimate, 0); - - // first batch rolled up - let (n, _) = write.step_rollup()?; - assert_eq!(n, 1); - - let JustCount { - creates, - dids_estimate, - .. - } = read.get_collection_counts( - &Nsid::new("a.a.a".to_string()).unwrap(), - beginning(), - None, - )?; - assert_eq!(creates, 2); - assert_eq!(dids_estimate, 2); - - // delete account rolled up - let (n, _) = write.step_rollup()?; - assert_eq!(n, 1); - - let JustCount { - creates, - dids_estimate, - .. - } = read.get_collection_counts( - &Nsid::new("a.a.a".to_string()).unwrap(), - beginning(), - None, - )?; - assert_eq!(creates, 2); - assert_eq!(dids_estimate, 2); - - // second batch rolled up - let (n, _) = write.step_rollup()?; - assert_eq!(n, 1); - - let JustCount { - creates, - dids_estimate, - .. - } = read.get_collection_counts( - &Nsid::new("a.a.a".to_string()).unwrap(), - beginning(), - None, - )?; - assert_eq!(creates, 3); - assert_eq!(dids_estimate, 2); - - // no more rollups left - let (n, _) = write.step_rollup()?; - assert_eq!(n, 0); - - Ok(()) - } - - #[test] - fn get_prefix_children_lexi_empty() { - let (read, _) = fjall_db(); - let ( - JustCount { - creates, - dids_estimate, - .. - }, - children, - cursor, - ) = read - .get_prefix( - NsidPrefix::new("aaa.aaa").unwrap(), - 10, - OrderCollectionsBy::Lexi { cursor: None }, - None, - None, - ) - .unwrap(); - - assert_eq!(creates, 0); - assert_eq!(dids_estimate, 0); - assert_eq!(children, vec![]); - assert_eq!(cursor, None); - } - - #[test] - fn get_prefix_excludes_exact_collection() -> anyhow::Result<()> { - let (read, mut write) = fjall_db(); - - let mut batch = TestBatch::default(); - batch.create( - "did:plc:person-a", - "a.a.a", - "rkey-aaa", - "{}", - Some("rev-aaa"), - None, - 10_000, - ); - write.insert_batch(batch.batch)?; - write.step_rollup()?; - - let ( - JustCount { - creates, - dids_estimate, - .. - }, - children, - cursor, - ) = read.get_prefix( - NsidPrefix::new("a.a.a").unwrap(), - 10, - OrderCollectionsBy::Lexi { cursor: None }, - None, - None, - )?; - assert_eq!(creates, 0); - assert_eq!(dids_estimate, 0); - assert_eq!(children, vec![]); - assert_eq!(cursor, None); - Ok(()) - } - - #[test] - fn get_prefix_excludes_neighbour_collection() -> anyhow::Result<()> { - let (read, mut write) = fjall_db(); - - let mut batch = TestBatch::default(); - batch.create( - "did:plc:person-a", - "a.a.aa", - "rkey-aaa", - "{}", - Some("rev-aaa"), - None, - 10_000, - ); - write.insert_batch(batch.batch)?; - write.step_rollup()?; - - let ( - JustCount { - creates, - dids_estimate, - .. - }, - children, - cursor, - ) = read.get_prefix( - NsidPrefix::new("a.a.a").unwrap(), - 10, - OrderCollectionsBy::Lexi { cursor: None }, - None, - None, - )?; - assert_eq!(creates, 0); - assert_eq!(dids_estimate, 0); - assert_eq!(children, vec![]); - assert_eq!(cursor, None); - Ok(()) - } - - #[test] - fn get_prefix_includes_child_collection() -> anyhow::Result<()> { - let (read, mut write) = fjall_db(); - - let mut batch = TestBatch::default(); - batch.create( - "did:plc:person-a", - "a.a.a", - "rkey-aaa", - "{}", - Some("rev-aaa"), - None, - 10_000, - ); - write.insert_batch(batch.batch)?; - write.step_rollup()?; - - let ( - JustCount { - creates, - dids_estimate, - .. - }, - children, - cursor, - ) = read.get_prefix( - NsidPrefix::new("a.a").unwrap(), - 10, - OrderCollectionsBy::Lexi { cursor: None }, - None, - None, - )?; - assert_eq!(creates, 1); - assert_eq!(dids_estimate, 1); - assert_eq!( - children, - vec![PrefixChild::Collection(NsidCount { - nsid: "a.a.a".to_string(), - creates: 1, - updates: 0, - deletes: 0, - dids_estimate: 1 - }),] - ); - assert_eq!(cursor, None); - Ok(()) - } - - #[test] - fn get_prefix_includes_child_prefix() -> anyhow::Result<()> { - let (read, mut write) = fjall_db(); - - let mut batch = TestBatch::default(); - batch.create( - "did:plc:person-a", - "a.a.a.a", - "rkey-aaaa", - "{}", - Some("rev-aaaa"), - None, - 10_000, - ); - write.insert_batch(batch.batch)?; - write.step_rollup()?; - - let ( - JustCount { - creates, - dids_estimate, - .. - }, - children, - cursor, - ) = read.get_prefix( - NsidPrefix::new("a.a").unwrap(), - 10, - OrderCollectionsBy::Lexi { cursor: None }, - None, - None, - )?; - assert_eq!(creates, 1); - assert_eq!(dids_estimate, 1); - assert_eq!( - children, - vec![PrefixChild::Prefix(PrefixCount { - prefix: "a.a.a".to_string(), - creates: 1, - updates: 0, - deletes: 0, - dids_estimate: 1, - }),] - ); - assert_eq!(cursor, None); - Ok(()) - } - - #[test] - fn get_prefix_merges_child_prefixes() -> anyhow::Result<()> { - let (read, mut write) = fjall_db(); - - let mut batch = TestBatch::default(); - batch.create( - "did:plc:person-a", - "a.a.a.a", - "rkey-aaaa", - "{}", - Some("rev-aaaa"), - None, - 10_000, - ); - batch.create( - "did:plc:person-a", - "a.a.a.b", - "rkey-aaab", - "{}", - Some("rev-aaab"), - None, - 10_001, - ); - write.insert_batch(batch.batch)?; - write.step_rollup()?; - - let ( - JustCount { - creates, - dids_estimate, - .. - }, - children, - cursor, - ) = read.get_prefix( - NsidPrefix::new("a.a").unwrap(), - 10, - OrderCollectionsBy::Lexi { cursor: None }, - None, - None, - )?; - assert_eq!(creates, 2); - assert_eq!(dids_estimate, 1); - assert_eq!( - children, - vec![PrefixChild::Prefix(PrefixCount { - prefix: "a.a.a".to_string(), - creates: 2, - updates: 0, - deletes: 0, - dids_estimate: 1 - }),] - ); - assert_eq!(cursor, None); - Ok(()) - } - - #[test] - fn get_prefix_exact_and_child_and_prefix() -> anyhow::Result<()> { - let (read, mut write) = fjall_db(); - - let mut batch = TestBatch::default(); - // exact: - batch.create( - "did:plc:person-a", - "a.a.a", - "rkey-aaa", - "{}", - Some("rev-aaa"), - None, - 10_000, - ); - // child: - batch.create( - "did:plc:person-a", - "a.a.a.a", - "rkey-aaaa", - "{}", - Some("rev-aaaa"), - None, - 10_001, - ); - // prefix: - batch.create( - "did:plc:person-a", - "a.a.a.a.a", - "rkey-aaaaa", - "{}", - Some("rev-aaaaa"), - None, - 10_002, - ); - write.insert_batch(batch.batch)?; - write.step_rollup()?; - - let ( - JustCount { - creates, - dids_estimate, - .. - }, - children, - cursor, - ) = read.get_prefix( - NsidPrefix::new("a.a.a").unwrap(), - 10, - OrderCollectionsBy::Lexi { cursor: None }, - None, - None, - )?; - assert_eq!(creates, 2); - assert_eq!(dids_estimate, 1); - assert_eq!( - children, - vec![ - PrefixChild::Collection(NsidCount { - nsid: "a.a.a.a".to_string(), - creates: 1, - updates: 0, - deletes: 0, - dids_estimate: 1 - }), - PrefixChild::Prefix(PrefixCount { - prefix: "a.a.a.a".to_string(), - creates: 1, - updates: 0, - deletes: 0, - dids_estimate: 1 - }), - ] - ); - assert_eq!(cursor, None); - Ok(()) - } -} diff --git a/ufos/src/store_types.rs b/ufos/src/store_types.rs deleted file mode 100644 index 0e5a8e9..0000000 --- a/ufos/src/store_types.rs +++ /dev/null @@ -1,839 +0,0 @@ -use crate::db_types::{ - DbBytes, DbConcat, DbStaticStr, EncodingError, EncodingResult, SerdeBytes, StaticStr, - UseBincodePlz, -}; -use crate::{Cursor, Did, JustCount, Nsid, PutAction, RecordKey, UFOsCommit}; -use bincode::{Decode, Encode}; -use cardinality_estimator_safe::Sketch; -use std::ops::{Bound, Range}; - -macro_rules! static_str { - ($prefix:expr, $name:ident) => { - #[derive(Debug, PartialEq)] - pub struct $name {} - impl StaticStr for $name { - fn static_str() -> &'static str { - $prefix - } - } - }; -} - -// key format: ["js_cursor"] -static_str!("js_cursor", JetstreamCursorKey); -pub type JetstreamCursorValue = Cursor; - -// key format: ["sketch_secret"] -static_str!("sketch_secret", SketchSecretKey); -pub type SketchSecretPrefix = [u8; 16]; - -// key format: ["rollup_cursor"] -static_str!("rollup_cursor", NewRollupCursorKey); -// pub type NewRollupCursorKey = DbStaticStr<_NewRollupCursorKey>; -/// value format: [rollup_cursor(Cursor)|collection(Nsid)] -pub type NewRollupCursorValue = Cursor; - -static_str!("trim_cursor", _TrimCollectionStaticStr); -type TrimCollectionCursorPrefix = DbStaticStr<_TrimCollectionStaticStr>; -pub type TrimCollectionCursorKey = DbConcat; -impl TrimCollectionCursorKey { - pub fn new(collection: Nsid) -> Self { - Self::from_pair(Default::default(), collection) - } -} -pub type TrimCollectionCursorVal = Cursor; - -// key format: ["js_endpoint"] -static_str!("takeoff", TakeoffKey); -pub type TakeoffValue = Cursor; - -// key format: ["js_endpoint"] -static_str!("js_endpoint", JetstreamEndpointKey); -#[derive(Debug, PartialEq)] -pub struct JetstreamEndpointValue(pub String); -/// String wrapper for jetstream endpoint value -/// -/// Warning: this is a non-terminating byte representation of a string: it cannot be used in prefix position of DbConcat -impl DbBytes for JetstreamEndpointValue { - // TODO: maybe make a helper type in db_types - fn to_db_bytes(&self) -> Result, EncodingError> { - Ok(self.0.as_bytes().to_vec()) - } - fn from_db_bytes(bytes: &[u8]) -> Result<(Self, usize), EncodingError> { - let s = std::str::from_utf8(bytes)?.to_string(); - Ok((Self(s), bytes.len())) - } -} - -pub trait WithCollection { - fn collection(&self) -> &Nsid; -} - -pub trait WithRank { - fn rank(&self) -> u64; -} - -pub type NsidRecordFeedKey = DbConcat; -impl NsidRecordFeedKey { - pub fn collection(&self) -> &Nsid { - &self.prefix - } - pub fn cursor(&self) -> Cursor { - self.suffix - } -} -pub type NsidRecordFeedVal = DbConcat>; -impl NsidRecordFeedVal { - pub fn did(&self) -> &Did { - &self.prefix - } - pub fn rkey(&self) -> &RecordKey { - &self.suffix.prefix - } - pub fn rev(&self) -> &str { - &self.suffix.suffix - } -} -impl From<(&Did, &RecordKey, &str)> for NsidRecordFeedVal { - fn from((did, rkey, rev): (&Did, &RecordKey, &str)) -> Self { - Self::from_pair( - did.clone(), - DbConcat::from_pair(rkey.clone(), rev.to_string()), - ) - } -} - -pub type RecordLocationKey = DbConcat>; -impl RecordLocationKey { - pub fn did(&self) -> &Did { - &self.prefix - } - pub fn collection(&self) -> &Nsid { - &self.suffix.prefix - } - pub fn rkey(&self) -> &RecordKey { - &self.suffix.suffix - } -} -impl From<(&UFOsCommit, &Nsid)> for RecordLocationKey { - fn from((commit, collection): (&UFOsCommit, &Nsid)) -> Self { - Self::from_pair( - commit.did.clone(), - DbConcat::from_pair(collection.clone(), commit.rkey.clone()), - ) - } -} -impl From<(&NsidRecordFeedKey, &NsidRecordFeedVal)> for RecordLocationKey { - fn from((key, val): (&NsidRecordFeedKey, &NsidRecordFeedVal)) -> Self { - Self::from_pair( - val.did().clone(), - DbConcat::from_pair(key.collection().clone(), val.rkey().clone()), - ) - } -} - -#[derive(Debug, PartialEq, Encode, Decode)] -pub struct RecordLocationMeta { - cursor: u64, // ugh no bincode impl - pub is_update: bool, - pub rev: String, -} -impl RecordLocationMeta { - pub fn cursor(&self) -> Cursor { - Cursor::from_raw_u64(self.cursor) - } -} -impl UseBincodePlz for RecordLocationMeta {} - -#[derive(Debug, Clone, PartialEq)] -pub struct RecordRawValue(Vec); -impl DbBytes for RecordRawValue { - fn to_db_bytes(&self) -> Result, EncodingError> { - self.0.to_db_bytes() - } - fn from_db_bytes(bytes: &[u8]) -> Result<(Self, usize), EncodingError> { - let (v, n) = DbBytes::from_db_bytes(bytes)?; - Ok((Self(v), n)) - } -} -impl From> for RecordRawValue { - fn from(v: Box) -> Self { - Self(v.get().into()) - } -} -impl TryFrom for Box { - type Error = EncodingError; - fn try_from(rrv: RecordRawValue) -> Result { - let s = String::from_utf8(rrv.0)?; - let rv = serde_json::value::RawValue::from_string(s)?; - Ok(rv) - } -} - -pub type RecordLocationVal = DbConcat; -impl From<(Cursor, &str, PutAction)> for RecordLocationVal { - fn from((cursor, rev, put): (Cursor, &str, PutAction)) -> Self { - let meta = RecordLocationMeta { - cursor: cursor.to_raw_u64(), - is_update: put.is_update, - rev: rev.to_string(), - }; - Self::from_pair(meta, put.record.into()) - } -} - -static_str!("live_counts", _LiveRecordsStaticStr); - -type LiveCountsStaticPrefix = DbStaticStr<_LiveRecordsStaticStr>; -type LiveCountsCursorPrefix = DbConcat; -pub type LiveCountsKey = DbConcat; -impl LiveCountsKey { - pub fn range_from_cursor(cursor: Cursor) -> Result>, EncodingError> { - let prefix = LiveCountsCursorPrefix::from_pair(Default::default(), cursor); - prefix.range_to_prefix_end() - } - pub fn cursor(&self) -> Cursor { - self.prefix.suffix - } -} -impl WithCollection for LiveCountsKey { - fn collection(&self) -> &Nsid { - &self.suffix - } -} -impl From<(Cursor, &Nsid)> for LiveCountsKey { - fn from((cursor, collection): (Cursor, &Nsid)) -> Self { - Self::from_pair( - LiveCountsCursorPrefix::from_pair(Default::default(), cursor), - collection.clone(), - ) - } -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Decode, Encode)] -pub struct CommitCounts { - pub creates: u64, - pub updates: u64, - pub deletes: u64, -} -impl CommitCounts { - pub fn merge(&mut self, other: &Self) { - self.creates += other.creates; - self.updates += other.updates; - self.deletes += other.deletes; - } -} -impl UseBincodePlz for CommitCounts {} - -#[derive(Debug, Default, PartialEq, serde::Serialize, serde::Deserialize)] -pub struct EstimatedDidsValue(pub Sketch<14>); -impl SerdeBytes for EstimatedDidsValue {} -impl DbBytes for EstimatedDidsValue { - #[cfg(test)] - fn to_db_bytes(&self) -> Result, EncodingError> { - SerdeBytes::to_bytes(self) - } - #[cfg(test)] - fn from_db_bytes(bytes: &[u8]) -> Result<(Self, usize), EncodingError> { - SerdeBytes::from_bytes(bytes) - } - - #[cfg(not(test))] - fn to_db_bytes(&self) -> Result, EncodingError> { - SerdeBytes::to_bytes(self) - } - #[cfg(not(test))] - fn from_db_bytes(bytes: &[u8]) -> Result<(Self, usize), EncodingError> { - SerdeBytes::from_bytes(bytes) - } -} - -pub type CountsValue = DbConcat; -impl CountsValue { - pub fn new(counts: CommitCounts, dids: Sketch<14>) -> Self { - Self { - prefix: counts, - suffix: EstimatedDidsValue(dids), - } - } - pub fn counts(&self) -> CommitCounts { - self.prefix - } - pub fn dids(&self) -> &Sketch<14> { - &self.suffix.0 - } - pub fn merge(&mut self, other: &Self) { - self.prefix.merge(&other.prefix); - self.suffix.0.merge(&other.suffix.0); - } -} -impl From<&CountsValue> for JustCount { - fn from(cv: &CountsValue) -> Self { - let CommitCounts { - creates, - updates, - deletes, - } = cv.counts(); - Self { - creates, - updates, - deletes, - dids_estimate: cv.dids().estimate() as u64, - } - } -} - -static_str!("delete_acount", _DeleteAccountStaticStr); -pub type DeleteAccountStaticPrefix = DbStaticStr<_DeleteAccountStaticStr>; -pub type DeleteAccountQueueKey = DbConcat; -impl DeleteAccountQueueKey { - pub fn new(cursor: Cursor) -> Self { - Self::from_pair(Default::default(), cursor) - } -} -pub type DeleteAccountQueueVal = Did; - -/// big-endian encoded u64 for LSM prefix-fiendly key -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct KeyRank(u64); -impl DbBytes for KeyRank { - fn to_db_bytes(&self) -> Result, EncodingError> { - Ok(self.0.to_be_bytes().to_vec()) - } - fn from_db_bytes(bytes: &[u8]) -> Result<(Self, usize), EncodingError> { - if bytes.len() < 8 { - return Err(EncodingError::DecodeNotEnoughBytes); - } - let bytes8 = TryInto::<[u8; 8]>::try_into(&bytes[..8])?; - let rank = KeyRank(u64::from_be_bytes(bytes8)); - Ok((rank, 8)) - } -} -impl From for KeyRank { - fn from(n: u64) -> Self { - Self(n) - } -} -impl From for u64 { - fn from(kr: KeyRank) -> Self { - kr.0 - } -} - -pub type BucketedRankRecordsKey = - DbConcat, C>, DbConcat>; -impl BucketedRankRecordsKey -where - P: StaticStr + PartialEq + std::fmt::Debug, - C: DbBytes + PartialEq + std::fmt::Debug + Clone, -{ - pub fn new(cursor: C, rank: KeyRank, nsid: &Nsid) -> Self { - Self::from_pair( - DbConcat::from_pair(Default::default(), cursor), - DbConcat::from_pair(rank, nsid.clone()), - ) - } - pub fn with_rank(&self, new_rank: KeyRank) -> Self { - Self::new(self.prefix.suffix.clone(), new_rank, &self.suffix.suffix) - } - pub fn start(cursor: C) -> EncodingResult>> { - let prefix: DbConcat, C> = DbConcat::from_pair(Default::default(), cursor); - Ok(Bound::Included(Self::from_prefix_to_db_bytes(&prefix)?)) - } - pub fn end(cursor: C) -> EncodingResult>> { - let prefix: DbConcat, C> = DbConcat::from_pair(Default::default(), cursor); - Ok(Bound::Excluded(Self::prefix_range_end(&prefix)?)) - } -} -impl WithCollection for BucketedRankRecordsKey { - fn collection(&self) -> &Nsid { - &self.suffix.suffix - } -} -impl WithRank for BucketedRankRecordsKey { - fn rank(&self) -> u64 { - self.suffix.prefix.into() - } -} - -static_str!("hourly_counts", _HourlyRollupStaticStr); -pub type HourlyRollupStaticPrefix = DbStaticStr<_HourlyRollupStaticStr>; -pub type HourlyRollupKeyHourPrefix = DbConcat; -pub type HourlyRollupKey = DbConcat; -pub type HourlyRollupPre = DbConcat>; // bit hack but -impl HourlyRollupKey { - pub fn new(cursor: HourTruncatedCursor, nsid: &Nsid) -> Self { - Self::from_pair( - DbConcat::from_pair(Default::default(), cursor), - nsid.clone(), - ) - } - pub fn new_nsid_prefix(cursor: HourTruncatedCursor, pre: &[u8]) -> HourlyRollupPre { - HourlyRollupPre::from_pair( - DbConcat::from_pair(Default::default(), cursor), - pre.to_vec(), - ) - } - pub fn cursor(&self) -> HourTruncatedCursor { - self.prefix.suffix - } - pub fn start(hour: HourTruncatedCursor) -> EncodingResult>> { - let prefix = HourlyRollupKeyHourPrefix::from_pair(Default::default(), hour); - let prefix_bytes = Self::from_prefix_to_db_bytes(&prefix)?; - Ok(Bound::Included(prefix_bytes)) - } - pub fn after_nsid(hour: HourTruncatedCursor, nsid: &Nsid) -> EncodingResult>> { - Ok(Bound::Excluded(Self::new(hour, nsid).to_db_bytes()?)) - } - pub fn after_nsid_prefix( - hour: HourTruncatedCursor, - pre: &[u8], - ) -> EncodingResult>> { - Ok(Bound::Excluded( - Self::new_nsid_prefix(hour, pre).to_db_bytes()?, - )) - } - pub fn end(hour: HourTruncatedCursor) -> EncodingResult>> { - let prefix = HourlyRollupKeyHourPrefix::from_pair(Default::default(), hour); - Ok(Bound::Excluded(Self::prefix_range_end(&prefix)?)) - } - pub fn nsid_prefix_end( - hour: HourTruncatedCursor, - pre: &[u8], - ) -> EncodingResult>> { - Ok(Bound::Excluded( - Self::new_nsid_prefix(hour, pre).as_prefix_range_end()?, - )) - } -} -impl WithCollection for HourlyRollupKey { - fn collection(&self) -> &Nsid { - &self.suffix - } -} -pub type HourlyRollupVal = CountsValue; - -static_str!("hourly_rank_records", _HourlyRecordsStaticStr); -pub type HourlyRecordsKey = BucketedRankRecordsKey<_HourlyRecordsStaticStr, HourTruncatedCursor>; - -static_str!("hourly_rank_dids", _HourlyDidsStaticStr); -pub type HourlyDidsKey = BucketedRankRecordsKey<_HourlyDidsStaticStr, HourTruncatedCursor>; - -static_str!("weekly_counts", _WeeklyRollupStaticStr); -pub type WeeklyRollupStaticPrefix = DbStaticStr<_WeeklyRollupStaticStr>; -pub type WeeklyRollupKeyWeekPrefix = DbConcat; -pub type WeeklyRollupKey = DbConcat; -pub type WeeklyRollupPre = DbConcat>; -impl WeeklyRollupKey { - pub fn new(cursor: WeekTruncatedCursor, nsid: &Nsid) -> Self { - Self::from_pair( - DbConcat::from_pair(Default::default(), cursor), - nsid.clone(), - ) - } - pub fn new_nsid_prefix(cursor: WeekTruncatedCursor, pre: &[u8]) -> WeeklyRollupPre { - WeeklyRollupPre::from_pair( - DbConcat::from_pair(Default::default(), cursor), - pre.to_vec(), - ) - } - pub fn cursor(&self) -> WeekTruncatedCursor { - self.prefix.suffix - } - pub fn start(week: WeekTruncatedCursor) -> EncodingResult>> { - let prefix = WeeklyRollupKeyWeekPrefix::from_pair(Default::default(), week); - let prefix_bytes = Self::from_prefix_to_db_bytes(&prefix)?; - Ok(Bound::Included(prefix_bytes)) - } - pub fn after_nsid(week: WeekTruncatedCursor, nsid: &Nsid) -> EncodingResult>> { - Ok(Bound::Excluded(Self::new(week, nsid).to_db_bytes()?)) - } - pub fn after_nsid_prefix( - week: WeekTruncatedCursor, - prefix: &[u8], - ) -> EncodingResult>> { - Ok(Bound::Excluded( - Self::new_nsid_prefix(week, prefix).to_db_bytes()?, - )) - } - pub fn end(week: WeekTruncatedCursor) -> EncodingResult>> { - let prefix = WeeklyRollupKeyWeekPrefix::from_pair(Default::default(), week); - Ok(Bound::Excluded(Self::prefix_range_end(&prefix)?)) - } - pub fn nsid_prefix_end( - week: WeekTruncatedCursor, - prefix: &[u8], - ) -> EncodingResult>> { - Ok(Bound::Excluded( - Self::new_nsid_prefix(week, prefix).as_prefix_range_end()?, - )) - } -} -impl WithCollection for WeeklyRollupKey { - fn collection(&self) -> &Nsid { - &self.suffix - } -} -pub type WeeklyRollupVal = CountsValue; - -static_str!("weekly_rank_records", _WeeklyRecordsStaticStr); -pub type WeeklyRecordsKey = BucketedRankRecordsKey<_WeeklyRecordsStaticStr, WeekTruncatedCursor>; - -static_str!("weekly_rank_dids", _WeeklyDidsStaticStr); -pub type WeeklyDidsKey = BucketedRankRecordsKey<_WeeklyDidsStaticStr, WeekTruncatedCursor>; - -static_str!("ever_counts", _AllTimeRollupStaticStr); -pub type AllTimeRollupStaticPrefix = DbStaticStr<_AllTimeRollupStaticStr>; -pub type AllTimeRollupKey = DbConcat; -pub type AllTimeRollupPre = DbConcat>; -impl AllTimeRollupKey { - pub fn new(nsid: &Nsid) -> Self { - Self::from_pair(Default::default(), nsid.clone()) - } - pub fn new_nsid_prefix(pre: &[u8]) -> AllTimeRollupPre { - AllTimeRollupPre::from_pair(Default::default(), pre.to_vec()) - } - pub fn start() -> EncodingResult>> { - Ok(Bound::Included(Self::from_prefix_to_db_bytes( - &Default::default(), - )?)) - } - pub fn after_nsid(nsid: &Nsid) -> EncodingResult>> { - Ok(Bound::Excluded(Self::new(nsid).to_db_bytes()?)) - } - pub fn after_nsid_prefix(prefix: &[u8]) -> EncodingResult>> { - Ok(Bound::Excluded( - Self::new_nsid_prefix(prefix).to_db_bytes()?, - )) - } - pub fn end() -> EncodingResult>> { - Ok(Bound::Excluded( - Self::prefix_range_end(&Default::default())?, - )) - } - pub fn nsid_prefix_end(prefix: &[u8]) -> EncodingResult>> { - Ok(Bound::Excluded( - Self::new_nsid_prefix(prefix).as_prefix_range_end()?, - )) - } -} -impl WithCollection for AllTimeRollupKey { - fn collection(&self) -> &Nsid { - &self.suffix - } -} -pub type AllTimeRollupVal = CountsValue; - -pub type AllTimeRankRecordsKey

= DbConcat, DbConcat>; -impl

AllTimeRankRecordsKey

-where - P: StaticStr + PartialEq + std::fmt::Debug, -{ - pub fn new(rank: KeyRank, nsid: &Nsid) -> Self { - Self::from_pair(Default::default(), DbConcat::from_pair(rank, nsid.clone())) - } - pub fn with_rank(&self, new_rank: KeyRank) -> Self { - Self::new(new_rank, &self.suffix.suffix) - } - pub fn count(&self) -> u64 { - self.suffix.prefix.0 - } - pub fn start() -> EncodingResult>> { - Ok(Bound::Included(Self::from_prefix_to_db_bytes( - &Default::default(), - )?)) - } - pub fn end() -> EncodingResult>> { - Ok(Bound::Excluded( - Self::prefix_range_end(&Default::default())?, - )) - } -} -impl WithCollection for AllTimeRankRecordsKey

{ - fn collection(&self) -> &Nsid { - &self.suffix.suffix - } -} -impl WithRank for AllTimeRankRecordsKey

{ - fn rank(&self) -> u64 { - self.suffix.prefix.into() - } -} - -static_str!("ever_rank_records", _AllTimeRecordsStaticStr); -pub type AllTimeRecordsKey = AllTimeRankRecordsKey<_AllTimeRecordsStaticStr>; - -static_str!("ever_rank_dids", _AllTimeDidsStaticStr); -pub type AllTimeDidsKey = AllTimeRankRecordsKey<_AllTimeDidsStaticStr>; - -#[derive(Debug, Copy, Clone, PartialEq, Hash, PartialOrd, Eq)] -pub struct TruncatedCursor(u64); -impl TruncatedCursor { - pub fn truncate(raw: u64) -> u64 { - (raw / MOD) * MOD - } - pub fn try_from_raw_u64(time_us: u64) -> Result { - let rem = time_us % MOD; - if rem != 0 { - return Err(EncodingError::InvalidTruncated(MOD, rem)); - } - Ok(Self(time_us)) - } - pub fn try_from_cursor(cursor: Cursor) -> Result { - Self::try_from_raw_u64(cursor.to_raw_u64()) - } - pub fn truncate_raw_u64(raw: u64) -> Self { - let truncated = Self::truncate(raw); - Self(truncated) - } - pub fn truncate_cursor(cursor: Cursor) -> Self { - let raw = cursor.to_raw_u64(); - let truncated = Self::truncate(raw); - Self(truncated) - } - pub fn to_raw_u64(&self) -> u64 { - self.0 - } - pub fn try_as(&self) -> Result, EncodingError> { - TruncatedCursor::::try_from_raw_u64(self.0) - } - pub fn cycles_until(&self, other: Self) -> u64 { - if other < *self { - panic!("other must be greater than or equal to self"); - } - (other.0 - self.0) / MOD - } - pub fn next(&self) -> Self { - Self(self.0 + MOD) - } - pub fn nth_next(&self, n: u64) -> Self { - Self(self.0 + (n * MOD)) - } - pub fn prev(&self) -> Self { - if self.0 < MOD { - panic!("underflow: previous truncation start would be less than zero"); - } - Self(self.0 - MOD) - } -} -impl From> for Cursor { - fn from(truncated: TruncatedCursor) -> Self { - Cursor::from_raw_u64(truncated.0) - } -} -impl From for TruncatedCursor { - fn from(cursor: Cursor) -> Self { - Self::truncate_cursor(cursor) - } -} -impl DbBytes for TruncatedCursor { - fn to_db_bytes(&self) -> Result, EncodingError> { - let as_cursor: Cursor = (*self).into(); - as_cursor.to_db_bytes() - } - fn from_db_bytes(bytes: &[u8]) -> Result<(Self, usize), EncodingError> { - let (cursor, n) = Cursor::from_db_bytes(bytes)?; - let me = Self::try_from_cursor(cursor)?; - Ok((me, n)) - } -} - -pub const HOUR_IN_MICROS: u64 = 1_000_000 * 3600; -pub type HourTruncatedCursor = TruncatedCursor; - -pub const WEEK_IN_MICROS: u64 = HOUR_IN_MICROS * 24 * 7; -pub type WeekTruncatedCursor = TruncatedCursor; - -#[derive(Debug, PartialEq)] -pub enum CursorBucket { - Hour(HourTruncatedCursor), - Week(WeekTruncatedCursor), - AllTime, -} - -impl CursorBucket { - pub fn buckets_spanning( - since: HourTruncatedCursor, - until: HourTruncatedCursor, - ) -> Vec { - if until <= since { - return vec![]; - } - let mut out = vec![]; - let mut current_lower = since; - while current_lower < until { - if current_lower.cycles_until(until) >= (WEEK_IN_MICROS / HOUR_IN_MICROS) { - if let Ok(week) = current_lower.try_as::() { - out.push(CursorBucket::Week(week)); - current_lower = week.next().try_as().unwrap(); - continue; - } - } - out.push(CursorBucket::Hour(current_lower)); - current_lower = current_lower.next(); - } - out - } -} - -#[cfg(test)] -mod test { - use super::{ - CommitCounts, CountsValue, Cursor, CursorBucket, Did, EncodingError, HourTruncatedCursor, - HourlyRollupKey, Nsid, Sketch, HOUR_IN_MICROS, WEEK_IN_MICROS, - }; - use crate::db_types::DbBytes; - use cardinality_estimator_safe::Element; - use sha2::Sha256; - - #[test] - fn test_by_hourly_rollup_key() -> Result<(), EncodingError> { - let nsid = Nsid::new("ab.cd.efg".to_string()).unwrap(); - let original = HourlyRollupKey::new(Cursor::from_raw_u64(4567890).into(), &nsid); - let serialized = original.to_db_bytes()?; - let (restored, bytes_consumed) = HourlyRollupKey::from_db_bytes(&serialized)?; - assert_eq!(restored, original); - assert_eq!(bytes_consumed, serialized.len()); - - let serialized_prefix = original.to_prefix_db_bytes()?; - assert!(serialized_prefix.starts_with("hourly_counts".as_bytes())); - assert!(serialized_prefix.starts_with(&serialized_prefix)); - - Ok(()) - } - - #[test] - fn test_by_hourly_rollup_value() -> Result<(), EncodingError> { - let mut estimator = Sketch::<14>::default(); - fn to_element(d: Did) -> Element<14> { - Element::from_digest_oneshot::(d.to_string().as_bytes()) - } - for i in 0..10 { - estimator.insert(to_element( - Did::new(format!("did:plc:inze6wrmsm7pjl7yta3oig7{i}")).unwrap(), - )); - } - let original = CountsValue::new( - CommitCounts { - creates: 123, - ..Default::default() - }, - estimator.clone(), - ); - let serialized = original.to_db_bytes()?; - let (restored, bytes_consumed) = CountsValue::from_db_bytes(&serialized)?; - assert_eq!(restored, original); - assert_eq!(bytes_consumed, serialized.len()); - - for i in 10..1_000 { - estimator.insert(to_element( - Did::new(format!("did:plc:inze6wrmsm7pjl7yta3oig{i}")).unwrap(), - )); - } - let original = CountsValue::new( - CommitCounts { - creates: 123, - ..Default::default() - }, - estimator, - ); - let serialized = original.to_db_bytes()?; - let (restored, bytes_consumed) = CountsValue::from_db_bytes(&serialized)?; - assert_eq!(restored, original); - assert_eq!(bytes_consumed, serialized.len()); - - Ok(()) - } - - #[test] - fn test_hour_truncated_cursor() { - let us = Cursor::from_raw_u64(1_743_778_483_483_895); - let hr = HourTruncatedCursor::truncate_cursor(us); - let back: Cursor = hr.into(); - assert!(back < us); - let diff = us.to_raw_u64() - back.to_raw_u64(); - assert!(diff < HOUR_IN_MICROS); - } - - #[test] - fn test_hour_truncated_cursor_already_truncated() { - let us = Cursor::from_raw_u64(1_743_775_200_000_000); - let hr = HourTruncatedCursor::truncate_cursor(us); - let back: Cursor = hr.into(); - assert_eq!(back, us); - let diff = us.to_raw_u64() - back.to_raw_u64(); - assert_eq!(diff, 0); - } - - #[test] - fn test_spanning_nothing() { - let from = Cursor::from_raw_u64(1_743_775_200_000_000).into(); - let until = Cursor::from_raw_u64(1_743_775_200_000_000).into(); - assert!(CursorBucket::buckets_spanning(from, until).is_empty()); - let until = Cursor::from_raw_u64(0).into(); - assert!(CursorBucket::buckets_spanning(from, until).is_empty()); - } - - #[test] - fn test_spanning_low_hours() { - let from = HourTruncatedCursor::truncate_cursor(Cursor::from_start()); - let until = from.next(); - assert_eq!( - CursorBucket::buckets_spanning(from, until), - vec![CursorBucket::Hour(from)] - ); - let until2 = until.next(); - let until3 = until2.next(); - assert_eq!( - CursorBucket::buckets_spanning(from, until3), - vec![ - CursorBucket::Hour(from), - CursorBucket::Hour(until), - CursorBucket::Hour(until2), - ] - ); - } - - #[test] - fn test_spanning_week_aligned() { - let from = HourTruncatedCursor::truncate_cursor(Cursor::from_start()); - let until = HourTruncatedCursor::truncate_cursor(Cursor::from_raw_u64(WEEK_IN_MICROS)); - assert_eq!( - CursorBucket::buckets_spanning(from, until), - vec![CursorBucket::Week(from.try_as().unwrap()),] - ); - let next_hour = until.next(); - assert_eq!( - CursorBucket::buckets_spanning(from, next_hour), - vec![ - CursorBucket::Week(from.try_as().unwrap()), - CursorBucket::Hour(until), - ] - ); - } - - #[test] - fn test_spanning_week_unaligned() { - let from = HourTruncatedCursor::truncate_cursor(Cursor::from_raw_u64( - WEEK_IN_MICROS - HOUR_IN_MICROS, - )); - let until = HourTruncatedCursor::truncate_cursor(Cursor::from_raw_u64( - from.to_raw_u64() + WEEK_IN_MICROS, - )); - let span = CursorBucket::buckets_spanning(from, until); - assert_eq!(span.len(), 168); - for b in &span { - let CursorBucket::Hour(_) = b else { - panic!("found week bucket in a span that should only have hourlies"); - }; - } - let until2 = until.next(); - assert_eq!( - CursorBucket::buckets_spanning(from, until2), - vec![ - CursorBucket::Hour(from), - CursorBucket::Week(from.next().try_as().unwrap()), - ] - ); - } -} diff --git a/who-am-i/.gitignore b/who-am-i/.gitignore deleted file mode 100644 index 9dab1bc..0000000 --- a/who-am-i/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.pem -jwks.json diff --git a/who-am-i/Cargo.toml b/who-am-i/Cargo.toml deleted file mode 100644 index 29f097f..0000000 --- a/who-am-i/Cargo.toml +++ /dev/null @@ -1,37 +0,0 @@ -[package] -name = "who-am-i" -version = "0.1.0" -edition = "2024" - -[dependencies] -atrium-api = { version = "0.25.4", default-features = false } -atrium-common = "0.1.2" -atrium-identity = "0.1.5" -atrium-oauth = "0.1.3" -axum = "0.8.4" -axum-extra = { version = "0.10.1", features = ["cookie-signed", "typed-header"] } -axum-template = { version = "3.0.0", features = ["handlebars"] } -clap = { version = "4.5.40", features = ["derive", "env"] } -ctrlc = "3.4.7" -dashmap = "6.1.0" -elliptic-curve = "0.13.8" -handlebars = { version = "6.3.2", features = ["dir_source"] } -hickory-resolver = "0.25.2" -jose-jwk = "0.1.2" -jsonwebtoken = "9.3.1" -metrics = "0.24.2" -p256 = "0.13.2" -pkcs8 = "0.10.2" -rand = "0.9.1" -reqwest = { version = "0.12.22", features = ["native-tls-vendored"] } -serde = { version = "1.0.219", features = ["derive"] } -serde_json = "1.0.140" -thiserror = "2.0.12" -tokio = { version = "1.45.1", features = ["full", "macros"] } -tokio-util = "0.7.15" -url = "2.5.4" - -[dependencies.metrics-exporter-prometheus] -version = "0.17.2" -default-features = false -features = ["http-listener", "async-runtime"] diff --git a/who-am-i/demo/index.html b/who-am-i/demo/index.html deleted file mode 100644 index 801a226..0000000 --- a/who-am-i/demo/index.html +++ /dev/null @@ -1,37 +0,0 @@ - - - - - - - -

hey

-

- - - - - - diff --git a/who-am-i/demo/serve b/who-am-i/demo/serve deleted file mode 100755 index 77f2bc3..0000000 --- a/who-am-i/demo/serve +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -echo "note: you might need to access via http://127.0.0.1:8888 (not localhost) for the iframe to get its cookies" -python3 -m http.server 8888 diff --git a/who-am-i/readme.md b/who-am-i/readme.md deleted file mode 100644 index 812eec5..0000000 --- a/who-am-i/readme.md +++ /dev/null @@ -1,66 +0,0 @@ -# who am i - -a little auth service for microcosm demos - -**you probably SHOULD NOT USE THIS in any serious environment** - -for now the deployment is restricted to microcosm -- expanding it for wider use likely requires solving a number of challenges that oauth exists for. - - -## a little auth service - -- you drop an iframe and a short few lines of JS on your web page, and get a nice-ish atproto login prompt. -- if the user has ever authorized this service before (and within some expiration), they will be presented with an in-frame one-click option to proceed. -- otherwise they get bounced over to the normal atproto oauth flow (in a popup or new tab) -- you get a callback containing - - a verified DID and handle - - a JWT containing the same that can be verified by public key -- **no write permissions** or any atproto permissions at all, just a verified identity - -**you probably SHOULD NOT USE THIS in any serious environment** - - -### problems - -- clickjacking: if this were allowed on arbitrary domains, malicious sites could trick users into proving their atproto identity. -- all the other problems oauth exists to solve: it's a little tricky to hook around the oauth flow so there are probably some annoying attacks. -- auth in front of auth: it's just a bit awkward to run an auth service that acts as an intermediary for a more-real auth behind it, but that's worse, less secure, and doesn't conform to any standards. - -so, **you probably SHOULD NOT USE THIS in any serious environment** - - -## why - -sometimes you want to make a thing that people can use with an atproto identity, and you might not want to let them put in any else's identity. apps that operate on public data like skircle, cred.blue, and the microcosm spacedust notifications demo don't require any special permission to operate for any user, and that's sometimes fine, but sometimes creepy/stalker-y/etc. - -to avoid building a small torment nexus for a microcosm demo (while also not wanting to get deep into oauth or operate a demo-specific auth backend), i made this little service to just get a verified identity. - -note: **you probably SHOULD NOT USE THIS in any serious environment** - ---- - -since the requirements (read-only, just verifying identity) seem modest, i was hoping that a fairly simple implementation could be Good Enough, but in the time that i was willing to spend on it, the simple version without major obvious weaknesses i was hoping for didn't emerge. - -it's still nice to have an explicit opt-in on a per-demo basis for microcosm so it will be used for that. it's allow-listed for the microcosm domain however (so not deployed on any adversarial hosting pages), so it's simultaenously overkill and restrictive. - -i will get back to oauth eventually and hopefully roll out a microcosm service to make it easy for clients (and demos), but there are a few more things in the pipeline to get to first. - - -### todo - -provide a pubkey-signed JWT of the identity (just the DID as `sub` probably). (**you probably SHOULD NOT USE THIS in any serious environment**) - - -## building - -for raspi 1 model b: - -atrium-oauth uses reqwest with default tls config that requires openssl which `cross` doesn't have a good time getting the os deps for. - -fortunately, simply *enabling* a differnent tls feature for reqwest actually stops the default problematic one from causing problems, so we have a `reqwest` direct dependency with a feature enabled, even though it's never imported into actual code, - -it builds with - -```bash -cross build --release --target arm-unknown-linux-gnueabihf -``` diff --git a/who-am-i/src/expiring_task_map.rs b/who-am-i/src/expiring_task_map.rs deleted file mode 100644 index 7277b71..0000000 --- a/who-am-i/src/expiring_task_map.rs +++ /dev/null @@ -1,89 +0,0 @@ -use dashmap::DashMap; -use rand::{Rng, distr::Alphanumeric}; -use std::sync::Arc; -use std::time::Duration; -use tokio::task::{JoinHandle, spawn}; -use tokio::time::sleep; -use tokio_util::sync::{CancellationToken, DropGuard}; - -pub struct ExpiringTaskMap(TaskMap); - -/// need to manually implement clone because T is allowed to not be clone -impl Clone for ExpiringTaskMap { - fn clone(&self) -> Self { - Self(self.0.clone()) - } -} - -impl ExpiringTaskMap { - pub fn new(expiration: Duration) -> Self { - let map = TaskMap { - map: Arc::new(DashMap::new()), - expiration, - }; - Self(map) - } - - pub fn dispatch(&self, task: F, cancel: CancellationToken) -> String - where - F: Future + Send + 'static, - { - let TaskMap { - ref map, - expiration, - } = self.0; - let task_key: String = rand::rng() - .sample_iter(&Alphanumeric) - .take(24) - .map(char::from) - .collect(); - - // spawn a tokio task and put the join handle in the map for later retrieval - map.insert(task_key.clone(), (cancel.clone().drop_guard(), spawn(task))); - - // spawn a second task to clean up the map in case it doesn't get claimed - let k = task_key.clone(); - let map = map.clone(); - spawn(async move { - if cancel - .run_until_cancelled(sleep(expiration)) - .await - .is_some() - // the (sleep) task completed first - { - map.remove(&k); - cancel.cancel(); - metrics::counter!("whoami_task_map_completions", "result" => "expired") - .increment(1); - } - }); - - task_key - } - - pub fn take(&self, key: &str) -> Option> { - if let Some((_key, (_guard, handle))) = self.0.map.remove(key) { - // when the _guard drops, it cancels the token for us - metrics::counter!("whoami_task_map_completions", "result" => "retrieved").increment(1); - Some(handle) - } else { - metrics::counter!("whoami_task_map_gones").increment(1); - None - } - } -} - -struct TaskMap { - map: Arc)>>, - expiration: Duration, -} - -/// need to manually implement clone because T is allowed to not be clone -impl Clone for TaskMap { - fn clone(&self) -> Self { - Self { - map: self.map.clone(), - expiration: self.expiration, - } - } -} diff --git a/who-am-i/src/jwt.rs b/who-am-i/src/jwt.rs deleted file mode 100644 index 70ad46a..0000000 --- a/who-am-i/src/jwt.rs +++ /dev/null @@ -1,93 +0,0 @@ -use elliptic_curve::SecretKey; -use jose_jwk::{Class, Jwk, Key, Parameters}; -use jsonwebtoken::{Algorithm, EncodingKey, Header, encode, errors::Error as JWTError}; -use pkcs8::DecodePrivateKey; -use serde::Serialize; -use std::fs; -use std::io::Error as IOError; -use std::path::Path; -use std::string::FromUtf8Error; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use thiserror::Error; - -#[derive(Debug, Error)] -pub enum TokensSetupError { - #[error("failed to read private key")] - ReadPrivateKey(IOError), - #[error("failed to retrieve private key: {0}")] - PrivateKey(JWTError), - #[error("failed to read private key")] - ReadJwks(IOError), - #[error("failed to retrieve jwks: {0}")] - DecodeJwks(FromUtf8Error), -} - -#[derive(Debug, Error)] -pub enum TokenMintingError { - #[error("failed to mint: {0}")] - EncodingError(#[from] JWTError), -} - -pub struct Tokens { - encoding_key: EncodingKey, - jwk: Jwk, -} - -impl Tokens { - pub fn from_files(priv_f: impl AsRef) -> Result { - let private_key_data: Vec = - fs::read(priv_f).map_err(TokensSetupError::ReadPrivateKey)?; - let encoding_key = - EncodingKey::from_ec_pem(&private_key_data).map_err(TokensSetupError::PrivateKey)?; - - let jwk_key_string = String::from_utf8(private_key_data).unwrap(); - let mut jwk = SecretKey::::from_pkcs8_pem(&jwk_key_string) - .map(|secret_key| Jwk { - key: Key::from(&secret_key.into()), - prm: Parameters { - kid: Some("who-am-i-00".to_string()), - cls: Some(Class::Signing), - ..Default::default() - }, - }) - .expect("to get private key"); - - // CRITICAL: this is what turns the private jwk into a public one: the - // `d` parameter is the secret for an EC key; a pubkey just has no `d`. - // - // this feels baaaadd but hey we're just copying atrium - // https://github.com/atrium-rs/atrium/blob/b48810f84d83d037ee89b79b8566df9e0f2a6dae/atrium-oauth/src/keyset.rs#L41 - let Key::Ec(ref mut ec) = jwk.key else { - unimplemented!() - }; - ec.d = None; // CRITICAL - - Ok(Self { encoding_key, jwk }) - } - - pub fn mint(&self, t: impl ToString) -> Result { - let sub = t.to_string(); - - let dt_now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("unix epoch is in the past"); - let dt_exp = dt_now + Duration::from_secs(30 * 86_400); - let exp = dt_exp.as_secs(); - - let mut header = Header::new(Algorithm::ES256); - header.kid = Some("who-am-i-00".to_string()); - // todo: consider setting jku? - - Ok(encode(&header, &Claims { sub, exp }, &self.encoding_key)?) - } - - pub fn jwk(&self) -> Jwk { - self.jwk.clone() - } -} - -#[derive(Debug, Serialize)] -struct Claims { - sub: String, - exp: u64, -} diff --git a/who-am-i/src/lib.rs b/who-am-i/src/lib.rs deleted file mode 100644 index 975fb4b..0000000 --- a/who-am-i/src/lib.rs +++ /dev/null @@ -1,9 +0,0 @@ -mod expiring_task_map; -mod jwt; -mod oauth; -mod server; - -pub use expiring_task_map::ExpiringTaskMap; -pub use jwt::Tokens; -pub use oauth::{OAuth, OAuthCallbackParams, OAuthCompleteError, ResolveHandleError}; -pub use server::serve; diff --git a/who-am-i/src/main.rs b/who-am-i/src/main.rs deleted file mode 100644 index 33282d9..0000000 --- a/who-am-i/src/main.rs +++ /dev/null @@ -1,120 +0,0 @@ -use clap::{ArgAction, Parser}; -use metrics_exporter_prometheus::{BuildError as PromBuildError, PrometheusBuilder}; -use std::path::PathBuf; -use tokio_util::sync::CancellationToken; -use who_am_i::{Tokens, serve}; - -/// Aggregate links in the at-mosphere -#[derive(Parser, Debug, Clone)] -#[command(version, about, long_about = None)] -struct Args { - /// secret key from which the cookie-signing key is derived - /// - /// must have at least 512 bits (64 bytes) of randomness - /// - /// eg: `cat /dev/urandom | head -c 64 | base64` - #[arg(long, env)] - app_secret: String, - /// path to at-oauth private key (PEM pk8 format) - /// - /// generate with: - /// - /// openssl ecparam -genkey -noout -name prime256v1 \ - /// | openssl pkcs8 -topk8 -nocrypt -out .pem - #[arg(long, env)] - oauth_private_key: Option, - /// path to jwt private key (PEM pk8 format) - /// - /// generate with: - /// - /// openssl ecparam -genkey -noout -name prime256v1 \ - /// | openssl pkcs8 -topk8 -nocrypt -out .pem - #[arg(long)] - jwt_private_key: PathBuf, - /// this server's client-reachable base url, for oauth redirect + jwt check - /// - /// required unless running in localhost mode with --dev - #[arg(long, env)] - base_url: Option, - /// host:port to bind to on startup - #[arg(long, env, default_value = "127.0.0.1:9997")] - bind: String, - /// Enable dev mode - /// - /// enables automatic template reloading, uses localhost oauth config, etc - #[arg(long, action)] - dev: bool, - /// Hosts who are allowed to one-click auth - /// - /// Pass this argument multiple times to allow multiple hosts - #[arg(long = "allow_host", short = 'a', action = ArgAction::Append)] - allowed_hosts: Vec, -} - -#[tokio::main(flavor = "current_thread")] -async fn main() { - let shutdown = CancellationToken::new(); - - let ctrlc_shutdown = shutdown.clone(); - ctrlc::set_handler(move || ctrlc_shutdown.cancel()).expect("failed to set ctrl-c handler"); - - let args = Args::parse(); - - // let bind = args.bind.to_socket_addrs().expect("--bind must be ToSocketAddrs"); - - let base = args.base_url.unwrap_or_else(|| { - if args.dev { - format!("http://{}", args.bind) - } else { - panic!("not in --dev mode so --base-url is required") - } - }); - - if !args.dev && args.oauth_private_key.is_none() { - panic!("--at-oauth-key is required except in --dev"); - } else if args.dev && args.oauth_private_key.is_some() { - eprintln!("warn: --at-oauth-key is ignored in dev (localhost config)"); - } - - if args.allowed_hosts.is_empty() { - panic!("at least one --allowed-host host must be set"); - } - - println!("starting with allowed_hosts hosts:"); - for host in &args.allowed_hosts { - println!(" - {host}"); - } - - let tokens = Tokens::from_files(args.jwt_private_key).unwrap(); - - if let Err(e) = install_metrics_server() { - eprintln!("failed to install metrics server: {e:?}"); - }; - - serve( - shutdown, - args.app_secret, - args.oauth_private_key, - tokens, - base, - args.bind, - args.allowed_hosts, - args.dev, - ) - .await; -} - -fn install_metrics_server() -> Result<(), PromBuildError> { - println!("installing metrics server..."); - let host = [0, 0, 0, 0]; - let port = 8765; - PrometheusBuilder::new() - .set_enable_unit_suffix(false) - .with_http_listener((host, port)) - .install()?; - println!( - "metrics server installed! listening on http://{}.{}.{}.{}:{port}", - host[0], host[1], host[2], host[3] - ); - Ok(()) -} diff --git a/who-am-i/src/oauth.rs b/who-am-i/src/oauth.rs deleted file mode 100644 index bdd87e1..0000000 --- a/who-am-i/src/oauth.rs +++ /dev/null @@ -1,268 +0,0 @@ -use jose_jwk::Class; -use jose_jwk::Jwk; -use jose_jwk::Key; -use jose_jwk::Parameters; -use std::fs; -use std::path::PathBuf; -// use p256::SecretKey; -use atrium_api::{agent::SessionManager, types::string::Did}; -use atrium_common::resolver::Resolver; -use atrium_identity::{ - did::{CommonDidResolver, CommonDidResolverConfig, DEFAULT_PLC_DIRECTORY_URL}, - handle::{AtprotoHandleResolver, AtprotoHandleResolverConfig, DnsTxtResolver}, -}; -use atrium_oauth::{ - AtprotoClientMetadata, AtprotoLocalhostClientMetadata, AuthMethod, AuthorizeOptions, - CallbackParams, DefaultHttpClient, GrantType, KnownScope, OAuthClient, OAuthClientConfig, - OAuthClientMetadata, OAuthResolverConfig, Scope, - store::{session::MemorySessionStore, state::MemoryStateStore}, -}; -use elliptic_curve::SecretKey; -use hickory_resolver::{ResolveError, TokioResolver}; -use jose_jwk::JwkSet; -use pkcs8::DecodePrivateKey; -use serde::Deserialize; -use std::sync::Arc; -use thiserror::Error; - -const READONLY_SCOPE: [Scope; 1] = [Scope::Known(KnownScope::Atproto)]; - -#[derive(Debug, Deserialize)] -pub struct CallbackErrorParams { - error: String, - error_description: Option, - #[allow(dead_code)] - state: Option, // TODO: we _should_ use state to associate the auth request but how to do that with atrium is unclear - iss: Option, -} - -#[derive(Debug, Deserialize)] -#[serde(untagged)] -pub enum OAuthCallbackParams { - Granted(CallbackParams), - Failed(CallbackErrorParams), -} - -type Client = OAuthClient< - MemoryStateStore, - MemorySessionStore, - CommonDidResolver, - AtprotoHandleResolver, ->; - -#[derive(Clone)] -pub struct OAuth { - client: Arc, - did_resolver: Arc>, -} - -#[derive(Debug, Error)] -pub enum AuthSetupError { - #[error("failed to intiialize atrium client: {0}")] - AtriumClientError(atrium_oauth::Error), - #[error("failed to initialize hickory dns resolver: {0}")] - HickoryResolverError(ResolveError), -} - -#[derive(Debug, Error)] -pub enum OAuthCompleteError { - #[error("the user denied request: {description:?} (from {issuer:?})")] - Denied { - description: Option, - issuer: Option, - }, - #[error("the request failed: {error}: {description:?} (from {issuer:?})")] - Failed { - error: String, - description: Option, - issuer: Option, - }, - #[error("failed to complete oauth callback: {0}")] - CallbackFailed(atrium_oauth::Error), - #[error("the authorized session did not contain a DID")] - NoDid, -} - -#[derive(Debug, Error)] -pub enum ResolveHandleError { - #[error("failed to resolve: {0}")] - ResolutionFailed(#[from] atrium_identity::Error), - #[error("identity resolved but no handle found for user")] - NoHandle, - #[error("found handle {0:?} but it appears invalid: {1}")] - InvalidHandle(String, &'static str), -} - -impl OAuth { - pub fn new(oauth_private_key: Option, base: String) -> Result { - let http_client = Arc::new(DefaultHttpClient::default()); - let did_resolver = || { - CommonDidResolver::new(CommonDidResolverConfig { - plc_directory_url: DEFAULT_PLC_DIRECTORY_URL.to_string(), - http_client: http_client.clone(), - }) - }; - let dns_txt_resolver = - HickoryDnsTxtResolver::new().map_err(AuthSetupError::HickoryResolverError)?; - - let resolver = OAuthResolverConfig { - did_resolver: did_resolver(), - handle_resolver: AtprotoHandleResolver::new(AtprotoHandleResolverConfig { - dns_txt_resolver, - http_client: Arc::clone(&http_client), - }), - authorization_server_metadata: Default::default(), - protected_resource_metadata: Default::default(), - }; - - let state_store = MemoryStateStore::default(); - let session_store = MemorySessionStore::default(); - - let client = if let Some(path) = oauth_private_key { - let key_contents: Vec = fs::read(path).unwrap(); - let key_string = String::from_utf8(key_contents).unwrap(); - let key = SecretKey::::from_pkcs8_pem(&key_string) - .map(|secret_key| Jwk { - key: Key::from(&secret_key.into()), - prm: Parameters { - kid: Some("at-oauth-00".to_string()), - cls: Some(Class::Signing), - ..Default::default() - }, - }) - .expect("to get private key"); - OAuthClient::new(OAuthClientConfig { - client_metadata: AtprotoClientMetadata { - client_id: format!("{base}/client-metadata.json"), - client_uri: Some(base.clone()), - redirect_uris: vec![format!("{base}/authorized")], - token_endpoint_auth_method: AuthMethod::PrivateKeyJwt, - grant_types: vec![GrantType::AuthorizationCode, GrantType::RefreshToken], - scopes: READONLY_SCOPE.to_vec(), - jwks_uri: Some(format!("{base}/.well-known/jwks.json")), - token_endpoint_auth_signing_alg: Some(String::from("ES256")), - }, - keys: Some(vec![key]), - resolver, - state_store, - session_store, - }) - .map_err(AuthSetupError::AtriumClientError)? - } else { - OAuthClient::new(OAuthClientConfig { - client_metadata: AtprotoLocalhostClientMetadata { - redirect_uris: Some(vec![String::from("http://127.0.0.1:9997/authorized")]), - scopes: Some(READONLY_SCOPE.to_vec()), - }, - keys: None, - resolver, - state_store, - session_store, - }) - .map_err(AuthSetupError::AtriumClientError)? - }; - - Ok(Self { - client: Arc::new(client), - did_resolver: Arc::new(did_resolver()), - }) - } - - pub fn client_metadata(&self) -> OAuthClientMetadata { - self.client.client_metadata.clone() - } - - pub fn jwks(&self) -> JwkSet { - self.client.jwks() - } - - pub async fn begin(&self, handle: &str) -> Result { - let auth_opts = AuthorizeOptions { - scopes: READONLY_SCOPE.to_vec(), - ..Default::default() - }; - self.client.authorize(handle, auth_opts).await - } - - /// Finally, resolve the oauth flow to a verified DID - pub async fn complete(&self, params: OAuthCallbackParams) -> Result { - let params = match params { - OAuthCallbackParams::Granted(params) => params, - OAuthCallbackParams::Failed(p) if p.error == "access_denied" => { - return Err(OAuthCompleteError::Denied { - description: p.error_description.clone(), - issuer: p.iss.clone(), - }); - } - OAuthCallbackParams::Failed(p) => { - return Err(OAuthCompleteError::Failed { - error: p.error.clone(), - description: p.error_description.clone(), - issuer: p.iss.clone(), - }); - } - }; - let (session, _) = self - .client - .callback(params) - .await - .map_err(OAuthCompleteError::CallbackFailed)?; - let Some(did) = session.did().await else { - return Err(OAuthCompleteError::NoDid); - }; - Ok(did) - } - - pub async fn resolve_handle(&self, did: Did) -> Result { - // TODO: this is only half the resolution? or is atrium checking dns? - let doc = self.did_resolver.resolve(&did).await?; - let Some(aka) = doc.also_known_as else { - return Err(ResolveHandleError::NoHandle); - }; - let Some(at_uri_handle) = aka.first() else { - return Err(ResolveHandleError::NoHandle); - }; - if aka.len() > 1 { - eprintln!("more than one handle found for {did:?}"); - } - let Some(bare_handle) = at_uri_handle.strip_prefix("at://") else { - return Err(ResolveHandleError::InvalidHandle( - at_uri_handle.to_string(), - "did not start with 'at://'", - )); - }; - if bare_handle.is_empty() { - return Err(ResolveHandleError::InvalidHandle( - at_uri_handle.to_string(), - "empty handle", - )); - } - Ok(bare_handle.to_string()) - } -} - -pub struct HickoryDnsTxtResolver(TokioResolver); - -impl HickoryDnsTxtResolver { - fn new() -> Result { - Ok(Self(TokioResolver::builder_tokio()?.build())) - } -} - -impl DnsTxtResolver for HickoryDnsTxtResolver { - async fn resolve( - &self, - query: &str, - ) -> core::result::Result, Box> { - match self.0.txt_lookup(query).await { - Ok(r) => { - metrics::counter!("whoami_resolve_dns_txt", "success" => "true").increment(1); - Ok(r.iter().map(|r| r.to_string()).collect()) - } - Err(e) => { - metrics::counter!("whoami_resolve_dns_txt", "success" => "false").increment(1); - Err(e.into()) - } - } - } -} diff --git a/who-am-i/src/server.rs b/who-am-i/src/server.rs deleted file mode 100644 index f6a2008..0000000 --- a/who-am-i/src/server.rs +++ /dev/null @@ -1,525 +0,0 @@ -use atrium_api::types::string::Did; -use atrium_oauth::OAuthClientMetadata; -use axum::{ - Router, - extract::{FromRef, Json as ExtractJson, Query, State}, - http::{ - StatusCode, - header::{CONTENT_SECURITY_POLICY, CONTENT_TYPE, HeaderMap, ORIGIN, REFERER}, - }, - response::{IntoResponse, Json, Redirect, Response}, - routing::{get, post}, -}; -use axum_extra::extract::cookie::{Cookie, Expiration, Key, SameSite, SignedCookieJar}; -use axum_template::{RenderHtml, engine::Engine}; -use handlebars::{Handlebars, handlebars_helper}; -use jose_jwk::JwkSet; -use std::path::PathBuf; - -use serde::Deserialize; -use serde_json::{Value, json}; -use std::collections::HashSet; -use std::sync::Arc; -use std::time::{Duration, SystemTime}; -use tokio::net::TcpListener; -use tokio_util::sync::CancellationToken; -use url::Url; - -use crate::{ - ExpiringTaskMap, OAuth, OAuthCallbackParams, OAuthCompleteError, ResolveHandleError, Tokens, -}; - -const FAVICON: &[u8] = include_bytes!("../static/favicon.ico"); -const STYLE_CSS: &str = include_str!("../static/style.css"); - -const HELLO_COOKIE_KEY: &str = "hello-who-am-i"; -const DID_COOKIE_KEY: &str = "did"; - -const COOKIE_EXPIRATION: Duration = Duration::from_secs(30 * 86_400); - -type AppEngine = Engine>; - -#[derive(Clone)] -struct AppState { - pub key: Key, - pub allowed_hosts: Arc>, - pub engine: AppEngine, - pub oauth: Arc, - pub resolve_handles: ExpiringTaskMap>, - pub shutdown: CancellationToken, - pub tokens: Arc, -} - -impl FromRef for Key { - fn from_ref(state: &AppState) -> Self { - state.key.clone() - } -} - -#[allow(clippy::too_many_arguments)] -pub async fn serve( - shutdown: CancellationToken, - app_secret: String, - oauth_private_key: Option, - tokens: Tokens, - base: String, - bind: String, - allowed_hosts: Vec, - dev: bool, -) { - let mut hbs = Handlebars::new(); - hbs.set_dev_mode(dev); - hbs.register_templates_directory("templates", Default::default()) - .unwrap(); - - handlebars_helper!(json: |v: Value| serde_json::to_string(&v).unwrap()); - hbs.register_helper("json", Box::new(json)); - - // clients have to pick up their identity-resolving tasks within this period - let task_pickup_expiration = Duration::from_secs(15); - - let oauth = OAuth::new(oauth_private_key, base).unwrap(); - - let state = AppState { - engine: Engine::new(hbs), - key: Key::from(app_secret.as_bytes()), // TODO: via config - allowed_hosts: Arc::new(HashSet::from_iter(allowed_hosts)), - oauth: Arc::new(oauth), - resolve_handles: ExpiringTaskMap::new(task_pickup_expiration), - shutdown: shutdown.clone(), - tokens: Arc::new(tokens), - }; - - let app = Router::new() - .route("/", get(hello)) - .route("/favicon.ico", get(favicon)) // todo MIME - .route("/style.css", get(css)) - .route("/prompt", get(prompt)) - .route("/user-info", post(user_info)) - .route("/client-metadata.json", get(client_metadata)) - .route("/auth", get(start_oauth)) - .route("/authorized", get(complete_oauth)) - .route("/disconnect", post(disconnect)) - .route("/.well-known/jwks.json", get(jwks)) - .with_state(state); - - eprintln!("starting server at http://{bind}"); - let listener = TcpListener::bind(bind) - .await - .expect("listener binding to work"); - - axum::serve(listener, app) - .with_graceful_shutdown(async move { shutdown.cancelled().await }) - .await - .unwrap(); -} - -#[derive(Debug, Deserialize)] -struct HelloQuery { - auth_reload: Option, - auth_failed: Option, -} -async fn hello( - State(AppState { - engine, - resolve_handles, - shutdown, - oauth, - .. - }): State, - Query(params): Query, - mut jar: SignedCookieJar, -) -> Response { - let is_auth_reload = params.auth_reload.is_some(); - let auth_failed = params.auth_failed.is_some(); - let no_cookie = jar.get(HELLO_COOKIE_KEY).is_none(); - jar = jar.add(hello_cookie()); - - let info = if let Some(did) = jar.get(DID_COOKIE_KEY) { - if let Ok(did) = Did::new(did.value_trimmed().to_string()) { - // push cookie expiry - jar = jar.add(cookie(&did)); - let fetch_key = resolve_handles.dispatch( - { - let oauth = oauth.clone(); - let did = did.clone(); - async move { oauth.resolve_handle(did.clone()).await } - }, - shutdown.child_token(), - ); - json!({ - "did": did, - "fetch_key": fetch_key, - "is_auth_reload": is_auth_reload, - "auth_failed": auth_failed, - "no_cookie": no_cookie, - }) - } else { - jar = jar.remove(DID_COOKIE_KEY); - json!({ - "is_auth_reload": is_auth_reload, - "auth_failed": auth_failed, - "no_cookie": no_cookie, - }) - } - } else { - json!({ - "is_auth_reload": is_auth_reload, - "auth_failed": auth_failed, - "no_cookie": no_cookie, - }) - }; - let frame_headers = [(CONTENT_SECURITY_POLICY, "frame-ancestors 'none'")]; - (frame_headers, jar, RenderHtml("hello", engine, info)).into_response() -} - -async fn css() -> impl IntoResponse { - let headers = [ - (CONTENT_TYPE, "text/css"), - // (CACHE_CONTROL, "") // TODO - ]; - (headers, STYLE_CSS) -} - -async fn favicon() -> impl IntoResponse { - ([(CONTENT_TYPE, "image/x-icon")], FAVICON) -} - -fn hello_cookie() -> Cookie<'static> { - Cookie::build((HELLO_COOKIE_KEY, "hiiii")) - .http_only(true) - .secure(true) - .same_site(SameSite::None) - .expires(Expiration::DateTime( - (SystemTime::now() + COOKIE_EXPIRATION).into(), - )) // wtf safari needs this to not be a session cookie?? - .max_age(COOKIE_EXPIRATION.try_into().unwrap()) - .path("/") - .into() -} - -fn cookie(did: &Did) -> Cookie<'static> { - Cookie::build((DID_COOKIE_KEY, did.to_string())) - .http_only(true) - .secure(true) - .same_site(SameSite::None) - .expires(Expiration::DateTime( - (SystemTime::now() + COOKIE_EXPIRATION).into(), - )) // wtf safari needs this to not be a session cookie?? - .max_age(COOKIE_EXPIRATION.try_into().unwrap()) - .path("/") - .into() -} - -#[derive(Debug, Deserialize)] -struct PromptQuery { - // this must *ONLY* be used for the postmessage target origin - app: Option, -} -async fn prompt( - State(AppState { - allowed_hosts, - engine, - oauth, - resolve_handles, - shutdown, - tokens, - .. - }): State, - Query(params): Query, - jar: SignedCookieJar, - headers: HeaderMap, -) -> impl IntoResponse { - let err = |reason, check_frame, detail| { - metrics::counter!("whoami_auth_prompt", "ok" => "false", "reason" => reason).increment(1); - let info = json!({ - "reason": reason, - "check_frame": check_frame, - "detail": detail, - }); - let html = RenderHtml("prompt-error", engine.clone(), info); - (StatusCode::BAD_REQUEST, html).into_response() - }; - - let Some(parent) = headers.get(ORIGIN).or_else(|| { - eprintln!("referrer fallback"); - // TODO: referer should only be used for localhost?? - headers.get(REFERER) - }) else { - return err("Missing origin and no referrer for fallback", true, None); - }; - let Ok(parent) = parent.to_str() else { - return err("Unreadable origin or referrer", true, None); - }; - eprintln!( - "rolling with parent: {parent:?} (from origin? {})", - headers.get(ORIGIN).is_some() - ); - let Ok(url) = Url::parse(parent) else { - return err("Bad origin or referrer", true, None); - }; - let Some(parent_host) = url.host_str() else { - return err("Origin or referrer missing host", true, None); - }; - if !allowed_hosts.contains(parent_host) { - return err( - "Login is not allowed on this page", - false, - Some(parent_host), - ); - } - if let Some(ref app) = params.app { - if !allowed_hosts.contains(app) { - return err("Login is not allowed for this app", false, Some(app)); - } - } - let parent_origin = url.origin().ascii_serialization(); - if parent_origin == "null" { - return err("Origin or referrer header value is opaque", true, None); - } - - let all_allowed = allowed_hosts - .iter() - .map(|h| format!("https://{h}")) - .collect::>() - .join(" "); - let csp = format!("frame-ancestors 'self' {parent_origin} {all_allowed}"); - let frame_headers = [(CONTENT_SECURITY_POLICY, &csp)]; - - if let Some(did) = jar.get(DID_COOKIE_KEY) { - let Ok(did) = Did::new(did.value_trimmed().to_string()) else { - return err("Bad cookie", false, None); - }; - - // push cookie expiry - let jar = jar.add(cookie(&did)); - - let token = match tokens.mint(&*did) { - Ok(t) => t, - Err(e) => { - eprintln!("failed to create JWT: {e:?}"); - return err("failed to create JWT", false, None); - } - }; - - let fetch_key = resolve_handles.dispatch( - { - let oauth = oauth.clone(); - let did = did.clone(); - async move { oauth.resolve_handle(did.clone()).await } - }, - shutdown.child_token(), - ); - - metrics::counter!("whoami_auth_prompt", "ok" => "true", "known" => "true").increment(1); - let info = json!({ - "did": did, - "token": token, - "fetch_key": fetch_key, - "parent_host": parent_host, - "parent_origin": parent_origin, - "parent_target": params.app.map(|h| format!("https://{h}")), - }); - (frame_headers, jar, RenderHtml("prompt", engine, info)).into_response() - } else { - metrics::counter!("whoami_auth_prompt", "ok" => "true", "known" => "false").increment(1); - let info = json!({ - "parent_host": parent_host, - "parent_origin": parent_origin, - }); - (frame_headers, RenderHtml("prompt", engine, info)).into_response() - } -} - -#[derive(Debug, Deserialize)] -struct UserInfoParams { - fetch_key: String, -} -async fn user_info( - State(AppState { - resolve_handles, .. - }): State, - ExtractJson(params): ExtractJson, -) -> impl IntoResponse { - let err = |status, reason: &str| { - metrics::counter!("whoami_user_info", "found" => "false", "reason" => reason.to_string()) - .increment(1); - (status, Json(json!({ "reason": reason }))).into_response() - }; - - let Some(task_handle) = resolve_handles.take(¶ms.fetch_key) else { - return err(StatusCode::NOT_FOUND, "fetch key does not exist or expired"); - }; - - match task_handle.await { - Err(task_err) => { - eprintln!("task join error? {task_err:?}"); - err(StatusCode::INTERNAL_SERVER_ERROR, "server errored") - } - Ok(Err(ResolveHandleError::ResolutionFailed(atrium_identity::Error::NotFound))) => { - err(StatusCode::NOT_FOUND, "handle not found") - } - Ok(Err(ResolveHandleError::ResolutionFailed(e))) => { - eprintln!("handle resolution failed: {e:?}"); - err( - StatusCode::INTERNAL_SERVER_ERROR, - "handle resolution failed", - ) - } - Ok(Err(ResolveHandleError::NoHandle)) => err( - StatusCode::INTERNAL_SERVER_ERROR, - "resolved identity but did not find a handle", - ), - Ok(Err(ResolveHandleError::InvalidHandle(_h, reason))) => err( - StatusCode::INTERNAL_SERVER_ERROR, - &format!("handle appears invalid: {reason}"), - ), - Ok(Ok(handle)) => { - metrics::counter!("whoami_user_info", "found" => "true").increment(1); - Json(json!({ "handle": handle })).into_response() - } - } -} - -async fn client_metadata( - State(AppState { oauth, .. }): State, -) -> Json { - Json(oauth.client_metadata()) -} - -#[derive(Debug, Deserialize)] -struct BeginOauthParams { - handle: String, -} -async fn start_oauth( - State(AppState { oauth, engine, .. }): State, - Query(params): Query, - jar: SignedCookieJar, -) -> Response { - // if any existing session was active, clear it first - // ...this might help a confusion attack w multiple sign-in flows or smth - let jar = jar.remove(DID_COOKIE_KEY); - - use atrium_identity::Error as IdError; - use atrium_oauth::Error as OAuthError; - - let err = |code, reason: &str| { - metrics::counter!("whoami_auth_start", "ok" => "false", "reason" => reason.to_string()) - .increment(1); - let info = json!({ - "result": "fail", - "reason": reason, - }); - (code, RenderHtml("auth-fail", engine.clone(), info)).into_response() - }; - - match oauth.begin(¶ms.handle).await { - Err(OAuthError::Identity( - IdError::NotFound | IdError::HttpStatus(StatusCode::NOT_FOUND), - )) => err(StatusCode::NOT_FOUND, "handle not found"), - Err(OAuthError::Identity(IdError::AtIdentifier(r))) => err(StatusCode::BAD_REQUEST, &r), - Err(e) => { - eprintln!("begin auth failed: {e:?}"); - err(StatusCode::INTERNAL_SERVER_ERROR, "unknown") - } - Ok(auth_url) => { - metrics::counter!("whoami_auth_start", "ok" => "true").increment(1); - (jar, Redirect::to(&auth_url)).into_response() - } - } -} - -async fn complete_oauth( - State(AppState { - engine, - resolve_handles, - oauth, - shutdown, - tokens, - .. - }): State, - Query(params): Query, - jar: SignedCookieJar, -) -> Response { - let err = |code, result, reason: &str| { - metrics::counter!("whoami_auth_complete", "ok" => "false", "reason" => reason.to_string()) - .increment(1); - let info = json!({ - "result": result, - "reason": reason, - }); - (code, RenderHtml("auth-fail", engine.clone(), info)).into_response() - }; - - let did = match oauth.complete(params).await { - Ok(did) => did, - Err(e) => { - return match e { - OAuthCompleteError::Denied { description, .. } => { - let desc = description.unwrap_or("permission to share was denied".to_string()); - err(StatusCode::FORBIDDEN, "deny", desc.as_str()) - } - OAuthCompleteError::Failed { .. } => { - eprintln!("auth completion failed: {e:?}"); - err( - StatusCode::INTERNAL_SERVER_ERROR, - "fail", - "failed to complete", - ) - } - OAuthCompleteError::CallbackFailed(e) => { - eprintln!("auth callback failed: {e:?}"); - err( - StatusCode::INTERNAL_SERVER_ERROR, - "fail", - "failed to complete callback", - ) - } - OAuthCompleteError::NoDid => err(StatusCode::BAD_REQUEST, "fail", "no DID found"), - }; - } - }; - - let jar = jar.add(cookie(&did)); - - let token = match tokens.mint(&*did) { - Ok(t) => t, - Err(e) => { - eprintln!("failed to create JWT: {e:?}"); - return err( - StatusCode::INTERNAL_SERVER_ERROR, - "fail", - "failed to create JWT", - ); - } - }; - - let fetch_key = resolve_handles.dispatch( - { - let oauth = oauth.clone(); - let did = did.clone(); - async move { oauth.resolve_handle(did.clone()).await } - }, - shutdown.child_token(), - ); - - metrics::counter!("whoami_auth_complete", "ok" => "true").increment(1); - let info = json!({ - "did": did, - "token": token, - "fetch_key": fetch_key, - }); - (jar, RenderHtml("authorized", engine, info)).into_response() -} - -async fn disconnect(jar: SignedCookieJar) -> impl IntoResponse { - metrics::counter!("whoami_disconnect").increment(1); - let jar = jar.remove(DID_COOKIE_KEY); - (jar, Json(json!({ "ok": true }))) -} - -async fn jwks(State(AppState { oauth, tokens, .. }): State) -> Json { - let mut jwks = oauth.jwks(); - jwks.keys.push(tokens.jwk()); - Json(jwks) -} diff --git a/who-am-i/static/favicon.ico b/who-am-i/static/favicon.ico deleted file mode 100644 index f4fb8642604f730454cf4a51425c542d3c6af4b4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 15406 zcmZQzU}Rus5D);-3Je)63=C!r3=9ei5Wa>W1H(KP1_lEI2tPxOf#H}a1A_(w1A_oa z9Roz1fkeRR-7C!#wZ@Joa+N)&$1E8lwKH<3Y>(tFIdhJ?>DFKF&WHcGIv@V$ZoK)A ztN8RquB2VDVCP^n%%Yc*JNMWc?)Jz3cxHV1&%f~df1#znz*u16kN+SsuJ*_OxbseI zWN_`~Ml*tuEAQlL?*2Fbg%U#4yCf3Bd8|1yn-|I0QX{m&iL z`CqDGDMW6~=l>x8a8_Qu1lI36L7cPe>0gQN6aRJV=l?gHviZMZ`~3fk9ozqhp8onj z;_SEoik&AM|Rvi6ya z6|MiTvH0A7_0vE9OHMxYpFMWQf1^z&|0}ny|F7J+^}q4DBVaMfNr%8{)K^^jFP^{Z zKXdke2Zq3DVv22-{uoc$`rq>M@BiAXul^TIpYh*f?&|;UTlfEW-*)i7#jNH31v6*; z*IIS?zvY$R{|#pD`mfY__8&vSdRb5yi1oZaVH}nB-+uOn|DH#0{db(V>c7X78UK^M z{{v&M8FT+T%vt{5^T^Hrc5^oVHwaDnFW&e1B1kU?bCzEYW|KAiCnzlZUrI*izo3Z7 zf7_yx{~f>o{qOw!|G!IJ!+!xmL9m>#sOWz-S-pQ8<(FbWdSO_!>dZ2$(*FN0sTu!M zCQtdl^!=y*oB#X=^e{?D8>=f8Je!GGKGzW>4v7uF%CAFvs@+t?KszF(eq|L?#3 zzyJTg@b~}!YfubgAO8LSf5pSU|CAShT+5KQi3_X;3n4o9S%}e*FDFax{r)@e^}qk~ zU;q1Geecg-`@`SPNX~r`iD54mGeAP(-511kmOio6Ui8FDzU!m}NQ@R3RL&?cFfcfP z9R_02!WO!l$;fF|Cc_grH;^}GZ3%bGs#fmk)onacYsz>67ld$FHp_#(OKoTA=L>L0 zuPWe4+kJ+o_|z|+>dXIm>aYLjX}kf(+#ohctoZbAp7h-pLGEENt{0))8TKsP5i4?e za*n*v7U-aWY|H2<& zHb{(n))%lEkXn!)p31BLcvAKpWq0oruI?BAL1yq|A9=*>Ia!yO zaNx3Sk>t)ga*Mn1{(r9ixBq!(eS_3p%m4frU-|35#LDmgC0Bg@FS+95f2rl~!B}$T z=l_x`zk=mJYDAa+0qX(j<(lyBKX=oE|2)~p9uW=)dlsIQy}P&@?)?Yp=N&k7gvi@^g2L9(X^ZC!A=lGvP*Zx1ZedK>`r`Z48u1Wv-qMQGV z&$#nnVil-P{QaLF6c!WT{|EVpJAMB#1{Du>T;aeOxi+1<>gGSLzPJDR=l=LFw*1e3 zskQ(9%dGqJpT{%(Ka-%ue|4h6>|Cx9N|1)v%{%7Lm|IfrH{GXAVA6zFh@d$$5 zC$sh^Sg+XfzyJB?{Q!q0cg^j8T#@T4u-b2y%g3E}>KRwZb8wi8uK4p`dfos3a-0AE zSJ?XRzew)X|BS5c2+YFvpOKyKKO+bKe+Cw|{|wA5{~4HAz_dU_<$r~(f5Cb|W`NA% zoBjPiSLciWTm@&|GT2m#qPu}Jc3lQn{k?x&lRy0zUi$mL)cSw_wS-7oRK(DXb1x#oQS&(ZquKS%WHFu46`+cP-2p8n^Z z`{BP_TK*02AzTVMQFj<5Z1J7Lj({i{F!8{PQvU+4T!uwIZEDhGf4H|g8>Um>F8KkwZ4 z|Jl2r{pU#A0Z#*L={qag`(FJQnt1)cQE=ja#XaA^ZAP_Izy52S{rO*M#kv1n$us{e zFFFBshgp2)f6dt!{(GPK_TTs1_y5`pul+ZPNcwMd_1k}y<){C1CC~b=xa8D-&9guM zgUkT6KNa_V|8E?b@n5Lt!hiPu*ZOi=#A*N8lNbKyjGgk|@W%K5rag21E0)jyue4|57t9 z{Fk0}`M*+N)_;?png5M0efZB7*acUsed^%6|Lavv|8IEW$A9AszyF(F`SaiO($D|g5k3D|gL?jR2DSaS zy!GY3aaGHIkDkf@9SSS|+vivO_n0vKzj0v|SdJ^S1FV)Sv=gisWCqAA{ZkPC+h(@> z=kIy_pEZ3KJpD1)&btUki`F%d!GKcyZrmV!;QcHo$mboue0*ff1c2e z|5{5={%T>K^ndT?U;g`i{0GL~ufG4cii-VjbL7f@?UiT$^Mrxap8M~3=kI@q zn}7e?UHSdreE;+Rh7oE1jb`8e&)xadpa;1;79MHO%>Mlv(xPY2V%d z-q*kV_q+f5zyG7(|9$R!`tNq~`hSy@jQ=4Uw*8O%`tN_#xBp-aVu$WL@ZTgp>A&l- zEC0RkeEjeC=r>p|$PA|g_x~H_H~#l1@B6R4^v8dW+J~S%0;2q3U|?X)IhHIj>Dxcm zh>HK3TH623f}{RhwoLi&vTFB#*VQ}!TebH7w+IaWZx#^vKjqD*|JlF(|IdNqtY82C zr+xVL-zqxxzlC4Gf6K%V2&)c>ZTvHx{+b^psah5eV9^65WY;rVh{ z+{0=6dLEv-`;XXUbpA6kGW{135e4fpHZ}ckY+~|XM@Q#Bs2{`3&i+63%=!OyzyJNO z|NZ}e!|(t9>wf?LpLh4+e>NVT|NH_1|FyNX{~MW@{0HgP)zkYgE+O%siG}q)i;(nx z?wZ?g7y=dw!|g?4ux1}kcfXJTdj&&b3KHiMOm`#*<}_B_Q};P+IoCqOJRXsnWHeHpoGO z?a|P=qGG-8uGnn;@xSKyo&PmkcK&bLf9U_rdk_Dw`|<1l)<6IL@BH)s|DM19|L=oh z5E~@6<wf!>{`fCE`RzSezZT8Y7+j8|E%K66 z-d{D{_xFGMtAGD@{{H`e-=F{g5B>fB|M=hk|4;w@|Nq?I|NqZJF^GNY@BjbD{zBA% z)b9TM|NpeN|NdJa{QF;g#>cyCIr~9fG7P(L3GgJY5*3^DVzbJwKmWq-{rkWE`~UwJ z{`~)c_3!`xxBmYBfA{bI|M#I7#0H6h`n8w-{{O%G$N&FX5C8qw*!}09*pyfMIMX*v zRy8HKk^xePz{crsJ?|AOdKa+hwf2pr}e?oG~gBolrIE8_9 z%22eMl|j^=he1AG0F2fBco`TN$Qo~;ljoT^jML<}JSQh|hc2GU8@FW-Pr|0tJV{$F z@g#1!!jrJ+3~&6_eLP{y7w~vZNoO<3S7%@_rnvl~$Zr;foNRv6)Hy>J<#5NXKfsfH z=rd2zslPns=l}CmU;58ed*we*-BmE=uD$Y~8zfe7;XhCD>AyTVhrjZ~Z8*ghx}=2L zceWmbRy_|zZXrptJqw5bOe?O~^?f`UdvEd-pZUd8b>$y-!_EKPEqDKOx848G-SO~0 zch{r;+})2M7{msNwcm%RX}tNLr~1l&p5imVc`^??;EvxkliP2OBZEpV&M^#~+;NfHZ>A$P>UXH$ zm^l6C`SPUizsOT|=`VM~-T&MjPyciEzy8lP<^6x|nV{0cUctLN2! z?$$^Dd8%*x=}3U&Hr3&PatD;(?0*_oAdp@;KE{{`m%_%FQV_kXb!pmE#Z|0R}w`Y%5F;eWA-m;Q@(9R4p-xAnhB{g(g2^qAh#=i?$v3FV=bFzgYjd|Kihb{g+t${=fK&UtqIDmiz{r4UP-Y zxGKmGHMjn8XCAr94l0{SNkLknJlrvx4{%pp|I5|-1RN$jv%dZpT=4V1$g=HEBt4XRQb;;ul1i@ z)9OE)vf+PLC4>L0%0~a$v~BPe`Zm+|4ag6|Ct2E!DGfu0?@H!&^R)W09cG!KLe ze`XPx|9la3|7BKx`!Buj-+z$Vpm-Hp_zOH%%r)Wtf3D_7|G7)A{Ns*TJ%>R)o&&p^ z5u%(vGfla3j^F04yZfK3`}KdG>0kZ}F8ui)G+r$Ua{K!K|7AD+|1Y=k_kYpq)!=b& zMlQbpjGR3G8M#1n(A*Fy{>OjW zjsO13tpERCa_ztWVk`ba<_Tte{m<3&`af6w{r_Bfr(Sah&T~Sz6bnl$l#MffTQ66| zwclKA&;D~y`S_oI{*V8lJPI;bW&>p0Tz>2S|BBoH{+C^N`#+nC;eSSE*8hwwAj}4( zS^qP#a{g!Jk^IldBlDk;l^Z;M&cMX{AB-89z%;Y4)PKqDeg74={rRu36=F8X4v<|S zJ3;A{d+MkEps{=Is+)hg6L-vJu*coj;Pjg##hG*T3RlB}f1G`9|MSlN_Fs6}pZ^kT z|NWQQ2pM-**!J(g;!eoeyYiM#{{@0_{xh($|7T!k|Ia9*@t;}2{y(##<9`m1j{iJK z8~^j9Z2r&T-S?ka#q&Rty#0Sh5zYS$EF56>^VkRfS6cn_KWH2uWG2XL(6~IvE|4EU zc7w_~&i?oRIYDVT=fp!+|M{S90hYjLVBid#6UtS7PH`@i(YzyHNo|NAev_}71~>0kbHg623XuKnSNT2qGA@eB-190{AZ za5mig$Jzh(KkuCH|Akll`7gQd&wrWKAOA})y#HT%=EeWA(@y@En||!S!mN}3l@?$8 zueSc)f3?-u{|lDv`Y$s1@qg*55B{sod;H&W$LIfUM}PizJNEOx)$T9(p<@_)4r_x>v{x%^*Y_9?KLGSg1{m!5h4zx2Yp|7BLa|1Y)fH+W8u zci#8^oD<*u=LF5YrS3k1we4+~B*mF`@;yiUlmDDkKmO-m^z*;yiXZwo_z-T3=I1Dq9JB_ zNBma{O#UyM)%{*T!u4()UPJ1pGy-*&~R|F(;c{L%#^?&a9U;hiveg0n}rSrd~ zyYGLS2^;=v?tJ%O{SajCMeF3x|JtX2{MSA6{lC$%*Z&=7Z~SlV?e||HtNy>@lr#UW zHa`C!d-2=<{QH0Y7d-g;KmPLf{|?(<{#TxH`M-Q#>why(-~TQX7XLRo`trZtnQvgT zG*A8n&+Dik`uSgT_s9PhGxq(r^A7wkmC*EGXy$|eybHhn=a}^NKS$HO|C|Y%mVmqU z2v0Dvr*2=!(emIw=hXNA`R0H4FWk8Ozkz4Sf3pdj|10nR`CsKQbl&F#bPnR&&;RGGtB&G_Ronnp-*c3v8G2fnWcP<{bNPg2pS|tjf6mMU2N@Vtko!7p>^X-vv9~||&o%wSe}S0~|0@J${5L7=_^-14{eP81 zzrgcG>L(#{9$M#r{?|G7;=l6Jlm9g~UHNZ#`OAO9i=Y2nH%<7j>YMamWB$$m?uXw0 zkGb~if832<|GkcW_^-R{-hZW#g8$Z4?f;F=zWZ-@_1k~VE!X}lFFE;N`|#8MTIYX& z=Tks-sDk`(=+}Sc-CzEjR80D>>K*%EVCwDvTr)rXXYYLapCk9+c?OkeK1A?Ka#7_Osp1j~cTiopbys0z(8{PN@p2M<8%=oVy zU-MsW)uaEWhd%!|KlNMU(F->zfoezf2BR2|EnDR`Cs)U)a_?~{RgETiQZlR*}^CNXHQx5pDkf7cpgsg zc7giH~(ekUjMHWQT*R5Ch5P))sO!T&b|H57SR5m zEq)HfjL0efC3<%L*E;v(zs5OGJVWdR#evfPZ~yg*d;e=Y`u-PeI`N;Q`^A5@!sF)| z%#hO$10!4ZzPX(3&;QHBbo@6mxB9P_+3;Uu-v1 z?|-(W`TyBsXZ~mNZT_!!Qgj3;wf(PWUh0w&lOxh3{ZHw9i5Ena0VV|J4tC z{jXg!>%XbF<$u|ThW}h`kN>k5oIuaN3~ceMDmXh{{nKsQ|KBMn_P<_K&VRk#Z~uel z2My2t`fqgc_kWYifBu_X`Tbvh>cRhPzODaR!zcb{4eb8U8QA*Y^zz65=I5XMH;PL5 z?~$JM-!VDkzph`@e{J9B|Mu~z|Gn~y|C{&*{5Lys_rKY-&;PkX+W)f#cKv4!pY)&A zr{%xg#C`uwF8}&(a^(-$E|8sir+@v|+4teUc24Vm2k*fD`VBk&b9cV_&z5~)5nB1l zz#cTqoulQ|Z>gEz{#)0s_;2X$|KD)s<^SfVzW=wl_~*aXl|TQjul@ONbK~!SgFTP` z^MrT(XSOT(&*D(}U%GX}f4jS1|C_Db``_Hh|G&%5BmceczWN_{?85(`BWM5n+`OoRs_+M|w{r}cC{{FYV{^!5tl|NuR zP0#-LufO5ef4#uC|2BpF|7GTU`On$@`X77D>TE>bg|dv(L^!JNyb+)JHq(I-~RgVUS0p+D5vPZ{mGmE?XSK2FW)b>#Jb-KM$!t$l+3+b+KQUt;Eu{~QfZzp=SAsX|?i zOfj8H?Ud05C8u6x&QaS*X`f`-LHK8?|Ashf3um}{u>9!{`Ug; zVg2|2!jr!LXDhjUoWZ^x>$nPY!a9Gx&JW+!=KcKdyXfkFYoDPdO{)9;JFPqZ z-|OhT|GpQV|M$Q0>c9V$=l^}r-Tm*n<-mXQx|aVIzJC8hX3zg0|L*Jm#2^3vC;j~Y zKN*UXe*6cEC4Bn%KWyo$|JDJ4|4mD)|2uEk^WW$6?f?FlpMlNvz3}3{*Rcoxoi?2O zZ_+gNzmb3Ve-lU7|4tLO|JPdlEpZ}?3&iK#C z%l}_fQ|rHzyXSvB55ND$c~$>yCd~WqG^8|Rh$H*oj-@8s_OUqfI2zthwi z|1&>*`=9sg|Ns17|Nj?2F^CNk%l`K3zt@U2|J965{=2w){5N#-_-~k&``@N_+JA>x z%m3RYwQ2UW@i75Oicfqn3?^zwz2tdYG(Fd zP)G=_`l%!>;Kii|NXD|{U3~Lp>*}{|Nl!re*JG=Qud#jjqN`VFYkY2W8?oe zP%{ioO#g%C#VxI@{##mF{+E)L{?EkB0`8BpNo)QWY`pV_t?c>~2G>RCYsry)z`z>2 z+K{XI&UG&9)c?$!ybv>(nE$h|u>R-e&cny|pM{MLJSWb=!TDd) z*YAJDgU8@?$Q{4`|L^<*!5x48{|DiQ_n-corKbI7;o$hs$i(!YnVI=NH#hfxF>!IQ znUa!{;CX&lcJ}{FtZe^TIXM5b@CyEC*Z2L;S$*pzOX@y*6vtxa)HAW9Y|G^>U4@+1URxvamqr{aM+-?qK8Q{?Eb3|6f>9@qb9)g#Z1YzW$&3 z```Z=fByfU`R6~F2C=98`Tu{?_h0{$SFZjqp`r1gm5b{?D+kAaW>!}48UpaT0A?1j z9ju&O|Jix?{&R~;{^vII{?A){^dD>Pu?a+v!^wnla+O?KD_(ixkC=PLe^Cv?|3WhI z|9M5l{? z)L+h;TSpmeGLhPASOXqKkS%t(nqcemQ|dF{{56?y_`h%Cl>ct&x&QUuJpSu@c>OmI z4gK$#U+_P5+Km6*=Pv$V_T~HkwZH%U-|*-E|4qe-r!W66eD(JKiVvUvumADu|CZnX z{_ptn@Bgkpko64v|Nj4f5V~doM1wF$Z111{U^UzS{QJKd z`S$;!XK(+{d-Caj#HpYEHP`?7FFf_rd$!_JK@6d*n2B>ThAKw3tlj=%6F)xCTL0&- z>*>G$C%yUif78$Z|F{2!xCP{<1AqVjKMY-GaT2-?EnKjPBg{~8EC?(cuq&42!iO@DuxyYfN=0|Nu4ZioAUH+!p^ zaOb^svTJ_-Q9u0mzx9oO|C`?Z|9||~|Noc&K->yiYXVwx^5F0P{}2EE|NjVzL2S^P z6p-BQzyJT=fco+Lpa1_CeER=C;P$`&dWZl1mtXh$uW;|@BjZV|3cQVfY!2r z*dVcofByZy_UqsOqd)%rU-a?c|G4}A{u`b7`%iV}@842OKHU>+xzfzwQO}2IF*QZ% znfR*?8i`JS)GEE|-8rSbKfi0A`|}U9o-g^y-~Y|8|Nfu#9d9`tP5VWXfTBn#*1<)Lrv_i~gE7 zdkj{;+N-Y!U?d?JH SCQKBXIjVg$1V%#uEd&5Rgoecc diff --git a/who-am-i/static/style.css b/who-am-i/static/style.css deleted file mode 100644 index 36a3a5c..0000000 --- a/who-am-i/static/style.css +++ /dev/null @@ -1,195 +0,0 @@ -body { - color: #434; - font-family: 'Iowan Old Style', 'Palatino Linotype', 'URW Palladio L', P052, serif; - margin: 0; - min-height: 100vh; - padding: 0; -} -.wrap { - border: 2px solid #221828; - border-radius: 0.5rem; - box-sizing: border-box; - overflow: hidden; - display: flex; - flex-direction: column; - min-height: 100vh; -} -.wrap.unframed { - border-radius: 0; - border-width: 0.4rem; -} -header { - background: #221828; - display: flex; - justify-content: space-between; - padding: 0 0.25rem; - color: #c9b; - display: flex; - gap: 0.5rem; - align-items: baseline; -} -header > * { - flex-basis: 33%; -} -header > .empty { - font-size: 0.8rem; - opacity: 0.5; -} -header > .title { - text-align: center; -} -header > a.micro { - text-decoration: none; - font-size: 0.8rem; - text-align: right; - opacity: 0.5; -} -header > a.micro:hover { - opacity: 1; -} -main { - background: #ccc; - display: flex; - flex-direction: column; - flex-grow: 1; - padding: 0.25rem 0.5rem; -} -.mini-content { - margin: 1rem auto 0; - padding: 1rem 0.5rem; - max-width: 21rem; -} - -.explain { - border-bottom: 1px dashed #888; - margin-bottom: 1rem; - padding-bottom: 2rem; -} -.explain p { - text-align: left; -} - -#error-message { - font-size: 0.8rem; - color: #a31; -} - -#error-message:not(.hidden) + #prompt { - display: none !important; -} - -#error-message, -p { - margin: 1rem 0 0; - text-align: center; -} -p.detail { - font-size: 0.8rem; -} -p.detail.no { - font-style: italic; -} -.parent-host { - font-weight: bold; - color: #48c; - display: inline-block; - padding: 0 0.125rem; - border-radius: 0.25rem; - border: 1px solid #aaa; - font-size: 0.8rem; -} - -#loader { - display: flex; - flex-grow: 1; - justify-content: center; - align-items: center; -} -.spinner { - animation: rotation 1.618s ease-in-out infinite; - border-radius: 50%; - border: 3px dashed #434; - box-sizing: border-box; - display: inline-block; - height: 1.5em; - width: 1.5em; -} -@keyframes rotation { - 0% { transform: rotate(0deg) } - 100% { transform: rotate(360deg) } -} -/* loader visibility is mutually exclusive with its immediate sibling */ -#loader:not(.hidden) + * { - display: none !important; -} - -#user-info { - flex-grow: 1; - display: flex; - flex-direction: column; - justify-content: center; -} -.action { - background: #eee; - display: flex; - justify-content: space-between; - padding: 0.5rem 0.25rem 0.5rem 0.5rem; - font-size: 0.8rem; - align-items: baseline; - border-radius: 0.5rem; - border: 1px solid #bbb; - cursor: pointer; -} -.action:hover { - background: #fff; -} -#form-action:not(.hidden) + .action { - display: none !important; -} - -#connect, -#allow, -#revoke { - background: transparent; - border: none; - border-left: 1px solid #bbb; - padding: 0 0.5rem; - color: #375; - font: inherit; - cursor: pointer; -} -#revoke { - color: #a31; -} -#action:hover #allow { - color: #285; -} - -#need-storage { - font-size: 0.8rem; -} -.problem { - color: #a31; -} - -#or { - font-size: 0.8rem; - text-align: center; -} -#or p { - margin: 0 0 1rem; -} - -input.handle { - border: none; - border-bottom: 1px dashed #aaa; - background: transparent; -} - -.hidden { - display: none !important; -} - -.hello-connect-plz { - margin: 1.667rem 0 0.667rem; -} diff --git a/who-am-i/templates/auth-fail.hbs b/who-am-i/templates/auth-fail.hbs deleted file mode 100644 index 4ba2b00..0000000 --- a/who-am-i/templates/auth-fail.hbs +++ /dev/null @@ -1,20 +0,0 @@ -{{#*inline "main"}} -

- Auth failed: {{ reason }} -

- -
- auth failed. -
- - -{{/inline}} - -{{#> base-framed}}{{/base-framed}} diff --git a/who-am-i/templates/authorized.hbs b/who-am-i/templates/authorized.hbs deleted file mode 100644 index 9138f4c..0000000 --- a/who-am-i/templates/authorized.hbs +++ /dev/null @@ -1,19 +0,0 @@ - - -great job! - -

oauth success!

-

this window should automatically close itself (probably a bug if it hasn't)

- - diff --git a/who-am-i/templates/base-base.hbs b/who-am-i/templates/base-base.hbs deleted file mode 100644 index 7bd308b..0000000 --- a/who-am-i/templates/base-base.hbs +++ /dev/null @@ -1,17 +0,0 @@ - - - - - who-am-i - - - - - - - - - - {{> body}} - - diff --git a/who-am-i/templates/base-framed.hbs b/who-am-i/templates/base-framed.hbs deleted file mode 100644 index ba0bccd..0000000 --- a/who-am-i/templates/base-framed.hbs +++ /dev/null @@ -1,28 +0,0 @@ -{{#*inline "description"}}{{/inline}} - -{{#*inline "body"}} -
-
-
🔒
- who-am-i - microcosm -
- -
- {{> main}} -
-
-{{/inline}} - -{{#> base-base}}{{/base-base}} diff --git a/who-am-i/templates/base-full.hbs b/who-am-i/templates/base-full.hbs deleted file mode 100644 index cd27a2b..0000000 --- a/who-am-i/templates/base-full.hbs +++ /dev/null @@ -1,26 +0,0 @@ -{{#*inline "body"}} -
-
-
🔒
- who-am-i - microcosm -
- -
- {{> main}} -
-
-{{/inline}} - -{{#> base-base}}{{/base-base}} diff --git a/who-am-i/templates/hello.hbs b/who-am-i/templates/hello.hbs deleted file mode 100644 index 24cbad2..0000000 --- a/who-am-i/templates/hello.hbs +++ /dev/null @@ -1,122 +0,0 @@ -{{#*inline "description"}}A little identity-verifying auth service for microcosm demos{{/inline}} - -{{#*inline "main"}} -
-
-

This is a little identity-verifying service for microcosm demos.

-

Only read access to your public data is required to connect: connecting does not grant any ability to modify your account or data.

-
- - {{#if did}} - - -

- Connected identity: -

- -
- -
- -
-
- - -
-
- - {{else}} - -

Connect your handle

- - {{#if is_auth_reload}} - {{#if no_cookie}} -

- No identity connected. Your browser may be blocking access for connecting. -

- {{else}} - {{#if auth_failed}} -

- No identity connected. Connecting failed or was denied. -

- {{else}} -

- No identity connected. -

- {{/if}} - {{/if}} - {{/if}} - -
-
- - -
-
- {{/if}} - -
- -{{/inline}} - -{{#> base-full}}{{/base-full}} diff --git a/who-am-i/templates/prompt-error.hbs b/who-am-i/templates/prompt-error.hbs deleted file mode 100644 index 26fa8c4..0000000 --- a/who-am-i/templates/prompt-error.hbs +++ /dev/null @@ -1,18 +0,0 @@ -{{#*inline "main"}} -
-

Something went wrong :(

-

{{ reason }}

-

{{ detail }}

- -
- - -{{/inline}} - -{{#> base-framed}}{{/base-framed}} diff --git a/who-am-i/templates/prompt.hbs b/who-am-i/templates/prompt.hbs deleted file mode 100644 index 27d2499..0000000 --- a/who-am-i/templates/prompt.hbs +++ /dev/null @@ -1,187 +0,0 @@ -{{#*inline "main"}} -

- Connect in the ATmosphere -

- - - -

- {{ parent_host }} wants to confirm your handle -

- -
- -
- -
-
- - -
- -
- - -
-
- - - - - - - -{{/inline}} - -{{#> base-framed}}{{/base-framed}}