From edd0c2e237d61ef59ae044ae657ab46cda18f7e8 Mon Sep 17 00:00:00 2001 From: phil Date: Mon, 28 Jul 2025 20:26:03 -0400 Subject: [PATCH] cachey --- Cargo.lock | 584 ++++++++++++++++++++++++++++---- Cargo.toml | 1 + slingshot/.gitignore | 1 + slingshot/Cargo.toml | 19 ++ slingshot/src/consumer.rs | 76 +++++ slingshot/src/error.rs | 21 ++ slingshot/src/firehose_cache.rs | 17 + slingshot/src/lib.rs | 8 + slingshot/src/main.rs | 107 ++++++ slingshot/src/record.rs | 24 ++ 10 files changed, 790 insertions(+), 68 deletions(-) create mode 100644 slingshot/.gitignore create mode 100644 slingshot/Cargo.toml create mode 100644 slingshot/src/consumer.rs create mode 100644 slingshot/src/error.rs create mode 100644 slingshot/src/firehose_cache.rs create mode 100644 slingshot/src/lib.rs create mode 100644 slingshot/src/main.rs create mode 100644 slingshot/src/record.rs diff --git a/Cargo.lock b/Cargo.lock index 4da0c78..2197e61 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,6 +24,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", + "getrandom 0.2.15", "once_cell", "version_check", "zerocopy 0.7.35", @@ -121,6 +122,12 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + [[package]] name = "arrayvec" version = "0.7.6" @@ -155,7 +162,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn", + "syn 2.0.103", ] [[package]] @@ -173,6 +180,18 @@ dependencies = [ "nom", ] +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + [[package]] name = "async-compression" version = "0.4.25" @@ -216,9 +235,15 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + [[package]] name = "async-trait" version = "0.1.88" @@ -227,7 +252,7 @@ checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -377,6 +402,18 @@ dependencies = [ "trait-variant", ] +[[package]] +name = "auto_enums" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c170965892137a3a9aeb000b4524aa3cc022a310e709d848b6e1cdce4ab4781" +dependencies = [ + "derive_utils", + "proc-macro2", + "quote", + "syn 2.0.103", +] + [[package]] name = "autocfg" version = "1.4.0" @@ -613,7 +650,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn", + "syn 2.0.103", "which", ] @@ -632,7 +669,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn", + "syn 2.0.103", ] [[package]] @@ -650,7 +687,7 @@ dependencies = [ "regex", "rustc-hash 2.1.1", "shlex", - "syn", + "syn 2.0.103", ] [[package]] @@ -813,9 +850,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.40" +version = "4.5.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40b6887a1d8685cebccf115538db5c0efe625ccac9696ad45c409d96566e910f" +checksum = "be92d32e80243a54711e5d7ce823c35c41c9d929dc4ab58e1276f625841aadf9" dependencies = [ "clap_builder", "clap_derive", @@ -823,26 +860,26 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.40" +version = "4.5.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0c66c08ce9f0c698cbce5c0279d0bb6ac936d8674174fe48f736533b964f59e" +checksum = "707eab41e9622f9139419d573eca0900137718000c517d47da73045f54331c3d" dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim", + "strsim 0.11.1", ] [[package]] name = "clap_derive" -version = "4.5.40" +version = "4.5.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2c7947ae4cc3d851207c1adb5b5e260ff0cca11446b1d6d1423788e442257ce" +checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491" dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -871,6 +908,15 @@ dependencies = [ "cc", ] +[[package]] +name = "cmsketch" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "553c840ee51da812c6cd621f9f7e07dfb00a49f91283a8e6380c78cba4f61aba" +dependencies = [ + "paste", +] + [[package]] name = "colorchoice" version = "1.0.3" @@ -911,7 +957,7 @@ dependencies = [ "clap", "ctrlc", "flume", - "fs4", + "fs4 0.12.0", "headers-accept", "links", "mediatype", @@ -1074,14 +1120,38 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core 0.14.4", + "darling_macro 0.14.4", +] + [[package]] name = "darling" version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.10.0", + "syn 1.0.109", ] [[package]] @@ -1094,8 +1164,19 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim", - "syn", + "strsim 0.11.1", + "syn 2.0.103", +] + +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core 0.14.4", + "quote", + "syn 1.0.109", ] [[package]] @@ -1104,9 +1185,9 @@ version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ - "darling_core", + "darling_core 0.20.11", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -1146,7 +1227,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18e4fdb82bd54a12e42fb58a800dcae6b9e13982238ce2296dc3570b92148e1f" dependencies = [ "data-encoding", - "syn", + "syn 2.0.103", ] [[package]] @@ -1191,10 +1272,10 @@ version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" dependencies = [ - "darling", + "darling 0.20.11", "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -1204,7 +1285,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn", + "syn 2.0.103", +] + +[[package]] +name = "derive_utils" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccfae181bab5ab6c5478b2ccb69e4c68a02f8c3ec72f6616bfec9dbc599d2ee0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.103", ] [[package]] @@ -1248,7 +1340,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -1257,6 +1349,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57" +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + [[package]] name = "dropshot" version = "0.16.2" @@ -1319,7 +1417,7 @@ dependencies = [ "semver", "serde", "serde_tokenstream", - "syn", + "syn 2.0.103", ] [[package]] @@ -1392,7 +1490,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -1404,7 +1502,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -1526,6 +1624,9 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" dependencies = [ + "futures-core", + "futures-sink", + "nanorand", "spin", ] @@ -1565,6 +1666,112 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "foyer" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b4d8e96374206ff1b4265f2e2e6e1f80bc3048957b2a1e7fdeef929d68f318f" +dependencies = [ + "equivalent", + "foyer-common", + "foyer-memory", + "foyer-storage", + "madsim-tokio", + "mixtrics", + "pin-project", + "serde", + "thiserror 2.0.12", + "tokio", + "tracing", +] + +[[package]] +name = "foyer-common" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "911b8e3f23d5fe55b0b240f75af1d2fa5cb7261d3f9b38ef1c57bbc9f0449317" +dependencies = [ + "bincode 1.3.3", + "bytes", + "cfg-if", + "itertools 0.14.0", + "madsim-tokio", + "mixtrics", + "parking_lot", + "pin-project", + "serde", + "thiserror 2.0.12", + "tokio", + "twox-hash", +] + +[[package]] +name = "foyer-intrusive-collections" +version = "0.10.0-dev" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4fee46bea69e0596130e3210e65d3424e0ac1e6df3bde6636304bdf1ca4a3b" +dependencies = [ + "memoffset", +] + +[[package]] +name = "foyer-memory" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "506883d5a8500dea1b1662f7180f3534bdcbfa718d3253db7179552ef83612fa" +dependencies = [ + "arc-swap", + "bitflags", + "cmsketch", + "equivalent", + "foyer-common", + "foyer-intrusive-collections", + "hashbrown 0.15.2", + "itertools 0.14.0", + "madsim-tokio", + "mixtrics", + "parking_lot", + "pin-project", + "serde", + "thiserror 2.0.12", + "tokio", + "tracing", +] + +[[package]] +name = "foyer-storage" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ba8403a54a2f2032fb647e49c442e5feeb33f3989f7024f1b178341a016f06d" +dependencies = [ + "allocator-api2", + "anyhow", + "auto_enums", + "bytes", + "equivalent", + "flume", + "foyer-common", + "foyer-memory", + "fs4 0.13.1", + "futures-core", + "futures-util", + "itertools 0.14.0", + "libc", + "lz4", + "madsim-tokio", + "ordered_hash_map", + "parking_lot", + "paste", + "pin-project", + "rand 0.9.1", + "serde", + "thiserror 2.0.12", + "tokio", + "tracing", + "twox-hash", + "zstd", +] + [[package]] name = "fs4" version = "0.12.0" @@ -1575,6 +1782,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "fs4" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" +dependencies = [ + "rustix 1.0.5", + "windows-sys 0.59.0", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -1637,7 +1854,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -1790,6 +2007,15 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" +dependencies = [ + "ahash", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -2073,7 +2299,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2", + "socket2 0.5.9", "system-configuration", "tokio", "tower-service", @@ -2220,7 +2446,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -2281,13 +2507,24 @@ dependencies = [ "compare", ] +[[package]] +name = "io-uring" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4" +dependencies = [ + "bitflags", + "cfg-if", + "libc", +] + [[package]] name = "ipconfig" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" dependencies = [ - "socket2", + "socket2 0.5.9", "widestring", "windows-sys 0.48.0", "winreg", @@ -2355,6 +2592,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.15" @@ -2403,7 +2649,7 @@ checksum = "43ce13c40ec6956157a3635d97a1ee2df323b263f09ea14165131289cb0f5c19" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -2503,9 +2749,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.171" +version = "0.2.174" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" +checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" [[package]] name = "libfuzzer-sys" @@ -2671,6 +2917,15 @@ dependencies = [ "xxhash-rust", ] +[[package]] +name = "lz4" +version = "1.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" +dependencies = [ + "lz4-sys", +] + [[package]] name = "lz4-sys" version = "1.11.1+lz4-1.10.0" @@ -2696,6 +2951,60 @@ dependencies = [ "libc", ] +[[package]] +name = "madsim" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db6694555643da293dfb89e33c2880a13b62711d64b6588bc7df6ce4110b27f1" +dependencies = [ + "ahash", + "async-channel", + "async-stream", + "async-task", + "bincode 1.3.3", + "bytes", + "downcast-rs", + "futures-util", + "lazy_static", + "libc", + "madsim-macros", + "naive-timer", + "panic-message", + "rand 0.8.5", + "rand_xoshiro 0.6.0", + "rustversion", + "serde", + "spin", + "tokio", + "tokio-util", + "toml", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "madsim-macros" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d248e97b1a48826a12c3828d921e8548e714394bf17274dd0a93910dc946e1" +dependencies = [ + "darling 0.14.4", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "madsim-tokio" +version = "0.2.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d3eb2acc57c82d21d699119b859e2df70a91dbdb84734885a1e72be83bdecb5" +dependencies = [ + "madsim", + "spin", + "tokio", +] + [[package]] name = "match_cfg" version = "0.1.0" @@ -2729,6 +3038,15 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + [[package]] name = "metrics" version = "0.24.2" @@ -2870,6 +3188,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "mixtrics" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adbcddf5a90b959eea97ae505e0391f5c6dd411fbf546d43b9c59ad1c3bd4391" +dependencies = [ + "itertools 0.14.0", + "parking_lot", +] + [[package]] name = "moka" version = "0.12.10" @@ -2931,6 +3259,21 @@ dependencies = [ "unsigned-varint", ] +[[package]] +name = "naive-timer" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "034a0ad7deebf0c2abcf2435950a6666c3c15ea9d8fad0c0f48efa8a7f843fed" + +[[package]] +name = "nanorand" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" +dependencies = [ + "getrandom 0.2.15", +] + [[package]] name = "native-tls" version = "0.2.14" @@ -3130,7 +3473,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -3161,6 +3504,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "ordered_hash_map" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab0e5f22bf6dd04abd854a8874247813a8fa2c8c1260eba6fbb150270ce7c176" +dependencies = [ + "hashbrown 0.13.2", +] + [[package]] name = "overload" version = "0.1.1" @@ -3189,6 +3541,12 @@ dependencies = [ "primeorder", ] +[[package]] +name = "panic-message" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384e52fd8fbd4cbe3c317e8216260c21a0f9134de108cea8a4dd4e7e152c472d" + [[package]] name = "parking" version = "2.2.1" @@ -3298,7 +3656,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -3311,6 +3669,26 @@ dependencies = [ "sha2", ] +[[package]] +name = "pin-project" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.103", +] + [[package]] name = "pin-project-lite" version = "0.2.16" @@ -3387,7 +3765,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6837b9e10d61f45f987d50808f83d1ee3d206c66acf650c3e4ae2e1f6ddedf55" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.103", ] [[package]] @@ -3604,7 +3982,7 @@ checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -3947,7 +4325,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn", + "syn 2.0.103", ] [[package]] @@ -4050,7 +4428,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -4061,7 +4439,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -4079,9 +4457,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.140" +version = "1.0.141" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +checksum = "30b9eff21ebe718216c6ec64e1d9ac57087aad11efc64e32002bce4a0d4c03d3" dependencies = [ "itoa", "memchr", @@ -4130,7 +4508,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn", + "syn 2.0.103", ] [[package]] @@ -4169,10 +4547,10 @@ version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e" dependencies = [ - "darling", + "darling 0.20.11", "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -4258,6 +4636,26 @@ dependencies = [ "autocfg", ] +[[package]] +name = "slingshot" +version = "0.1.0" +dependencies = [ + "axum", + "clap", + "ctrlc", + "env_logger", + "foyer", + "jetstream", + "log", + "metrics", + "metrics-exporter-prometheus 0.17.2", + "serde", + "serde_json", + "thiserror 2.0.12", + "tokio", + "tokio-util", +] + [[package]] name = "slog" version = "2.7.0" @@ -4329,6 +4727,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "socket2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "spacedust" version = "0.1.0" @@ -4389,6 +4797,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ae9eec00137a8eed469fb4148acd9fc6ac8c3f9b110f52cd34698c8b5bfa0e" +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "strsim" version = "0.11.1" @@ -4401,6 +4815,17 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.103" @@ -4429,7 +4854,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -4515,7 +4940,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -4526,7 +4951,7 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -4625,20 +5050,22 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.45.1" +version = "1.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" +checksum = "43864ed400b6043a4757a25c7a64a8efde741aed79a056a2fb348a406701bb35" dependencies = [ "backtrace", "bytes", + "io-uring", "libc", "mio", "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "slab", + "socket2 0.6.0", "tokio-macros", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4649,7 +5076,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -4817,9 +5244,21 @@ checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ "log", "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.103", +] + [[package]] name = "tracing-core" version = "0.1.33" @@ -4867,7 +5306,7 @@ checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -4912,6 +5351,15 @@ dependencies = [ "utf-8", ] +[[package]] +name = "twox-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b907da542cbced5261bd3256de1b3a1bf340a3d37f93425a07362a1d687de56" +dependencies = [ + "rand 0.9.1", +] + [[package]] name = "typenum" version = "1.18.0" @@ -5157,7 +5605,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn", + "syn 2.0.103", "wasm-bindgen-shared", ] @@ -5192,7 +5640,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -5352,7 +5800,7 @@ checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -5363,7 +5811,7 @@ checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -5374,7 +5822,7 @@ checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -5385,7 +5833,7 @@ checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -5665,7 +6113,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", "synstructure", ] @@ -5695,7 +6143,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -5706,7 +6154,7 @@ checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] @@ -5726,7 +6174,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", "synstructure", ] @@ -5758,7 +6206,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.103", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f0a3b13..6604a84 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,4 +8,5 @@ members = [ "ufos/fuzz", "spacedust", "who-am-i", + "slingshot", ] diff --git a/slingshot/.gitignore b/slingshot/.gitignore new file mode 100644 index 0000000..01c6be7 --- /dev/null +++ b/slingshot/.gitignore @@ -0,0 +1 @@ +foyer diff --git a/slingshot/Cargo.toml b/slingshot/Cargo.toml new file mode 100644 index 0000000..7d76e31 --- /dev/null +++ b/slingshot/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "slingshot" +version = "0.1.0" +edition = "2024" + +[dependencies] +clap = { version = "4.5.41", features = ["derive"] } +ctrlc = "3.4.7" +env_logger = "0.11.8" +foyer = { version = "0.18.0", features = ["serde"] } +jetstream = { path = "../jetstream", features = ["metrics"] } +log = "0.4.27" +metrics = "0.24.2" +metrics-exporter-prometheus = { version = "0.17.1", features = ["http-listener"] } +serde = { version = "1.0.219", features = ["derive"] } +serde_json = { version = "1.0.141", features = ["raw_value"] } +thiserror = "2.0.12" +tokio = { version = "1.47.0", features = ["full"] } +tokio-util = "0.7.15" diff --git a/slingshot/src/consumer.rs b/slingshot/src/consumer.rs new file mode 100644 index 0000000..84f2cc6 --- /dev/null +++ b/slingshot/src/consumer.rs @@ -0,0 +1,76 @@ +use crate::CachedRecord; +use foyer::HybridCache; +use crate::error::ConsumerError; +use jetstream::{ + DefaultJetstreamEndpoints, JetstreamCompression, JetstreamConfig, JetstreamConnector, + events::{CommitOp, Cursor, EventKind}, +}; +use tokio_util::sync::CancellationToken; + +pub async fn consume( + jetstream_endpoint: String, + cursor: Option, + no_zstd: bool, + shutdown: CancellationToken, + cache: HybridCache, +) -> Result<(), ConsumerError> { + let endpoint = DefaultJetstreamEndpoints::endpoint_or_shortcut(&jetstream_endpoint); + if endpoint == jetstream_endpoint { + log::info!("consumer: connecting jetstream at {endpoint}"); + } else { + log::info!("consumer: connecting jetstream at {jetstream_endpoint} => {endpoint}"); + } + let config: JetstreamConfig = JetstreamConfig { + endpoint, + compression: if no_zstd { + JetstreamCompression::None + } else { + JetstreamCompression::Zstd + }, + replay_on_reconnect: true, + channel_size: 1024, // buffer up to ~1s of jetstream events + ..Default::default() + }; + let mut receiver = JetstreamConnector::new(config)? + .connect_cursor(cursor) + .await?; + + log::info!("consumer: receiving messages.."); + loop { + if shutdown.is_cancelled() { + log::info!("consumer: exiting for shutdown"); + return Ok(()); + } + let Some(mut event) = receiver.recv().await else { + log::error!("consumer: could not receive event, bailing"); + break; + }; + + if event.kind != EventKind::Commit { + continue; + } + let Some(ref mut commit) = event.commit else { + log::warn!("consumer: commit event missing commit data, ignoring"); + continue; + }; + + // TODO: something a bit more robust + let at_uri = format!( + "at://{}/{}/{}", + &*event.did, &*commit.collection, &*commit.rkey + ); + + if commit.operation == CommitOp::Delete { + cache.insert(at_uri, CachedRecord::Deleted); + } else { + let Some(record) = commit.record.take() else { + log::warn!("consumer: commit update/delete missing record, ignoring"); + continue; + }; + + cache.insert(at_uri, CachedRecord::Found(record.into())); + } + } + + Err(ConsumerError::JetstreamEnded) +} diff --git a/slingshot/src/error.rs b/slingshot/src/error.rs new file mode 100644 index 0000000..b427c42 --- /dev/null +++ b/slingshot/src/error.rs @@ -0,0 +1,21 @@ +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum ConsumerError { + #[error(transparent)] + JetstreamConnectionError(#[from] jetstream::error::ConnectionError), + #[error(transparent)] + JetstreamConfigValidationError(#[from] jetstream::error::ConfigValidationError), + #[error("jetstream ended")] + JetstreamEnded, + #[error("delay queue output dropped")] + DelayQueueOutputDropped, +} + +#[derive(Debug, Error)] +pub enum MainTaskError { + #[error(transparent)] + ConsumerTaskError(#[from] ConsumerError), + // #[error(transparent)] + // ServerTaskError(#[from] ServerError), +} diff --git a/slingshot/src/firehose_cache.rs b/slingshot/src/firehose_cache.rs new file mode 100644 index 0000000..d3b9a6d --- /dev/null +++ b/slingshot/src/firehose_cache.rs @@ -0,0 +1,17 @@ +use std::path::Path; +use crate::CachedRecord; +use foyer::{HybridCache, DirectFsDeviceOptions, Engine, HybridCacheBuilder}; + + +pub async fn firehose_cache(dir: impl AsRef) -> Result, String> { + let cache = HybridCacheBuilder::new() + .with_name("firehose") + .memory(64 * 2_usize.pow(20)) + .with_weighter(|k: &String, v| k.len() + std::mem::size_of_val(v)) + .storage(Engine::large()) + .with_device_options(DirectFsDeviceOptions::new(dir)) + .build() + .await + .map_err(|e| format!("foyer setup error: {e:?}"))?; + Ok(cache) +} diff --git a/slingshot/src/lib.rs b/slingshot/src/lib.rs new file mode 100644 index 0000000..75dd162 --- /dev/null +++ b/slingshot/src/lib.rs @@ -0,0 +1,8 @@ +mod consumer; +pub mod error; +mod firehose_cache; +mod record; + +pub use consumer::consume; +pub use firehose_cache::firehose_cache; +pub use record::CachedRecord; diff --git a/slingshot/src/main.rs b/slingshot/src/main.rs new file mode 100644 index 0000000..4c46187 --- /dev/null +++ b/slingshot/src/main.rs @@ -0,0 +1,107 @@ +// use foyer::HybridCache; +// use foyer::{Engine, DirectFsDeviceOptions, HybridCacheBuilder}; +use metrics_exporter_prometheus::PrometheusBuilder; +use slingshot::{consume, error::MainTaskError, firehose_cache}; + +use clap::Parser; +use tokio_util::sync::CancellationToken; + + +/// Slingshot record edge cache +#[derive(Parser, Debug, Clone)] +#[command(version, about, long_about = None)] +struct Args { + /// Jetstream server to connect to (exclusive with --fixture). Provide either a wss:// URL, or a shorhand value: + /// 'us-east-1', 'us-east-2', 'us-west-1', or 'us-west-2' + #[arg(long)] + jetstream: String, + /// don't request zstd-compressed jetstream events + /// + /// reduces CPU at the expense of more ingress bandwidth + #[arg(long, action)] + jetstream_no_zstd: bool, +} + +#[tokio::main] +async fn main() -> Result<(), String> { + env_logger::init(); + + let shutdown = CancellationToken::new(); + + let ctrlc_shutdown = shutdown.clone(); + ctrlc::set_handler(move || ctrlc_shutdown.cancel()).expect("failed to set ctrl-c handler"); + + let args = Args::parse(); + + if let Err(e) = install_metrics_server() { + log::error!("failed to install metrics server: {e:?}"); + } else { + log::info!("metrics listening at http://0.0.0.0:8765"); + } + + log::info!("setting up firehose cache..."); + let cache = firehose_cache("./foyer").await?; + log::info!("firehose cache ready."); + + let mut tasks: tokio::task::JoinSet> = tokio::task::JoinSet::new(); + + let consumer_shutdown = shutdown.clone(); + tasks.spawn(async move { + consume( + args.jetstream, + None, + args.jetstream_no_zstd, + consumer_shutdown, + cache, + ) + .await?; + Ok(()) + }); + + + tokio::select! { + _ = shutdown.cancelled() => log::warn!("shutdown requested"), + Some(r) = tasks.join_next() => { + log::warn!("a task exited, shutting down: {r:?}"); + shutdown.cancel(); + } + } + + tokio::select! { + _ = async { + while let Some(completed) = tasks.join_next().await { + log::info!("shutdown: task completed: {completed:?}"); + } + } => {}, + _ = tokio::time::sleep(std::time::Duration::from_secs(3)) => { + log::info!("shutdown: not all tasks completed on time. aborting..."); + tasks.shutdown().await; + }, + } + + log::info!("bye!"); + + Ok(()) +} + +fn install_metrics_server() -> Result<(), metrics_exporter_prometheus::BuildError> { + log::info!("installing metrics server..."); + let host = [0, 0, 0, 0]; + let port = 8765; + PrometheusBuilder::new() + .set_quantiles(&[0.5, 0.9, 0.99, 1.0])? + .set_bucket_duration(std::time::Duration::from_secs(300))? + .set_bucket_count(std::num::NonZero::new(12).unwrap()) // count * duration = 60 mins. stuff doesn't happen that fast here. + .set_enable_unit_suffix(false) // this seemed buggy for constellation (sometimes wouldn't engage) + .with_http_listener((host, port)) + .install()?; + log::info!( + "metrics server installed! listening on http://{}.{}.{}.{}:{port}", + host[0], + host[1], + host[2], + host[3] + ); + Ok(()) +} + diff --git a/slingshot/src/record.rs b/slingshot/src/record.rs new file mode 100644 index 0000000..e6d7d7f --- /dev/null +++ b/slingshot/src/record.rs @@ -0,0 +1,24 @@ +use serde_json::value::RawValue; +use serde::{Serialize, Deserialize}; + +#[derive(Debug, Serialize, Deserialize)] +pub struct RawRecord(String); + +impl From> for RawRecord { + fn from(rv: Box) -> Self { + Self(rv.get().to_string()) + } +} + +/// only for use with stored (validated) values, not general strings +impl From for Box { + fn from(RawRecord(s): RawRecord) -> Self { + RawValue::from_string(s).expect("stored string from RawValue to be valid") + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub enum CachedRecord { + Found(RawRecord), + Deleted, +} -- 2.51.2