diff --git a/tools/parse-plays/Cargo.lock b/tools/parse-plays/Cargo.lock index 556327d..210e0a2 100644 --- a/tools/parse-plays/Cargo.lock +++ b/tools/parse-plays/Cargo.lock @@ -8,30 +8,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" -[[package]] -name = "aho-corasick" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" -dependencies = [ - "memchr", -] - -[[package]] -name = "alloc-no-stdlib" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" - -[[package]] -name = "alloc-stdlib" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" -dependencies = [ - "alloc-no-stdlib", -] - [[package]] name = "allocator-api2" version = "0.2.21" @@ -53,33 +29,6 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" -[[package]] -name = "async-compression" -version = "0.4.42" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" -dependencies = [ - "compression-codecs", - "compression-core", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "atomic-polyfill" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" -dependencies = [ - "critical-section", -] - -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - [[package]] name = "autocfg" version = "1.5.1" @@ -92,12 +41,6 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" -[[package]] -name = "base16ct" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" - [[package]] name = "base256emoji" version = "1.0.2" @@ -120,12 +63,6 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "base64ct" -version = "1.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" - [[package]] name = "bitflags" version = "2.13.1" @@ -141,77 +78,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "bon" -version = "3.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a602c73c7b0148ec6d12af6fd5cc7a46e2eacc8878271a999abac56eed12f561" -dependencies = [ - "bon-macros", - "rustversion", -] - -[[package]] -name = "bon-macros" -version = "3.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" -dependencies = [ - "darling", - "ident_case", - "prettyplease", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.119", -] - -[[package]] -name = "borrow-or-share" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" - -[[package]] -name = "borsh" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88b7ea17d208c4193f2c1e6de3c35fe71f98c96982d5ced308bdcc749ff6e1f" -dependencies = [ - "bytes", - "cfg_aliases", -] - -[[package]] -name = "brotli" -version = "8.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor", -] - -[[package]] -name = "brotli-decompressor" -version = "5.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", -] - -[[package]] -name = "bs58" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" -dependencies = [ - "tinyvec", -] - [[package]] name = "bumpalo" version = "3.20.3" @@ -219,19 +85,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] -name = "byteorder" -version = "1.5.0" +name = "byteorder-lite" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" [[package]] name = "bytes" version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" -dependencies = [ - "serde", -] + +[[package]] +name = "byteview" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d74937895e761d5984206f82ca8ff7725d0fd8021921011921894b41ab1b9f7" [[package]] name = "cbor4ii" @@ -258,23 +127,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" -[[package]] -name = "cfg_aliases" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" - -[[package]] -name = "chacha20" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" -dependencies = [ - "cfg-if", - "cpufeatures 0.3.0", - "rand_core 0.10.1", -] - [[package]] name = "chrono" version = "0.4.45" @@ -284,38 +136,10 @@ dependencies = [ "iana-time-zone", "js-sys", "num-traits", - "serde", "wasm-bindgen", "windows-link", ] -[[package]] -name = "ciborium" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" -dependencies = [ - "ciborium-io", - "ciborium-ll", - "serde", -] - -[[package]] -name = "ciborium-io" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" - -[[package]] -name = "ciborium-ll" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" -dependencies = [ - "ciborium-io", - "half", -] - [[package]] name = "cid" version = "0.11.3" @@ -330,37 +154,10 @@ dependencies = [ ] [[package]] -name = "cobs" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" -dependencies = [ - "thiserror 2.0.19", -] - -[[package]] -name = "compression-codecs" -version = "0.4.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" -dependencies = [ - "brotli", - "compression-core", - "flate2", - "memchr", -] - -[[package]] -name = "compression-core" -version = "0.4.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" - -[[package]] -name = "const-oid" -version = "0.9.6" +name = "compare" +version = "0.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +checksum = "ea0095f6103c2a8b44acd6fd15960c801dafebf02e21940360833e0673f48ba7" [[package]] name = "const-str" @@ -368,36 +165,6 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f421161cb492475f1661ddc9815a745a1c894592070661180fdec3d4872e9c3" -[[package]] -name = "cordyceps" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "688d7fbb8092b8de775ef2536f36c8c31f2bc4006ece2e8d8ad2d17d00ce0a2a" -dependencies = [ - "loom", - "tracing", -] - -[[package]] -name = "core-foundation" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -413,15 +180,6 @@ dependencies = [ "libc", ] -[[package]] -name = "cpufeatures" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" -dependencies = [ - "libc", -] - [[package]] name = "crc32fast" version = "1.5.0" @@ -431,12 +189,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "critical-section" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" - [[package]] name = "crossbeam-deque" version = "0.8.7" @@ -457,28 +209,20 @@ dependencies = [ ] [[package]] -name = "crossbeam-utils" -version = "0.8.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" - -[[package]] -name = "crunchy" -version = "0.2.4" +name = "crossbeam-skiplist" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] [[package]] -name = "crypto-bigint" -version = "0.5.5" +name = "crossbeam-utils" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" -dependencies = [ - "generic-array", - "rand_core 0.6.4", - "subtle", - "zeroize", -] +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" [[package]] name = "crypto-common" @@ -490,67 +234,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "curve25519-dalek" -version = "4.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "curve25519-dalek-derive", - "digest", - "fiat-crypto", - "rustc_version", - "subtle", - "zeroize", -] - -[[package]] -name = "curve25519-dalek-derive" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "darling" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" -dependencies = [ - "darling_core", - "darling_macro", -] - -[[package]] -name = "darling_core" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" -dependencies = [ - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn 2.0.119", -] - -[[package]] -name = "darling_macro" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" -dependencies = [ - "darling_core", - "quote", - "syn 2.0.119", -] - [[package]] name = "dashmap" version = "6.2.1" @@ -591,50 +274,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "der" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" -dependencies = [ - "const-oid", - "pem-rfc7468", - "zeroize", -] - -[[package]] -name = "deranged" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" - -[[package]] -name = "derive_more" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" -dependencies = [ - "derive_more-impl", -] - -[[package]] -name = "derive_more-impl" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", - "unicode-xid", -] - -[[package]] -name = "diatomic-waker" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab03c107fafeb3ee9f5925686dbb7a73bc76e3932abb0d2b365cb64b169cf04c" - [[package]] name = "digest" version = "0.10.7" @@ -642,9 +281,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", - "const-oid", "crypto-common", - "subtle", ] [[package]] @@ -659,125 +296,67 @@ dependencies = [ ] [[package]] -name = "ecdsa" -version = "0.16.9" +name = "either" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" -dependencies = [ - "der", - "digest", - "elliptic-curve", - "rfc6979", - "signature", - "spki", -] +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" [[package]] -name = "ed25519" -version = "2.2.3" +name = "enum_dispatch" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" dependencies = [ - "pkcs8", - "signature", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] -name = "ed25519-dalek" -version = "2.2.0" +name = "equivalent" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" -dependencies = [ - "curve25519-dalek", - "ed25519", - "rand_core 0.6.4", - "serde", - "sha2", - "subtle", - "zeroize", -] - -[[package]] -name = "either" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" - -[[package]] -name = "elliptic-curve" -version = "0.13.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" -dependencies = [ - "base16ct", - "crypto-bigint", - "digest", - "ff", - "generic-array", - "group", - "pem-rfc7468", - "pkcs8", - "rand_core 0.6.4", - "sec1", - "subtle", - "zeroize", -] - -[[package]] -name = "embedded-io" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" - -[[package]] -name = "embedded-io" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] -name = "encoding_rs" -version = "0.8.35" +name = "errno" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ - "cfg-if", + "libc", + "windows-sys 0.61.2", ] -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - [[package]] name = "fastrand" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" -[[package]] -name = "ff" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" -dependencies = [ - "rand_core 0.6.4", - "subtle", -] - -[[package]] -name = "fiat-crypto" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" - [[package]] name = "find-msvc-tools" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fjall" +version = "3.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "420a84699b8ccbb1ed573e38e88f4f23637b45beab6432066452f834be469c57" +dependencies = [ + "byteorder-lite", + "byteview", + "dashmap", + "flume", + "log", + "lsm-tree", + "tempfile", + "xxhash-rust", +] + [[package]] name = "flate2" version = "1.1.9" @@ -789,27 +368,19 @@ dependencies = [ ] [[package]] -name = "fluent-uri" -version = "0.4.1" +name = "flume" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" +checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be" dependencies = [ - "borrow-or-share", - "ref-cast", - "serde", + "spin", ] -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - [[package]] name = "foldhash" -version = "0.1.5" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" [[package]] name = "form_urlencoded" @@ -835,19 +406,6 @@ dependencies = [ "futures-util", ] -[[package]] -name = "futures-buffered" -version = "0.2.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4421cb78ee172b6b06080093479d3c50f058e7c81b7d577bbb8d118d551d4cd5" -dependencies = [ - "cordyceps", - "diatomic-waker", - "futures-core", - "pin-project-lite", - "spin 0.10.1", -] - [[package]] name = "futures-channel" version = "0.3.33" @@ -881,19 +439,6 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" -[[package]] -name = "futures-lite" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" -dependencies = [ - "fastrand", - "futures-core", - "futures-io", - "parking", - "pin-project-lite", -] - [[package]] name = "futures-macro" version = "0.3.33" @@ -934,21 +479,6 @@ dependencies = [ "slab", ] -[[package]] -name = "generator" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" -dependencies = [ - "cc", - "cfg-if", - "libc", - "log", - "rustversion", - "windows-link", - "windows-result", -] - [[package]] name = "generic-array" version = "0.14.9" @@ -957,7 +487,6 @@ checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" dependencies = [ "typenum", "version_check", - "zeroize", ] [[package]] @@ -967,24 +496,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", - "js-sys", "libc", "wasi", - "wasm-bindgen", -] - -[[package]] -name = "getrandom" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "r-efi 5.3.0", - "wasip2", - "wasm-bindgen", ] [[package]] @@ -994,61 +507,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", - "js-sys", "libc", - "r-efi 6.0.0", - "rand_core 0.10.1", - "wasm-bindgen", -] - -[[package]] -name = "group" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" -dependencies = [ - "ff", - "rand_core 0.6.4", - "subtle", -] - -[[package]] -name = "h2" -version = "0.4.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "half" -version = "2.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" -dependencies = [ - "cfg-if", - "crunchy", - "zerocopy", -] - -[[package]] -name = "hash32" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" -dependencies = [ - "byteorder", + "r-efi", ] [[package]] @@ -1059,157 +519,15 @@ checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" [[package]] name = "hashbrown" -version = "0.15.5" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "allocator-api2", "equivalent", "foldhash", ] -[[package]] -name = "hashbrown" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" - -[[package]] -name = "heapless" -version = "0.7.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" -dependencies = [ - "atomic-polyfill", - "hash32", - "rustc_version", - "serde", - "spin 0.9.9", - "stable_deref_trait", -] - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest", -] - -[[package]] -name = "http" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" -dependencies = [ - "bytes", - "itoa", -] - -[[package]] -name = "http-body" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" -dependencies = [ - "bytes", - "http", -] - -[[package]] -name = "http-body-util" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" -dependencies = [ - "bytes", - "futures-core", - "http", - "http-body", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" - -[[package]] -name = "hyper" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" -dependencies = [ - "atomic-waker", - "bytes", - "futures-channel", - "futures-core", - "h2", - "http", - "http-body", - "httparse", - "itoa", - "pin-project-lite", - "smallvec", - "tokio", - "want", -] - -[[package]] -name = "hyper-rustls" -version = "0.27.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" -dependencies = [ - "http", - "hyper", - "hyper-util", - "rustls", - "tokio", - "tokio-rustls", - "tower-service", - "webpki-roots 1.0.9", -] - -[[package]] -name = "hyper-util" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" -dependencies = [ - "base64", - "bytes", - "futures-channel", - "futures-util", - "http", - "http-body", - "hyper", - "ipnet", - "libc", - "percent-encoding", - "pin-project-lite", - "socket2", - "system-configuration", - "tokio", - "tower-service", - "tracing", - "windows-registry", -] - [[package]] name = "iana-time-zone" version = "0.1.65" @@ -1316,12 +634,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "ident_case" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" - [[package]] name = "idna" version = "1.1.0" @@ -1344,22 +656,12 @@ dependencies = [ ] [[package]] -name = "indexmap" -version = "2.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" -dependencies = [ - "equivalent", - "hashbrown 0.17.1", -] - -[[package]] -name = "inventory" -version = "0.3.24" +name = "interval-heap" +version = "0.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +checksum = "11274e5e8e89b8607cfedc2910b6626e998779b48a019151c7604d0adcb86ac6" dependencies = [ - "rustversion", + "compare", ] [[package]] @@ -1373,12 +675,6 @@ dependencies = [ "serde_bytes", ] -[[package]] -name = "ipnet" -version = "2.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" - [[package]] name = "iroh-car" version = "0.5.1" @@ -1402,177 +698,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] -name = "jacquard-api" -version = "0.12.1" +name = "js-sys" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5c803a3c097e3ef8aea63747b4fe3fc9e339cd18272dd0366b1d10dd90d5c3f" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" dependencies = [ - "jacquard-common", - "jacquard-derive", - "jacquard-lexicon", - "miette", - "serde", - "thiserror 2.0.19", + "cfg-if", + "futures-util", + "wasm-bindgen", ] [[package]] -name = "jacquard-common" +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linux-raw-sys" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fec8b6661e6fcfb4d8b9eb1132503be71bc3e3c6f6ff15fa5500ec62655f6da7" -dependencies = [ - "base64", - "bon", - "bytes", - "chrono", - "ciborium", - "ciborium-io", - "cid", - "ed25519-dalek", - "fluent-uri", - "futures", - "getrandom 0.2.17", - "getrandom 0.3.4", - "hashbrown 0.15.5", - "http", - "ipld-core", - "k256", - "maitake-sync", - "miette", - "multibase", - "multihash", - "n0-future", - "oxilangtag", - "p256", - "phf", - "postcard", - "rand 0.9.5", - "regex", - "regex-automata", - "regex-lite", - "reqwest", - "rustversion", - "serde", - "serde_bytes", - "serde_html_form", - "serde_ipld_dagcbor", - "serde_json", - "signature", - "smol_str", - "spin 0.10.1", - "thiserror 2.0.19", - "tokio", - "tokio-tungstenite-wasm", - "tokio-util", - "trait-variant", - "unicode-segmentation", -] - -[[package]] -name = "jacquard-derive" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e41b82a9a72c0d2b907a5e0734e0692615e717cda903502c6c034ae6f3ddddf7" -dependencies = [ - "heck", - "jacquard-lexicon", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "jacquard-lexicon" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "160049c269e3d7ec56f130736d25d61953fc6216da660c8b5e102f6b052dd3a1" -dependencies = [ - "cid", - "dashmap", - "heck", - "inventory", - "jacquard-common", - "miette", - "multihash", - "prettyplease", - "proc-macro2", - "quote", - "serde", - "serde_ipld_dagcbor", - "serde_json", - "serde_path_to_error", - "serde_repr", - "serde_with", - "sha2", - "syn 2.0.119", - "thiserror 2.0.19", - "unicode-segmentation", -] - -[[package]] -name = "jacquard-repo" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98986367bb78dadaa0f2f07196bab357786c0e3670d8311b350585b91f84d6eb" -dependencies = [ - "bytes", - "cid", - "ed25519-dalek", - "iroh-car", - "jacquard-api", - "jacquard-common", - "jacquard-derive", - "k256", - "miette", - "multihash", - "n0-future", - "p256", - "serde", - "serde_bytes", - "serde_ipld_dagcbor", - "sha2", - "smol_str", - "thiserror 2.0.19", - "tokio", - "trait-variant", -] - -[[package]] -name = "js-sys" -version = "0.3.103" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" -dependencies = [ - "cfg-if", - "futures-util", - "wasm-bindgen", -] - -[[package]] -name = "k256" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" -dependencies = [ - "cfg-if", - "ecdsa", - "elliptic-curve", - "once_cell", - "sha2", - "signature", -] - -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" - -[[package]] -name = "libc" -version = "0.2.189" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" @@ -1596,35 +742,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] -name = "loom" -version = "0.7.2" +name = "lsm-tree" +version = "3.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +checksum = "055a908d502129cf63bedae52f2db222e4436d2da32a69df9b84ac9fb9147761" dependencies = [ - "cfg-if", - "generator", - "scoped-tls", - "tracing", - "tracing-subscriber", -] - -[[package]] -name = "lru-slab" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" - -[[package]] -name = "maitake-sync" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6816ab14147f80234c675b80ed6dc4f440d8a1cefc158e766067aedb84c0bcd5" -dependencies = [ - "cordyceps", - "loom", - "mycelium-bitfield", - "pin-project", - "portable-atomic", + "byteorder-lite", + "byteview", + "crossbeam-skiplist", + "enum_dispatch", + "interval-heap", + "log", + "quick_cache", + "rustc-hash", + "self_cell", + "sfa", + "tempfile", + "varint-rs", + "xxhash-rust", ] [[package]] @@ -1638,49 +773,12 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "matchers" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" -dependencies = [ - "regex-automata", -] - [[package]] name = "memchr" version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" -[[package]] -name = "miette" -version = "7.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" -dependencies = [ - "cfg-if", - "miette-derive", - "unicode-width", -] - -[[package]] -name = "miette-derive" -version = "7.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - [[package]] name = "miniz_oxide" version = "0.8.9" @@ -1691,17 +789,6 @@ dependencies = [ "simd-adler32", ] -[[package]] -name = "mio" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" -dependencies = [ - "libc", - "wasi", - "windows-sys 0.61.2", -] - [[package]] name = "multibase" version = "0.9.3" @@ -1725,48 +812,6 @@ dependencies = [ "unsigned-varint 0.8.0", ] -[[package]] -name = "mycelium-bitfield" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24e0cc5e2c585acbd15c5ce911dff71e1f4d5313f43345873311c4f5efd741cc" - -[[package]] -name = "n0-future" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bb0e5d99e681ab3c938842b96fcb41bf8a7bb4bfdb11ccbd653a7e83e06c794" -dependencies = [ - "cfg_aliases", - "derive_more", - "futures-buffered", - "futures-lite", - "futures-util", - "js-sys", - "pin-project", - "send_wrapper", - "tokio", - "tokio-util", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-time", -] - -[[package]] -name = "nu-ansi-term" -version = "0.50.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "num-conv" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" - [[package]] name = "num-traits" version = "0.2.19" @@ -1782,39 +827,6 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" -[[package]] -name = "openssl-probe" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" - -[[package]] -name = "oxilangtag" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d3b4eb570abd4a1dcb062c31fd37b832264d9dc7292c3e69acfe926c87b063f" -dependencies = [ - "serde", -] - -[[package]] -name = "p256" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" -dependencies = [ - "ecdsa", - "elliptic-curve", - "primeorder", - "sha2", -] - -[[package]] -name = "parking" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" - [[package]] name = "parking_lot_core" version = "0.9.12" @@ -1825,352 +837,77 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-link", -] - -[[package]] -name = "parse-plays" -version = "0.1.0" -dependencies = [ - "bytes", - "chrono", - "cid", - "jacquard-common", - "jacquard-repo", - "rayon", - "serde", - "serde_ipld_dagcbor", - "serde_json", - "smol_str", - "tokio", - "ureq", -] - -[[package]] -name = "pem-rfc7468" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" -dependencies = [ - "base64ct", -] - -[[package]] -name = "percent-encoding" -version = "2.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" - -[[package]] -name = "phf" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" -dependencies = [ - "phf_macros", - "phf_shared", -] - -[[package]] -name = "phf_generator" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" -dependencies = [ - "phf_shared", - "rand 0.8.7", -] - -[[package]] -name = "phf_macros" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" -dependencies = [ - "phf_generator", - "phf_shared", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "phf_shared" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" -dependencies = [ - "siphasher", -] - -[[package]] -name = "pin-project" -version = "1.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" - -[[package]] -name = "pkcs8" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" -dependencies = [ - "der", - "spki", -] - -[[package]] -name = "portable-atomic" -version = "1.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" - -[[package]] -name = "postcard" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" -dependencies = [ - "cobs", - "embedded-io 0.4.0", - "embedded-io 0.6.1", - "heapless", - "serde", -] - -[[package]] -name = "potential_utf" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" -dependencies = [ - "zerovec", -] - -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" - -[[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] - -[[package]] -name = "prettyplease" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" -dependencies = [ - "proc-macro2", - "syn 2.0.119", -] - -[[package]] -name = "primeorder" -version = "0.13.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" -dependencies = [ - "elliptic-curve", -] - -[[package]] -name = "proc-macro2" -version = "1.0.107" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quinn" -version = "0.11.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" -dependencies = [ - "bytes", - "cfg_aliases", - "pin-project-lite", - "quinn-proto", - "quinn-udp", - "rustc-hash", - "rustls", - "socket2", - "thiserror 2.0.19", - "tokio", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-proto" -version = "0.11.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" -dependencies = [ - "bytes", - "getrandom 0.4.3", - "lru-slab", - "rand 0.10.2", - "rand_pcg", - "ring", - "rustc-hash", - "rustls", - "rustls-pki-types", - "slab", - "thiserror 2.0.19", - "tinyvec", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-udp" -version = "0.5.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" -dependencies = [ - "cfg_aliases", - "libc", - "once_cell", - "socket2", - "tracing", - "windows-sys 0.61.2", -] - -[[package]] -name = "quote" -version = "1.0.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - -[[package]] -name = "r-efi" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" - -[[package]] -name = "rand" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" -dependencies = [ - "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", + "windows-link", ] [[package]] -name = "rand" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +name = "parse-plays" +version = "0.1.0" dependencies = [ - "rand_chacha 0.9.0", - "rand_core 0.9.5", + "chrono", + "rayon", + "repo-stream", + "serde", + "serde_ipld_dagcbor", + "serde_json", + "tokio", + "ureq", ] [[package]] -name = "rand" -version = "0.10.2" +name = "percent-encoding" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" -dependencies = [ - "chacha20", - "getrandom 0.4.3", - "rand_core 0.10.1", -] +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] -name = "rand_chacha" -version = "0.3.1" +name = "pin-project-lite" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", -] +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] -name = "rand_chacha" -version = "0.9.0" +name = "potential_utf" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ - "ppv-lite86", - "rand_core 0.9.5", + "zerovec", ] [[package]] -name = "rand_core" -version = "0.6.4" +name = "proc-macro2" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ - "getrandom 0.2.17", + "unicode-ident", ] [[package]] -name = "rand_core" -version = "0.9.5" +name = "quick_cache" +version = "0.6.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +checksum = "b9c6658afe513a3b484e3abfdaa0d03ef3c0bbf017542c178dd55f94eb3051f9" dependencies = [ - "getrandom 0.3.4", + "equivalent", + "hashbrown 0.16.1", ] [[package]] -name = "rand_core" -version = "0.10.1" +name = "quote" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] [[package]] -name = "rand_pcg" -version = "0.10.2" +name = "r-efi" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" -dependencies = [ - "rand_core 0.10.1", -] +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rayon" @@ -2202,112 +939,22 @@ dependencies = [ ] [[package]] -name = "ref-cast" -version = "1.0.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d" -dependencies = [ - "ref-cast-impl", -] - -[[package]] -name = "ref-cast-impl" -version = "1.0.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "regex" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-lite" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" - -[[package]] -name = "regex-syntax" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" - -[[package]] -name = "reqwest" -version = "0.12.28" +name = "repo-stream" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +checksum = "6c95af4a1465ac3a5a5aa4921b60f8761c1e126deb53a5b1a2e40abd0ec6ae1f" dependencies = [ - "base64", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-util", - "js-sys", + "cid", + "fjall", + "hashbrown 0.16.1", + "iroh-car", "log", - "mime", - "percent-encoding", - "pin-project-lite", - "quinn", - "rustls", - "rustls-pki-types", "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", + "serde_bytes", + "serde_ipld_dagcbor", + "sha2", + "thiserror 2.0.19", "tokio", - "tokio-rustls", - "tokio-util", - "tower", - "tower-http", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-streams", - "web-sys", - "webpki-roots 1.0.9", -] - -[[package]] -name = "rfc6979" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" -dependencies = [ - "hmac", - "subtle", ] [[package]] @@ -2331,12 +978,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] -name = "rustc_version" -version = "0.4.1" +name = "rustix" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "semver", + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", ] [[package]] @@ -2354,25 +1005,12 @@ dependencies = [ "zeroize", ] -[[package]] -name = "rustls-native-certs" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" -dependencies = [ - "openssl-probe", - "rustls-pki-types", - "schannel", - "security-framework", -] - [[package]] name = "rustls-pki-types" version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ - "web-time", "zeroize", ] @@ -2393,27 +1031,6 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" -[[package]] -name = "ryu" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" - -[[package]] -name = "schannel" -version = "0.1.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "scoped-tls" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" - [[package]] name = "scopeguard" version = "1.2.0" @@ -2421,53 +1038,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] -name = "sec1" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" -dependencies = [ - "base16ct", - "der", - "generic-array", - "pkcs8", - "subtle", - "zeroize", -] - -[[package]] -name = "security-framework" -version = "3.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" -dependencies = [ - "bitflags", - "core-foundation 0.10.1", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "semver" -version = "1.0.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" - -[[package]] -name = "send_wrapper" -version = "0.6.0" +name = "self_cell" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" +checksum = "2ab42ca02749e120097e328d91d415325bdf43b1c72c4c8badf37375fe40a813" [[package]] name = "serde" @@ -2509,18 +1083,6 @@ dependencies = [ "syn 3.0.3", ] -[[package]] -name = "serde_html_form" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acf96b1d9364968fce46ebb548f1c0e1d7eceae27bdff73865d42e6c7369d94" -dependencies = [ - "form_urlencoded", - "indexmap", - "itoa", - "serde_core", -] - [[package]] name = "serde_ipld_dagcbor" version = "0.6.4" @@ -2547,76 +1109,14 @@ dependencies = [ ] [[package]] -name = "serde_path_to_error" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" -dependencies = [ - "itoa", - "serde", - "serde_core", -] - -[[package]] -name = "serde_repr" -version = "0.1.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "serde_with" -version = "3.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" -dependencies = [ - "base64", - "bs58", - "chrono", - "hex", - "serde_core", - "serde_json", - "serde_with_macros", - "time", -] - -[[package]] -name = "serde_with_macros" -version = "3.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" -dependencies = [ - "darling", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "sha1" -version = "0.10.7" +name = "sfa" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +checksum = "a1296838937cab56cd6c4eeeb8718ec777383700c33f060e2869867bd01d1175" dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest", + "byteorder-lite", + "log", + "xxhash-rust", ] [[package]] @@ -2626,47 +1126,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures 0.2.17", + "cpufeatures", "digest", ] -[[package]] -name = "sharded-slab" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" -dependencies = [ - "lazy_static", -] - [[package]] name = "shlex" version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" -[[package]] -name = "signature" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" -dependencies = [ - "digest", - "rand_core 0.6.4", -] - [[package]] name = "simd-adler32" version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" -[[package]] -name = "siphasher" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" - [[package]] name = "slab" version = "0.4.12" @@ -2679,26 +1154,6 @@ version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" -[[package]] -name = "smol_str" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4aaa7368fcf4852a4c2dd92df0cace6a71f2091ca0a23391ce7f3a31833f1523" -dependencies = [ - "borsh", - "serde_core", -] - -[[package]] -name = "socket2" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" -dependencies = [ - "libc", - "windows-sys 0.61.2", -] - [[package]] name = "spin" version = "0.9.9" @@ -2708,33 +1163,11 @@ dependencies = [ "lock_api", ] -[[package]] -name = "spin" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" - -[[package]] -name = "spki" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" -dependencies = [ - "base64ct", - "der", -] - [[package]] name = "stable_deref_trait" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" - -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "subtle" @@ -2764,15 +1197,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "sync_wrapper" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" -dependencies = [ - "futures-core", -] - [[package]] name = "synstructure" version = "0.13.2" @@ -2785,24 +1209,16 @@ dependencies = [ ] [[package]] -name = "system-configuration" -version = "0.7.0" +name = "tempfile" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ - "bitflags", - "core-foundation 0.9.4", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" -dependencies = [ - "core-foundation-sys", - "libc", + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", ] [[package]] @@ -2845,34 +1261,6 @@ dependencies = [ "syn 3.0.3", ] -[[package]] -name = "thread_local" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "time" -version = "0.3.54" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244" -dependencies = [ - "deranged", - "num-conv", - "powerfmt", - "serde_core", - "time-core", -] - -[[package]] -name = "time-core" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" - [[package]] name = "tinystr" version = "0.8.3" @@ -2883,21 +1271,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "tinyvec" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - [[package]] name = "tokio" version = "1.53.1" @@ -2905,12 +1278,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", - "libc", - "mio", "pin-project-lite", - "socket2", "tokio-macros", - "windows-sys 0.61.2", ] [[package]] @@ -2924,214 +1293,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "tokio-rustls" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" -dependencies = [ - "rustls", - "tokio", -] - -[[package]] -name = "tokio-tungstenite" -version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" -dependencies = [ - "futures-util", - "log", - "rustls", - "rustls-native-certs", - "rustls-pki-types", - "tokio", - "tokio-rustls", - "tungstenite", -] - -[[package]] -name = "tokio-tungstenite-wasm" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e21a5c399399c3db9f08d8297ac12b500e86bca82e930253fdc62eaf9c0de6ae" -dependencies = [ - "futures-channel", - "futures-util", - "http", - "httparse", - "js-sys", - "rustls", - "thiserror 1.0.69", - "tokio", - "tokio-tungstenite", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "tokio-util" -version = "0.7.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "futures-util", - "libc", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tower" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" -dependencies = [ - "futures-core", - "futures-util", - "pin-project-lite", - "sync_wrapper", - "tokio", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-http" -version = "0.6.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" -dependencies = [ - "async-compression", - "bitflags", - "bytes", - "futures-core", - "futures-util", - "http", - "http-body", - "http-body-util", - "pin-project-lite", - "tokio", - "tokio-util", - "tower", - "tower-layer", - "tower-service", - "url", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - -[[package]] -name = "tower-service" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" - -[[package]] -name = "tracing" -version = "0.1.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" -dependencies = [ - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "tracing-core" -version = "0.1.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" -dependencies = [ - "once_cell", - "valuable", -] - -[[package]] -name = "tracing-log" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" -dependencies = [ - "log", - "once_cell", - "tracing-core", -] - -[[package]] -name = "tracing-subscriber" -version = "0.3.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" -dependencies = [ - "matchers", - "nu-ansi-term", - "once_cell", - "regex-automata", - "sharded-slab", - "smallvec", - "thread_local", - "tracing", - "tracing-core", - "tracing-log", -] - -[[package]] -name = "trait-variant" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b19a4867a870f6edc4c283f2b455804b1879c0baf0e642f26b03ed8ee262d9d3" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - -[[package]] -name = "tungstenite" -version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" -dependencies = [ - "byteorder", - "bytes", - "data-encoding", - "http", - "httparse", - "log", - "rand 0.8.7", - "rustls", - "rustls-pki-types", - "sha1", - "thiserror 1.0.69", - "utf-8", -] - [[package]] name = "typenum" version = "1.20.1" @@ -3144,24 +1305,6 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "unicode-segmentation" -version = "1.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" - -[[package]] -name = "unicode-width" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" - -[[package]] -name = "unicode-xid" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" - [[package]] name = "unsigned-varint" version = "0.7.2" @@ -3208,12 +1351,6 @@ dependencies = [ "serde", ] -[[package]] -name = "utf-8" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" - [[package]] name = "utf8_iter" version = "1.0.4" @@ -3221,10 +1358,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] -name = "valuable" -version = "0.1.1" +name = "varint-rs" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +checksum = "bfa6c38708f6257f1ec2ca7e5a11f9bbf58a27d7060078b6b333624968183d96" [[package]] name = "version_check" @@ -3232,30 +1369,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" -[[package]] -name = "wasip2" -version = "1.0.4+wasi-0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" -dependencies = [ - "wit-bindgen", -] - [[package]] name = "wasm-bindgen" version = "0.2.126" @@ -3269,16 +1388,6 @@ dependencies = [ "wasm-bindgen-shared", ] -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.76" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - [[package]] name = "wasm-bindgen-macro" version = "0.2.126" @@ -3311,39 +1420,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "wasm-streams" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - -[[package]] -name = "web-sys" -version = "0.3.103" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "web-time" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - [[package]] name = "webpki-roots" version = "0.26.11" @@ -3403,17 +1479,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-registry" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" -dependencies = [ - "windows-link", - "windows-result", - "windows-strings", -] - [[package]] name = "windows-result" version = "0.4.1" @@ -3514,18 +1579,18 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "wit-bindgen" -version = "0.57.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" - [[package]] name = "writeable" version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "xxhash-rust" +version = "0.8.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" + [[package]] name = "yoke" version = "0.8.3" @@ -3549,26 +1614,6 @@ dependencies = [ "synstructure", ] -[[package]] -name = "zerocopy" -version = "0.8.55" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.55" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "zerofrom" version = "0.1.8" diff --git a/tools/parse-plays/Cargo.toml b/tools/parse-plays/Cargo.toml index e2f6247..a8b22eb 100644 --- a/tools/parse-plays/Cargo.toml +++ b/tools/parse-plays/Cargo.toml @@ -4,15 +4,11 @@ version = "0.1.0" edition = "2024" [dependencies] -bytes = "1" chrono = "0.4" -cid = { version = "0.11", features = ["serde"] } -jacquard-common = "0.12" -jacquard-repo = "0.12" rayon = "1" +repo-stream = "0.4" serde = { version = "1", features = ["derive"] } serde_ipld_dagcbor = "0.6" serde_json = "1" -smol_str = { version = "0.3", features = ["serde"] } tokio = { version = "1", features = ["fs", "macros", "rt", "rt-multi-thread"] } ureq = "2" diff --git a/tools/parse-plays/src/cache.rs b/tools/parse-plays/src/cache.rs new file mode 100644 index 0000000..9b71e53 --- /dev/null +++ b/tools/parse-plays/src/cache.rs @@ -0,0 +1,213 @@ +//! Typed lookup cache for the resolution pipeline. +//! +//! Every cacheable lookup is a `CacheKey` variant, so the five +//! namespaces (album covers, artist images, album/artist/track page +//! links) can never collide — the old string keys distinguished them +//! by counting separator characters. Values are either a found URL or +//! a negative entry (a lookup that found nothing) with a TTL, so a +//! permanent miss isn't re-queried on every refresh but a stale one +//! heals itself after `MISS_TTL`. +//! +//! The cache is read-only during resolution (phases hold `&Cache`), +//! and every write is staged through `merge` then committed once by +//! the caller — no shared mutable state mid-run. + +use crate::model::{AlbumRef, ArtistKey, Href, TrackRef}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +/// How long a negative entry stays authoritative before it is +/// re-queried. 14 days: misses are re-checked a couple of times a +/// month so new MusicBrainz/Cover Art Archive/Wikidata entries get +/// picked up, without paying for them on every refresh. +pub(crate) const MISS_TTL: Duration = Duration::from_secs(14 * 24 * 3600); + +/// Every cacheable lookup, distinguished by kind — collisions between +/// the five namespaces are impossible by construction. +#[derive(Clone, PartialEq, Eq, Hash, Debug, Serialize, Deserialize)] +pub enum CacheKey { + AlbumCover(AlbumRef), + ArtistImage(ArtistKey), + AlbumUrl(AlbumRef), + ArtistUrl(ArtistKey), + TrackUrl(TrackRef), +} + +#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] +pub enum CacheValue { + Found(Href), + /// A lookup that found nothing, timestamped (epoch seconds) so it + /// expires. + Missing { + since: u64, + }, +} + +impl CacheValue { + /// A `Missing` entry recorded within `MISS_TTL` is authoritative — + /// skip it. `Found` and stale entries are not fresh misses. + pub fn is_fresh_miss(&self, now: SystemTime) -> bool { + let CacheValue::Missing { since } = self else { + return false; + }; + let now = now + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + now.saturating_sub(*since) < MISS_TTL.as_secs() + } +} + +/// Owned, read-only during resolution, written exactly once. No Mutex. +#[derive(Default)] +pub struct Cache { + map: HashMap, +} + +impl Cache { + /// Load from disk; a missing, corrupt, or old-format file is an + /// empty cache (which re-resolves once), never a failure. + pub fn load(path: Option<&str>) -> Cache { + let Some(path) = path else { + return Cache::default(); + }; + let map: Option> = std::fs::read_to_string(path) + .ok() + .and_then(|s| serde_json::from_str(&s).ok()); + match map { + Some(entries) => { + let mut cache = Cache::default(); + cache.merge(entries); + cache + } + None => Cache::default(), + } + } + + pub fn lookup(&self, key: &CacheKey) -> Option<&CacheValue> { + self.map.get(key) + } + + /// Stage a batch of writes. Later writes win over earlier ones in + /// the same batch, so a found URL upgrades a prior miss. + pub fn merge(&mut self, writes: impl IntoIterator) { + for (key, value) in writes { + self.map.insert(key, value); + } + } + + /// Persist atomically (tmp + rename) so a crash mid-write never + /// corrupts the previous state. Stored as a list of pairs because + /// serde_json map keys must be strings and `CacheKey` is an enum. + pub fn save_atomically(&self, path: &str) -> Result<(), String> { + let entries: Vec<(CacheKey, CacheValue)> = self + .map + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + let serialized = serde_json::to_string_pretty(&entries) + .map_err(|e| format!("failed to serialize cache: {e}"))?; + let tmp = format!("{path}.tmp"); + std::fs::write(&tmp, serialized).map_err(|e| format!("failed to write {tmp}: {e}"))?; + std::fs::rename(&tmp, path).map_err(|e| format!("failed to rename {tmp}: {e}")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::{AlbumKey, ArtistKey, TrackKey}; + use std::str::FromStr; + + fn t0() -> SystemTime { + // Far enough past the epoch that `now - MISS_TTL` stays positive. + SystemTime::UNIX_EPOCH + Duration::from_secs(1_800_000_000) + } + + fn album_key() -> AlbumRef { + AlbumRef { + artist: ArtistKey::from("Artist"), + album: AlbumKey::from("Album"), + } + } + + #[test] + fn miss_ttl_boundary_fresh_and_stale() { + let now = t0(); + let fresh = CacheValue::Missing { + since: now.duration_since(UNIX_EPOCH).unwrap().as_secs() - MISS_TTL.as_secs() + 1, + }; + assert!(fresh.is_fresh_miss(now)); + let stale = CacheValue::Missing { + since: now.duration_since(UNIX_EPOCH).unwrap().as_secs() - MISS_TTL.as_secs(), + }; + assert!(!stale.is_fresh_miss(now)); + // Found is never a miss, regardless of timestamp. + let found = CacheValue::Found(Href::from_str("https://example.com/x.jpg").unwrap()); + assert!(!found.is_fresh_miss(now)); + } + + #[test] + fn merge_is_idempotent_and_upgrades() { + let mut cache = Cache::default(); + let key = CacheKey::AlbumCover(album_key()); + let url = Href::from_str("https://example.com/cover.jpg").unwrap(); + let miss = CacheValue::Missing { since: 0 }; + + cache.merge(vec![(key.clone(), miss.clone())]); + assert_eq!(cache.lookup(&key), Some(&miss)); + + // Merging the same batch again doesn't change the map. + cache.merge(vec![(key.clone(), miss.clone())]); + assert_eq!(cache.lookup(&key), Some(&miss)); + + // Found upgrades a prior miss. + cache.merge(vec![(key.clone(), CacheValue::Found(url.clone()))]); + assert_eq!(cache.lookup(&key), Some(&CacheValue::Found(url))); + } + + #[test] + fn persistence_roundtrip() { + let dir = std::env::temp_dir().join(format!("parse-plays-cache-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("cache.json"); + let path = path.to_str().unwrap().to_string(); + + let mut cache = Cache::default(); + cache.merge(vec![ + ( + CacheKey::AlbumCover(album_key()), + CacheValue::Found(Href::from_str("https://example.com/cover.jpg").unwrap()), + ), + ( + CacheKey::ArtistImage(ArtistKey::from("Mac Miller")), + CacheValue::Missing { since: 42 }, + ), + ( + CacheKey::TrackUrl(TrackRef { + artist: ArtistKey::from("a"), + track: TrackKey::from("b"), + }), + CacheValue::Missing { since: 7 }, + ), + ]); + cache.save_atomically(&path).unwrap(); + + let loaded = Cache::load(Some(&path)); + assert_eq!( + loaded.lookup(&CacheKey::AlbumCover(album_key())), + cache.lookup(&CacheKey::AlbumCover(album_key())) + ); + + // Corrupt file → empty cache, never a failure. + std::fs::write(&path, "{not json").unwrap(); + assert!( + Cache::load(Some(&path)) + .lookup(&CacheKey::AlbumCover(album_key())) + .is_none() + ); + + std::fs::remove_dir_all(&dir).ok(); + } +} diff --git a/tools/parse-plays/src/car.rs b/tools/parse-plays/src/car.rs index 9c7342b..b642063 100644 --- a/tools/parse-plays/src/car.rs +++ b/tools/parse-plays/src/car.rs @@ -1,51 +1,111 @@ -//! CAR file → play records (jacquard I/O). +//! CAR file → play records (repo-stream I/O). -use jacquard_common::IntoStatic; -use jacquard_common::types::string::Nsid; -use jacquard_repo::car::reader::{read_car, read_car_header}; -use jacquard_repo::storage::MemoryBlockStore; -use jacquard_repo::{BlockStore, Repository}; -use smol_str::SmolStr; -use std::sync::Arc; +use repo_stream::{DiskBuilder, Driver, DriverBuilder}; +use serde_json::Value; const PLAY_COLLECTION: &str = "fm.teal.alpha.feed.play"; +/// Cap on the buffered (play-filtered) blocks before repo-stream +/// spills to disk. The current repo needs ~75 MiB; 256 MiB leaves +/// room to grow while staying in the in-memory fast path. +const MEM_LIMIT_MB: usize = 256; + +/// Whether a decoded record block is a play record. Extracted from the +/// block processor so the filter is testable without a CAR fixture. +pub(crate) fn is_play_record(record: &Value) -> bool { + record.get("$type").and_then(Value::as_str) == Some(PLAY_COLLECTION) +} + /// Read every play record from a repository CAR file, decoded as /// JSON values. Records that fail to decode are dropped. pub async fn read_plays(car_path: &str) -> Result, String> { - let store = Arc::new(MemoryBlockStore::new()); - store - .put_many( - read_car(car_path) - .await - .map_err(|e| format!("failed to read CAR file: {e}"))?, - ) - .await - .map_err(|e| format!("failed to load blocks into store: {e}"))?; + let reader = tokio::io::BufReader::with_capacity( + 1 << 20, + tokio::fs::File::open(car_path) + .await + .map_err(|e| format!("failed to open {car_path}: {e}"))?, + ); - let roots = read_car_header(car_path) + // Stream the CAR through repo-stream. The block processor runs on + // every block as it is read; keeping only play records means the + // driver buffers a fraction of the raw CAR and the MST walk emits + // just the plays, in rkey order. + let driver = DriverBuilder::new() + .with_mem_limit_mb(MEM_LIMIT_MB) + .with_block_processor(|block| { + let keep = serde_ipld_dagcbor::from_slice::(block.as_slice()) + .is_ok_and(|record| is_play_record(&record)); + if keep { block } else { Vec::new() } + }) + .load_car(reader) .await - .map_err(|e| format!("failed to read CAR header: {e}"))?; - - let collection = Nsid::new(PLAY_COLLECTION) - .expect("invalid collection NSID") - .into_static(); - - let entries = Repository::::from_commit( - store.clone(), - roots.first().ok_or("CAR file has no roots")?, - ) - .await - .map_err(|e| format!("failed to load repository from commit: {e}"))? - .list_collection_data(&collection) - .await - .map_err(|e| format!("failed to list play records: {e}"))?; + .map_err(|e| format!("failed to read CAR file: {e}"))?; + // The current CAR fits in memory; keep the disk spill path so the + // pipeline degrades gracefully if the repo outgrows MEM_LIMIT_MB. let mut plays = Vec::new(); - for (_, _, data) in &entries { - if let Ok(record) = serde_ipld_dagcbor::from_slice::(data) { - plays.push(record); + match driver { + Driver::Memory(_commit, mut driver) => { + while let Some(chunk) = driver + .next_chunk(256) + .await + .map_err(|e| format!("failed to walk MST: {e}"))? + { + collect_plays(chunk, &mut plays); + } + } + Driver::Disk(need_disk) => { + let store = DiskBuilder::new() + .open(std::env::temp_dir().join(format!("parse-plays-{}.db", std::process::id()))) + .await + .map_err(|e| format!("failed to open disk store: {e}"))?; + let (_commit, mut driver) = need_disk + .finish_loading(store) + .await + .map_err(|e| format!("failed to finish loading CAR: {e}"))?; + while let Some(chunk) = driver + .next_chunk(256) + .await + .map_err(|e| format!("failed to walk MST: {e}"))? + { + collect_plays(chunk, &mut plays); + } } } Ok(plays) } + +/// Push the play records of one emitted chunk; blocks filtered out by +/// the processor (empty data) and records that fail to decode are +/// dropped. +fn collect_plays(chunk: Vec, plays: &mut Vec) { + for output in chunk { + if output.data.is_empty() { + continue; // filtered-out block + } + if let Ok(record) = serde_ipld_dagcbor::from_slice::(output.data.as_slice()) { + plays.push(record); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn is_play_record_filters_by_type() { + assert!(is_play_record(&json!({ + "$type": "fm.teal.alpha.feed.play", + "trackName": "Flute", + }))); + assert!(!is_play_record(&json!({ + "$type": "app.bsky.feed.post", + "text": "hi", + }))); + assert!(!is_play_record(&json!({ "trackName": "no type" }))); + assert!(!is_play_record(&json!([1, 2, 3]))); + assert!(!is_play_record(&json!("string"))); + } +} diff --git a/tools/parse-plays/src/covers.rs b/tools/parse-plays/src/covers.rs deleted file mode 100644 index 7e6e267..0000000 --- a/tools/parse-plays/src/covers.rs +++ /dev/null @@ -1,653 +0,0 @@ -//! Cover art resolution: the impure boundary between the pure stats -//! core and MusicBrainz / Cover Art Archive / Wikidata. -//! -//! All five `resolve_*` phases share one `Cache` (a mutex-guarded map, -//! loaded once and persisted once by the caller) so they can safely -//! run concurrently. Each phase is cache-first: known keys are served -//! instantly and only missing entries hit the network — parallel -//! workers with per-endpoint rate limiting, and every request goes -//! through `net::retry` (jittered backoff, `Retry-After` honor, and -//! retries on transport errors, not just HTTP status codes). - -use crate::{net, stats}; -use rayon::prelude::*; -use std::collections::HashMap; -use std::sync::Mutex; -use std::time::{Duration, Instant}; - -pub(crate) const USER_AGENT: &str = "karitham-blog/0.1.0 (https://karitham.dev)"; - -/// The shared lookup cache. One entry per resolved key (see the -/// `*_cache_key` helpers); the caller loads it once and persists it -/// once after all phases complete. -pub(crate) type Cache = Mutex>; - -/// Global rate limiter shared by workers: at most `rate_per_sec` -/// acquisitions per second across all threads. -pub(crate) struct Limiter { - min_interval: Duration, - last: Mutex, -} - -impl Limiter { - pub(crate) fn new(rate_per_sec: f64) -> Self { - Self { - min_interval: Duration::from_secs_f64(1.0 / rate_per_sec), - last: Mutex::new(Instant::now()), - } - } - - pub(crate) fn acquire(&self) { - let mut last = self.last.lock().unwrap(); - let since = Instant::now().duration_since(*last); - if since < self.min_interval { - std::thread::sleep(self.min_interval - since); - } - *last = Instant::now(); - } -} - -pub(crate) fn is_retryable(code: u16) -> bool { - code == 429 || code == 500 || code == 503 -} - -static RETRY_POLICY: net::RetryPolicy = net::RetryPolicy { - max_attempts: 5, - base_delay: Duration::from_millis(500), - max_delay: Duration::from_secs(8), -}; - -/// Load the cache from disk; a missing or corrupt file is an empty -/// cache, never a failure. -pub(crate) fn load_cache(path: &str) -> HashMap { - std::fs::read_to_string(path) - .ok() - .and_then(|s| serde_json::from_str(&s).ok()) - .unwrap_or_default() -} - -/// Persist the cache atomically (tmp + rename) so a crash mid-write -/// never corrupts the previous state. -pub(crate) fn save_cache_atomically(path: &str, cache: &HashMap) { - let Ok(serialized) = serde_json::to_string_pretty(cache) else { - return; - }; - let tmp = format!("{path}.tmp"); - if std::fs::write(&tmp, serialized).is_ok() { - let _ = std::fs::rename(&tmp, path); - } -} - -fn clean_query(s: &str) -> String { - s.replace(['"', '\\'], "") -} - -/// GET a JSON API with the given query params, retried through -/// `net::retry`. A 200 that fails to parse is a permanent miss — the -/// endpoint answered, the shape just isn't what we expected. -fn get_json(agent: &ureq::Agent, url: &str, params: &[(&str, &str)]) -> Option { - net::retry(&RETRY_POLICY, || { - let mut req = agent.get(url); - for (key, value) in params { - req = req.query(key, value); - } - match req.call() { - Ok(resp) if resp.status() == 200 => match resp.into_string() { - Ok(body) => match serde_json::from_str(&body) { - Ok(value) => net::Attempt::Done(value), - Err(_) => net::Attempt::Stop, - }, - Err(_) => net::Attempt::Stop, - }, - Ok(_) => net::Attempt::Stop, - Err(ureq::Error::Status(code, resp)) if is_retryable(code) => { - net::Attempt::Again(net::retry_after(&resp)) - } - // Transport-level failures (timeouts, resets, DNS) are - // transient — retry rather than dropping the link for this - // whole build. - Err(_) => net::Attempt::Again(None), - } - }) -} - -/// MusicBrainz release search → best-guess release MBID. Retries on -/// rate-limit/5xx with jittered backoff, which lets us run a bit -/// hotter than the nominal 1 req/s and have MB push back politely. -fn musicbrainz_release(agent: &ureq::Agent, artist: &str, album: &str) -> Option { - let query = format!( - "release:\"{}\" AND artist:\"{}\"", - clean_query(album), - clean_query(artist) - ); - get_json( - agent, - "https://musicbrainz.org/ws/2/release", - &[("query", &query), ("fmt", "json"), ("limit", "1")], - ) - .and_then(|body| { - body["releases"] - .as_array()? - .first()? - .get("id")? - .as_str() - .map(str::to_string) - }) -} - -/// Cover Art Archive front image for a release, 500px. A non-2xx -/// response (e.g. 404 — no front art) is a permanent miss. -fn coverart_url(agent: &ureq::Agent, mbid: &str) -> Option { - let url = format!("https://coverartarchive.org/release/{mbid}/front-500"); - net::retry(&RETRY_POLICY, || match agent.get(&url).call() { - Ok(resp) if resp.status() == 200 => net::Attempt::Done(url.clone()), - Ok(_) => net::Attempt::Stop, - Err(ureq::Error::Status(code, resp)) if is_retryable(code) => { - net::Attempt::Again(net::retry_after(&resp)) - } - Err(_) => net::Attempt::Again(None), - }) -} - -fn cache_key(artist: &str, album: &str) -> String { - format!( - "{}\u{1f}{}", - stats::normalize(artist), - stats::normalize(album) - ) -} - -/// Cache key for a real artist image. The leading separator can never -/// collide with an album key (`artist\x1falbum`, non-empty artist side). -fn artist_cache_key(name: &str) -> String { - format!("\u{1f}{}", stats::normalize(name)) -} - -/// Cache key for an artist's MusicBrainz page URL. Two leading -/// separators — distinct from artist images (one) and album keys -/// (none). `normalize` drops non-alphanumerics, so separators can -/// never appear inside a name. -fn artist_url_cache_key(name: &str) -> String { - format!("\u{1f}\u{1f}{}", stats::normalize(name)) -} - -/// Cache key for an album's MusicBrainz release URL: the album cover -/// key plus a suffix. Album keys contain no separator, so the suffix -/// is unambiguous. -fn album_url_cache_key(artist: &str, album: &str) -> String { - format!("{}\u{1f}url", cache_key(artist, album)) -} - -/// Cache key for a track's MusicBrainz recording URL. Three leading -/// separators; includes the artist so same-named tracks don't collide. -fn track_url_cache_key(artist: &str, track: &str) -> String { - format!( - "\u{1f}\u{1f}\u{1f}{}\u{1f}{}", - stats::normalize(artist), - stats::normalize(track) - ) -} - -/// Resolve cover URLs for the given (artist, album, optional MBID) -/// triples. Known MBIDs skip the MusicBrainz search phase, but when a -/// provided MBID has no Cover Art Archive image we fall back to a name -/// search — piper/lazuli MBIDs sometimes point at release variants -/// without art while the canonical release has it. -pub fn resolve( - pairs: &[(String, String, Option)], - cache: &Cache, -) -> HashMap<(String, String), String> { - let mut out: HashMap<(String, String), String> = HashMap::new(); - let missing: Vec<(String, String, Option)> = { - let cache = cache.lock().unwrap(); - pairs - .iter() - .filter_map( - |(artist, album, mbid)| match cache.get(&cache_key(artist, album)) { - Some(url) => { - out.insert( - (stats::normalize(artist), stats::normalize(album)), - url.clone(), - ); - None - } - None => Some((artist.clone(), album.clone(), mbid.clone())), - }, - ) - .collect() - }; - - if missing.is_empty() { - return out; - } - - let agent = ureq::AgentBuilder::new() - .user_agent(USER_AGENT) - .timeout(Duration::from_secs(15)) - .build(); - - // Phase 1: MusicBrainz searches for pairs without a known MBID, - // ~3 req/s across all workers. - let mb_limiter = &Limiter::new(3.0); - let searched: Vec<(String, String, String)> = missing - .par_iter() - .filter(|(_, _, mbid)| mbid.is_none()) - .filter_map(|(artist, album, _)| { - mb_limiter.acquire(); - musicbrainz_release(&agent, artist, album) - .map(|mbid| (artist.clone(), album.clone(), mbid)) - }) - .collect(); - - // `provided` records whether the MBID came from the play record — - // those get a name-search fallback if their art is missing. - let mut with_mbid: Vec<(String, String, String, bool)> = missing - .iter() - .filter_map(|(artist, album, mbid)| { - mbid.clone() - .map(|m| (artist.clone(), album.clone(), m, true)) - }) - .collect(); - with_mbid.extend( - searched - .into_iter() - .map(|(artist, album, mbid)| (artist, album, mbid, false)), - ); - - // Phase 2: Cover Art Archive fetches, ~6 req/s across all workers. - let caa_limiter = &Limiter::new(6.0); - let caa_outcomes: Vec<(String, String, bool, Option)> = with_mbid - .par_iter() - .map(|(artist, album, mbid, provided)| { - caa_limiter.acquire(); - ( - artist.clone(), - album.clone(), - *provided, - coverart_url(&agent, mbid), - ) - }) - .collect(); - - { - let mut cache = cache.lock().unwrap(); - for (artist, album, _, url) in &caa_outcomes { - let Some(url) = url else { continue }; - cache.insert(cache_key(artist, album), url.clone()); - // Keys are normalized so the pure `cover` lookup in main - // (which normalizes both sides) finds them regardless of case. - out.insert( - (stats::normalize(artist), stats::normalize(album)), - url.clone(), - ); - } - } - - // Fallback: provided MBIDs with no art get a name search — the - // search usually lands on the canonical release that has art. - let fallback: Vec<(String, String)> = caa_outcomes - .par_iter() - .filter(|(_, _, provided, url)| *provided && url.is_none()) - .map(|(artist, album, _, _)| (artist.clone(), album.clone())) - .collect(); - - if !fallback.is_empty() { - let searched2: Vec<(String, String, String)> = fallback - .par_iter() - .filter_map(|(artist, album)| { - mb_limiter.acquire(); - musicbrainz_release(&agent, artist, album) - .map(|mbid| (artist.clone(), album.clone(), mbid)) - }) - .collect(); - - let caa2: Vec<(String, String, String)> = searched2 - .par_iter() - .filter_map(|(artist, album, mbid)| { - caa_limiter.acquire(); - coverart_url(&agent, mbid).map(|url| (artist.clone(), album.clone(), url)) - }) - .collect(); - - let mut cache = cache.lock().unwrap(); - for (artist, album, url) in caa2 { - cache.insert(cache_key(&artist, &album), url.clone()); - out.insert((stats::normalize(&artist), stats::normalize(&album)), url); - } - } - - out -} - -/// Resolve real artist images via the Wikidata chain (MusicBrainz -/// artist search → url-rels → P18 → Commons). Cache-first, same -/// append-only pattern. -pub fn resolve_artists(artists: &[String], cache: &Cache) -> HashMap { - let mut out: HashMap = HashMap::new(); - let missing: Vec = { - let cache = cache.lock().unwrap(); - artists - .iter() - .filter_map(|name| match cache.get(&artist_cache_key(name)) { - Some(url) => { - out.insert(stats::normalize(name), url.clone()); - None - } - None => Some(name.clone()), - }) - .collect() - }; - - if missing.is_empty() { - return out; - } - - let agent = ureq::AgentBuilder::new() - .user_agent(USER_AGENT) - .timeout(Duration::from_secs(15)) - .build(); - - // Per artist: two MusicBrainz requests (search + url-rels lookup) - // share the MB limiter; Wikidata answers at ~1 req/s. - let mb_limiter = &Limiter::new(3.0); - let wd_limiter = &Limiter::new(1.0); - let results: Vec<(String, String)> = missing - .par_iter() - .filter_map(|name| { - let mbid = { - mb_limiter.acquire(); - musicbrainz_artist_mbid(&agent, name) - }?; - let qid = { - mb_limiter.acquire(); - wikidata_qid(&agent, &mbid) - }?; - wd_limiter.acquire(); - let filename = wikidata_p18(&agent, &qid)?; - Some((name.clone(), commons_url(&filename))) - }) - .collect(); - - let mut cache = cache.lock().unwrap(); - for (name, url) in results { - cache.insert(artist_cache_key(&name), url.clone()); - out.insert(stats::normalize(&name), url); - } - - out -} - -/// MusicBrainz artist search → best-guess artist MBID. -fn musicbrainz_artist_mbid(agent: &ureq::Agent, name: &str) -> Option { - let query = format!("artist:\"{}\"", clean_query(name)); - get_json( - agent, - "https://musicbrainz.org/ws/2/artist", - &[("query", &query), ("fmt", "json"), ("limit", "1")], - ) - .and_then(|body| { - body["artists"] - .as_array()? - .first()? - .get("id")? - .as_str() - .map(str::to_string) - }) -} - -/// MB artist lookup with URL relations → the artist's Wikidata QID. -fn wikidata_qid(agent: &ureq::Agent, mbid: &str) -> Option { - let body = get_json( - agent, - &format!("https://musicbrainz.org/ws/2/artist/{mbid}"), - &[("inc", "url-rels"), ("fmt", "json")], - )?; - for relation in body["relations"].as_array()? { - if relation["type"].as_str() == Some("wikidata") { - let url = relation["url"]["resource"].as_str()?; - // https://www.wikidata.org/wiki/Q130798 - return url.rsplit('/').next().map(str::to_string); - } - } - None -} - -/// Wikidata `P18` (image) claim → Commons filename. -fn wikidata_p18(agent: &ureq::Agent, qid: &str) -> Option { - let body = get_json( - agent, - "https://www.wikidata.org/w/api.php", - &[ - ("action", "wbgetclaims"), - ("property", "P18"), - ("entity", qid), - ("format", "json"), - ], - )?; - body["claims"]["P18"] - .as_array()? - .first()? - .get("mainsnak")? - .get("datavalue")? - .get("value")? - .as_str() - .map(str::to_string) -} - -/// Commons image URL for a filename, resized to 600px. -fn commons_url(filename: &str) -> String { - format!( - "https://commons.wikimedia.org/wiki/Special:FilePath/{}?width=600", - percent_encode(filename) - ) -} - -/// Minimal percent-encoding for the path component of a Commons URL. -fn percent_encode(s: &str) -> String { - let mut out = String::with_capacity(s.len()); - for b in s.bytes() { - match b { - b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { - out.push(b as char) - } - _ => out.push_str(&format!("%{b:02X}")), - } - } - out -} - -/// MusicBrainz recording search → best-guess recording MBID. -fn musicbrainz_recording(agent: &ureq::Agent, artist: &str, track: &str) -> Option { - let query = format!( - "recording:\"{}\" AND artist:\"{}\"", - clean_query(track), - clean_query(artist) - ); - get_json( - agent, - "https://musicbrainz.org/ws/2/recording", - &[("query", &query), ("fmt", "json"), ("limit", "1")], - ) - .and_then(|body| { - body["recordings"] - .as_array()? - .first()? - .get("id")? - .as_str() - .map(str::to_string) - }) -} - -/// Resolve MusicBrainz release pages for the top albums. Cache-first, -/// like `resolve`: provided MBIDs are used directly, otherwise a name -/// search finds one. Keys in the returned map are normalized -/// (artist, album) pairs. -pub fn resolve_album_urls( - pairs: &[(String, String, Option)], - cache: &Cache, -) -> HashMap<(String, String), String> { - let mut out: HashMap<(String, String), String> = HashMap::new(); - let missing: Vec<(String, String, Option)> = { - let cache = cache.lock().unwrap(); - pairs - .iter() - .filter_map(|(artist, album, mbid)| { - match cache.get(&album_url_cache_key(artist, album)) { - Some(url) => { - out.insert( - (stats::normalize(artist), stats::normalize(album)), - url.clone(), - ); - None - } - None => Some((artist.clone(), album.clone(), mbid.clone())), - } - }) - .collect() - }; - - if missing.is_empty() { - return out; - } - - let agent = ureq::AgentBuilder::new() - .user_agent(USER_AGENT) - .timeout(Duration::from_secs(15)) - .build(); - - let limiter = &Limiter::new(3.0); - let results: Vec<(String, String, String)> = missing - .par_iter() - .filter_map(|(artist, album, mbid)| { - limiter.acquire(); - let mbid = match mbid { - Some(mbid) => mbid.clone(), - None => musicbrainz_release(&agent, artist, album)?, - }; - Some(( - artist.clone(), - album.clone(), - format!("https://musicbrainz.org/release/{mbid}"), - )) - }) - .collect(); - - let mut cache = cache.lock().unwrap(); - for (artist, album, url) in results { - cache.insert(album_url_cache_key(&artist, &album), url.clone()); - out.insert((stats::normalize(&artist), stats::normalize(&album)), url); - } - - out -} - -/// Resolve MusicBrainz artist pages for the top artists. Cache-first; -/// a miss does the same artist search as the image pipeline. Keys are -/// normalized artist names. -pub fn resolve_artist_urls(artists: &[String], cache: &Cache) -> HashMap { - let mut out: HashMap = HashMap::new(); - let missing: Vec = { - let cache = cache.lock().unwrap(); - artists - .iter() - .filter_map(|name| match cache.get(&artist_url_cache_key(name)) { - Some(url) => { - out.insert(stats::normalize(name), url.clone()); - None - } - None => Some(name.clone()), - }) - .collect() - }; - - if missing.is_empty() { - return out; - } - - let agent = ureq::AgentBuilder::new() - .user_agent(USER_AGENT) - .timeout(Duration::from_secs(15)) - .build(); - - let limiter = &Limiter::new(3.0); - let results: Vec<(String, String)> = missing - .par_iter() - .filter_map(|name| { - limiter.acquire(); - musicbrainz_artist_mbid(&agent, name).map(|mbid| { - ( - name.clone(), - format!("https://musicbrainz.org/artist/{mbid}"), - ) - }) - }) - .collect(); - - let mut cache = cache.lock().unwrap(); - for (name, url) in results { - cache.insert(artist_url_cache_key(&name), url.clone()); - out.insert(stats::normalize(&name), url); - } - - out -} - -/// Resolve MusicBrainz recording pages for the top tracks. Cache-first; -/// a miss does a `recording:"..." AND artist:"..."` search. Keys are -/// normalized (artist, track) pairs. -pub fn resolve_track_urls( - tracks: &[(String, String)], - cache: &Cache, -) -> HashMap<(String, String), String> { - let mut out: HashMap<(String, String), String> = HashMap::new(); - let missing: Vec<(String, String)> = { - let cache = cache.lock().unwrap(); - tracks - .iter() - .filter_map( - |(artist, track)| match cache.get(&track_url_cache_key(artist, track)) { - Some(url) => { - out.insert( - (stats::normalize(artist), stats::normalize(track)), - url.clone(), - ); - None - } - None => Some((artist.clone(), track.clone())), - }, - ) - .collect() - }; - - if missing.is_empty() { - return out; - } - - let agent = ureq::AgentBuilder::new() - .user_agent(USER_AGENT) - .timeout(Duration::from_secs(15)) - .build(); - - let limiter = &Limiter::new(3.0); - let results: Vec<(String, String, String)> = missing - .par_iter() - .filter_map(|(artist, track)| { - limiter.acquire(); - musicbrainz_recording(&agent, artist, track).map(|mbid| { - ( - artist.clone(), - track.clone(), - format!("https://musicbrainz.org/recording/{mbid}"), - ) - }) - }) - .collect(); - - let mut cache = cache.lock().unwrap(); - for (artist, track, url) in results { - cache.insert(track_url_cache_key(&artist, &track), url.clone()); - out.insert((stats::normalize(&artist), stats::normalize(&track)), url); - } - - out -} diff --git a/tools/parse-plays/src/images.rs b/tools/parse-plays/src/images.rs index 800d4ec..127eaef 100644 --- a/tools/parse-plays/src/images.rs +++ b/tools/parse-plays/src/images.rs @@ -10,25 +10,16 @@ //! the rest of the site. Downloads that fail keep their remote URL in //! the JSON so the tile still renders. -use crate::{ - covers::{Limiter, USER_AGENT, is_retryable}, - net, -}; +use crate::model::Href; +use crate::net::HttpClient; +use crate::sources::RateLimits; use rayon::prelude::*; use std::collections::{HashMap, HashSet}; -use std::io::Read; use std::path::Path; -use std::time::Duration; +use std::str::FromStr; /// Generous cap — covers and artist photos are a few hundred KB. const MAX_IMAGE_BYTES: u64 = 8 * 1024 * 1024; -const RATE_PER_SEC: f64 = 8.0; - -static RETRY_POLICY: net::RetryPolicy = net::RetryPolicy { - max_attempts: 5, - base_delay: Duration::from_millis(500), - max_delay: Duration::from_secs(8), -}; /// 16 hex chars of fnv-1a 64. Collision-proof enough for a few /// hundred URLs and stable across runs, so the cache is append-only. @@ -78,82 +69,56 @@ fn scan_cache(dir: &Path) -> (HashMap, Vec) } /// Download `url` into `dir`, returning the local URL path -/// (`/img/.`), or `None` on failure. A non-2xx response -/// (404 — no image) is a permanent miss, not worth retrying. -fn fetch_image(agent: &ureq::Agent, dir: &Path, url: &str) -> Option { - let hash = short_hash(url); - net::retry(&RETRY_POLICY, || { - match agent.get(url).call() { - Ok(resp) if resp.status() == 200 => { - let ct = resp - .header("content-type") - .map(str::to_string) - .unwrap_or_default(); - let mut buf = Vec::new(); - let read = resp - .into_reader() - .take(MAX_IMAGE_BYTES + 1) - .read_to_end(&mut buf); - // Empty or oversized: don't write, and don't let a - // truncated copy become the cached artifact. - match read { - Ok(n) if !buf.is_empty() && n as u64 <= MAX_IMAGE_BYTES => { - let name = format!("{hash}.{}", ext_for_content_type(&ct)); - // Write to a temp file and rename so a crash - // mid-write never leaves a truncated file that - // later builds mistake for a complete cache entry. - let tmp = dir.join(format!("{name}.tmp")); - match std::fs::write(&tmp, &buf) - .and_then(|_| std::fs::rename(&tmp, dir.join(&name))) - { - Ok(_) => net::Attempt::Done(format!("/img/{name}")), - Err(_) => net::Attempt::Stop, - } - } - _ => net::Attempt::Stop, - } - } - Ok(_) => net::Attempt::Stop, - Err(ureq::Error::Status(code, resp)) if is_retryable(code) => { - net::Attempt::Again(net::retry_after(&resp)) - } - // Transport-level failures (timeouts, resets) are transient. - Err(_) => net::Attempt::Again(None), - } - }) +/// (`/img/.`), or `None` on failure. Empty or oversized +/// bodies are dropped so a truncated copy never becomes the cached +/// artifact; writes go through a temp file + rename so a crash +/// mid-write can't leave a partial file behind. +fn fetch_image(client: &HttpClient, dir: &Path, url: &Href) -> Option { + let hash = short_hash(url.as_ref()); + let (ct, buf) = client.get_bytes(url.as_ref(), MAX_IMAGE_BYTES as usize)?; + if buf.is_empty() || buf.len() as u64 > MAX_IMAGE_BYTES { + return None; + } + let name = format!("{hash}.{}", ext_for_content_type(&ct)); + let tmp = dir.join(format!("{name}.tmp")); + std::fs::write(&tmp, &buf).ok()?; + std::fs::rename(&tmp, dir.join(&name)).ok()?; + Some(format!("/img/{name}")) } /// Mirror every unique remote `url` into `dir` (cache-first) and /// return the remote→local path map. Downloads run in parallel under -/// a shared rate limiter; a failed download is simply absent from the +/// per-host rate limiters; a failed download is simply absent from the /// map so the caller can keep the remote URL. -pub fn download_many(urls: &[String], dir: &Path) -> HashMap { +pub fn download_many( + client: &HttpClient, + limits: &RateLimits, + urls: &[Href], + dir: &Path, +) -> HashMap { let _ = std::fs::create_dir_all(dir); let (by_hash, stale) = scan_cache(dir); for stale_path in stale { let _ = std::fs::remove_file(stale_path); } - let unique: HashSet<&String> = urls.iter().collect(); - let agent = ureq::AgentBuilder::new() - .user_agent(USER_AGENT) - .timeout(Duration::from_secs(30)) - .build(); - let limiter = &Limiter::new(RATE_PER_SEC); - - let urls: Vec<&String> = unique.into_iter().collect(); + let unique: HashSet<&Href> = urls.iter().collect(); + let urls: Vec<&Href> = unique.into_iter().collect(); // Parallel downloads under a shared limiter: with server latency // far above the limiter interval, workers overlap the waits. - let results: Vec<(String, Option)> = urls + let results: Vec<(Href, Option)> = urls .par_iter() .map(|url| { - let url = *url; - let hash = short_hash(url); + let hash = short_hash(url.as_ref()); match by_hash.get(&hash) { - Some(local) => (url.clone(), Some(local.clone())), + Some(local) => ((*url).clone(), Some(local.clone())), None => { - limiter.acquire(); - (url.clone(), fetch_image(&agent, dir, url)) + if url.as_ref().contains("coverartarchive.org") { + limits.coverart_img.acquire(); + } else { + limits.commons.acquire(); + } + ((*url).clone(), fetch_image(client, dir, url)) } } }) @@ -169,7 +134,7 @@ pub fn download_many(urls: &[String], dir: &Path) -> HashMap { } /// Point every `image` field at its local mirror, when one exists. -pub fn apply_rewrites(value: &mut serde_json::Value, rewrites: &HashMap) { +pub fn apply_rewrites(value: &mut serde_json::Value, rewrites: &HashMap) { match value { serde_json::Value::Array(items) => { for item in items { @@ -178,7 +143,8 @@ pub fn apply_rewrites(value: &mut serde_json::Value, rewrites: &HashMap { if let Some(serde_json::Value::String(url)) = map.get("image") - && let Some(local) = rewrites.get(url) + && let Ok(href) = Href::from_str(url) + && let Some(local) = rewrites.get(&href) { map.insert("image".into(), serde_json::Value::String(local.clone())); } @@ -194,6 +160,10 @@ pub fn apply_rewrites(value: &mut serde_json::Value, rewrites: &HashMap Result<(), String> { // Process (pure): aggregate and figure out which covers we need. let agg = stats::aggregate(&plays, chrono::Utc::now().date_naive()); - // Commit (impure): resolve covers through the cache, then serialize. + // Commit (impure): resolve through the cache, then serialize. write_stats( &agg, cache_path.as_deref(), @@ -133,115 +140,80 @@ async fn cmd_refresh(args: &[String]) -> Result<(), String> { ) } -/// Shared tail of `stats` and `refresh`: resolve covers, artist -/// images, and MusicBrainz page links for the top-N entries, mirror -/// the remote images into a local dir, serialize the grids, write the -/// stats file. +/// The impure-pure-impure sandwich: load the cache and the needs, +/// plan the resolution (pure), fetch (impure), finalize (pure), then +/// commit — staged cache writes, image mirroring, and the stats file. fn write_stats( agg: &stats::Aggregated, cache_path: Option<&str>, stats_out: &str, images_dir: Option<&str>, ) -> Result<(), String> { - let pairs = stats::needed_pairs(agg); - let artists = stats::needed_artists(agg); - let tracks = stats::needed_tracks(agg); - + // Gather: needs + cache. + let needs = stats::needed(agg); eprintln!( "resolving {} album covers, {} artist images, {} links", - pairs.len(), - artists.len(), - pairs.len() + artists.len() + tracks.len() - ); - - // Shared lookup cache: loaded once, written by every phase under - // the mutex, persisted once after all phases. Without a cache path - // we still share an (empty) cache — behavior is unchanged. - let cache = std::sync::Mutex::new(cache_path.map(covers::load_cache).unwrap_or_default()); - - // Stage 1: the image sources — cover art and artist photos. These - // two are the slow ones (MusicBrainz + Wikidata chains), so they - // run first and in parallel. - let (cover_map, artist_map) = rayon::join( - || covers::resolve(&pairs, &cache), - || covers::resolve_artists(&artists, &cache), + needs.albums.len(), + needs.artists.len(), + needs.albums.len() + needs.artists.len() + needs.tracks.len() ); - - // Stage 2: mirror the images in parallel with resolving the - // MusicBrainz page links — the downloads only need the two image - // maps from stage 1, so the URL lookups overlap them instead of - // running serially before them. - let image_urls: Vec = cover_map - .values() - .chain(artist_map.values()) - .cloned() - .collect(); - let mut download_map: HashMap = HashMap::new(); - let mut album_url_map = HashMap::new(); - let mut artist_url_map = HashMap::new(); - let mut track_url_map = HashMap::new(); - rayon::scope(|s| { - if let Some(dir) = images_dir { - s.spawn(|_| { - download_map = images::download_many(&image_urls, std::path::Path::new(dir)); - }); - } - s.spawn(|_| album_url_map = covers::resolve_album_urls(&pairs, &cache)); - s.spawn(|_| artist_url_map = covers::resolve_artist_urls(&artists, &cache)); - s.spawn(|_| track_url_map = covers::resolve_track_urls(&tracks, &cache)); - }); - if let Some(dir) = images_dir { - eprintln!("mirrored images into {dir}"); + let mut cache = cache::Cache::load(cache_path); + let now = SystemTime::now(); + + // Process: decide what the cache can't satisfy. + let plan = resolve::plan(&needs, &cache, now); + + // Gather: fetch the queries (MusicBrainz/CAA/Wikidata). + let sources = sources::MusicSources::new(); + let results = sources.run_queries(&plan); + + // Process: fold outcomes into resolutions + staged cache writes. + let mut fin = resolve::finalize(&plan, results, now); + + // One fallback round: provided MBIDs with no Cover Art Archive art + // get a name search (usually lands on the canonical release). + if !fin.pending.is_empty() { + let fallback = resolve::ResolvePlan { + resolved: Vec::new(), + queries: std::mem::take(&mut fin.pending), + }; + let results = sources.run_queries(&fallback); + let fin2 = resolve::finalize(&fallback, results, now); + fin.resolutions.extend(fin2.resolutions); + fin.writes.extend(fin2.writes); } - // Persist the merged cache once, after every phase has finished. + // Commit: staged cache writes once, then the stats document. + let resolved = resolve::collect(&plan, &fin); + cache.merge(fin.writes); if let Some(path) = cache_path { - covers::save_cache_atomically(path, &cache.into_inner().unwrap()); + cache + .save_atomically(path) + .map_err(|e| format!("failed to save cache: {e}"))?; } - let cover = |artist: &str, album: &str| -> String { - cover_map - .get(&(stats::normalize(artist), stats::normalize(album))) - .cloned() - .unwrap_or_default() - }; - let artist_cover = |name: &str| -> String { - artist_map - .get(&stats::normalize(name)) - .cloned() - .unwrap_or_default() - }; - let album_url = |artist: &str, album: &str| -> String { - album_url_map - .get(&(stats::normalize(artist), stats::normalize(album))) - .cloned() - .unwrap_or_default() - }; - let artist_url = |name: &str| -> String { - artist_url_map - .get(&stats::normalize(name)) - .cloned() - .unwrap_or_default() - }; - let track_url = |artist: &str, track: &str| -> String { - track_url_map - .get(&(stats::normalize(artist), stats::normalize(track))) + let mut doc = json!({ "ranges": stats::build_ranges(agg, &resolved) }); + if let Some(dir) = images_dir { + // Mirror the remote images beside the site so the browser + // never hits Cover Art Archive / Wikimedia at page load. + let image_urls: Vec = resolved + .cover + .values() + .chain(resolved.artist_cover.values()) .cloned() - .unwrap_or_default() - }; - let lookups = stats::Lookups { - cover: &cover, - artist_cover: &artist_cover, - album_url: &album_url, - artist_url: &artist_url, - track_url: &track_url, - }; + .collect(); + let download_map = images::download_many( + &sources.client, + &sources.limits, + &image_urls, + std::path::Path::new(dir), + ); + eprintln!("mirrored images into {dir}"); + // Point the image fields at the local mirrors; any URL missing + // from the map (failed download) keeps its remote value. + images::apply_rewrites(&mut doc, &download_map); + } - let stats_json = stats::build_ranges(agg, &lookups); - let mut doc = json!({ "ranges": stats_json }); - // Point the image fields at the local mirrors; any URL missing - // from the map (failed download) keeps its remote value. - images::apply_rewrites(&mut doc, &download_map); std::fs::write( stats_out, serde_json::to_string_pretty(&doc).expect("serialize"), diff --git a/tools/parse-plays/src/model.rs b/tools/parse-plays/src/model.rs new file mode 100644 index 0000000..9cf232b --- /dev/null +++ b/tools/parse-plays/src/model.rs @@ -0,0 +1,344 @@ +//! Typed domain model: the shared vocabulary between the pure stats +//! core, the resolution pipeline, and the cache. +//! +//! Keys are newtypes so a display name can never be used where a +//! normalized key is expected and an artist key can never be passed +//! where an album key belongs — the invariants the old `(String, +//! String)` soup enforced by convention are now compile-time. `Href` +//! carries a scheme allow-list because these strings get baked into +//! the site's HTML; `MusicBrainzId` validates the UUID shape so bogus +//! IDs from play records can't leak into API calls. + +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::str::FromStr; + +// ---------------------------------------------------------------- keys + +/// Sanitized grouping key for artist names: lowercase, punctuation +/// dropped, collaboration credit cut to the first artist, junction +/// words (feat/with/&/vs...) removed. Two spellings of the same +/// artist ("JAY-Z" vs "Jay Z", "A$AP" vs "ASAP") land in the same +/// bucket; display names keep their original spelling. +pub fn normalize(s: &str) -> String { + const JUNCTION_WORDS: [&str; 12] = [ + "and", + "with", + "feat", + "featuring", + "ft", + "vs", + "versus", + "present", + "presents", + "pres", + "b2b", + "x", + ]; + + let lower = s.to_lowercase(); + // Collab credit strings: keep only the part before the first + // comma/ampersand/plus — "tofubeats, HITOMITOI" groups as tofubeats. + let head = lower + .split(['&', '+', ',']) + .next() + .unwrap_or(lower.as_str()); + head + // Any run of non-alphanumerics is a word boundary ("JAY-Z", + // "A$AP", "Café" -> "caf", accents are dropped with them). + .split(|c: char| !c.is_alphanumeric()) + .take_while(|word| !JUNCTION_WORDS.contains(word)) + .filter(|word| !word.is_empty()) + .collect::>() + .join(" ") +} + +/// Plain key for album/track name components: trim + lowercase only. +pub fn normalize_name(s: &str) -> String { + s.trim().to_lowercase() +} + +macro_rules! key_type { + ($name:ident, $normalize:expr) => { + /// Normalized key; the only constructor is `From<&str>`, which + /// applies the shared normalization so a raw display name can + /// never be a key. + #[derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] + pub struct $name(String); + + impl From<&str> for $name { + fn from(s: &str) -> Self { + $name($normalize(s)) + } + } + + impl From for $name { + fn from(s: String) -> Self { + $name($normalize(&s)) + } + } + + impl std::fmt::Debug for $name { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, concat!(stringify!($name), "({:?})"), self.0) + } + } + + impl AsRef for $name { + fn as_ref(&self) -> &str { + &self.0 + } + } + }; +} + +key_type!(ArtistKey, normalize); +key_type!(AlbumKey, normalize_name); +key_type!(TrackKey, normalize_name); + +/// An (artist, album) pair key — distinct from `TrackRef` by type. +#[derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)] +pub struct AlbumRef { + pub artist: ArtistKey, + pub album: AlbumKey, +} + +/// An (artist, track) pair key — distinct from `AlbumRef` by type. +#[derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)] +pub struct TrackRef { + pub artist: ArtistKey, + pub track: TrackKey, +} + +// ---------------------------------------------------------------- ids + +/// Validated MusicBrainz ID (8-4-4-4-12 lowercase hex). +#[derive(Clone, PartialEq, Eq, Hash, Debug, Serialize, Deserialize)] +pub struct MusicBrainzId(String); + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct InvalidId; + +impl MusicBrainzId { + fn validate(s: &str) -> bool { + let b = s.as_bytes(); + if b.len() != 36 { + return false; + } + b.iter().enumerate().all(|(i, c)| { + if matches!(i, 8 | 13 | 18 | 23) { + *c == b'-' + } else { + c.is_ascii_hexdigit() + } + }) + } +} + +impl FromStr for MusicBrainzId { + type Err = InvalidId; + + fn from_str(s: &str) -> Result { + // Strip any `mbid:` (or other) prefix — some clients emit it. + let id = s.rsplit(':').next().unwrap_or(s); + if !Self::validate(id) { + return Err(InvalidId); + } + Ok(MusicBrainzId(id.to_ascii_lowercase())) + } +} + +impl AsRef for MusicBrainzId { + fn as_ref(&self) -> &str { + &self.0 + } +} + +/// Validated Wikidata entity ID (`Q` + digits). +#[derive(Clone, PartialEq, Eq, Hash, Debug)] +pub struct WikidataId(String); + +impl FromStr for WikidataId { + type Err = InvalidId; + + fn from_str(s: &str) -> Result { + let digits = s.strip_prefix('Q').ok_or(InvalidId)?; + if digits.is_empty() || !digits.bytes().all(|b| b.is_ascii_digit()) { + return Err(InvalidId); + } + Ok(WikidataId(s.to_string())) + } +} + +impl AsRef for WikidataId { + fn as_ref(&self) -> &str { + &self.0 + } +} + +/// A Wikimedia Commons image filename (no validation beyond non-empty — +/// the P18 chain is where filename-vs-URL confusion bites, so the type +/// keeps them apart). +#[derive(Clone, PartialEq, Eq, Hash, Debug)] +pub struct CommonsFilename(String); + +impl FromStr for CommonsFilename { + type Err = InvalidId; + + fn from_str(s: &str) -> Result { + if s.is_empty() { + return Err(InvalidId); + } + Ok(CommonsFilename(s.to_string())) + } +} + +impl AsRef for CommonsFilename { + fn as_ref(&self) -> &str { + &self.0 + } +} + +// ---------------------------------------------------------------- urls + +/// Validated absolute http(s) URL. Only the two safe schemes survive — +/// everything else would be baked into the site's HTML unescaped. +#[derive(Clone, PartialEq, Eq, Hash, Debug, Serialize, Deserialize)] +pub struct Href(String); + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct InvalidHref; + +impl FromStr for Href { + type Err = InvalidHref; + + fn from_str(s: &str) -> Result { + let rest = s + .strip_prefix("https://") + .or_else(|| s.strip_prefix("http://")) + .ok_or(InvalidHref)?; + if rest.is_empty() + || rest.starts_with('/') + || rest.contains(|c: char| c.is_whitespace() || c.is_control()) + { + return Err(InvalidHref); + } + Ok(Href(s.to_string())) + } +} + +impl AsRef for Href { + fn as_ref(&self) -> &str { + &self.0 + } +} + +// ---------------------------------------------------------------- resolved + +/// What resolution must produce before serialization: cover art, +/// artist images, and MusicBrainz page links for the top-N entries. +#[derive(Default)] +pub struct ResolvedStats { + pub cover: HashMap, + pub artist_cover: HashMap, + pub album_url: HashMap, + pub artist_url: HashMap, + pub track_url: HashMap, +} + +// ---------------------------------------------------------------- tests + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn key_constructors_normalize() { + // Display-name variants collapse to one key. + assert_eq!( + ArtistKey::from("JAY-Z feat. Beyoncé"), + ArtistKey::from("Jay Z") + ); + assert_eq!( + ArtistKey::from("tofubeats, HITOMITOI"), + ArtistKey::from("Tofubeats") + ); + assert_eq!(ArtistKey::from("A$AP Rocky"), ArtistKey::from("a$ap-rocky")); + // Album/track keys are trim+lowercase only — junction words stay. + assert_eq!(AlbumKey::from(" Animals "), AlbumKey::from("animals")); + assert_eq!(AlbumKey::from("Red & Blue"), AlbumKey::from("red & blue")); + // Key kinds are distinct types: this is a compile-time property, + // asserted here so it can't silently regress to one string type. + let _: AlbumRef = AlbumRef { + artist: ArtistKey::from("a"), + album: AlbumKey::from("b"), + }; + let _: TrackRef = TrackRef { + artist: ArtistKey::from("a"), + track: TrackKey::from("c"), + }; + } + + #[test] + fn mbid_validates_uuid_shape() { + let ok = "0a0a0a0a-0b0b-0c0c-0d0d-0e0e0f0f0f0f"; + assert_eq!(MusicBrainzId::from_str(ok).unwrap().as_ref(), ok); + // Mixed case normalizes to lowercase; `mbid:` prefix is stripped. + assert_eq!( + MusicBrainzId::from_str(&format!("mbid:{ok}")) + .unwrap() + .as_ref(), + ok + ); + assert_eq!( + MusicBrainzId::from_str("0A0A0A0A-0B0B-0C0C-0D0D-0E0E0F0F0F0F") + .unwrap() + .as_ref(), + ok + ); + for bad in [ + "", + "0a0a0a0a-0b0b-0c0c-0d0d", // too short + "0a0a0a0a-0b0b-0c0c-0d0d-0e0e0f0f0f0g", // non-hex + "0a0a0a0a0b0b0c0c0d0d0e0e0f0f0f0f", // no hyphens + "0a0a0a0a-0b0b-0c0c-0d0d0e0e-0f0f0f0f", // wrong grouping + "mbid:", // empty after prefix + ] { + assert!(MusicBrainzId::from_str(bad).is_err(), "{bad:?} should fail"); + } + } + + #[test] + fn href_rejects_non_http_schemes() { + for ok in [ + "https://example.com", + "http://example.com/a/b.png", + "https://host", + ] { + assert!(Href::from_str(ok).is_ok(), "{ok:?} should pass"); + } + for bad in [ + "", + "example.com", // relative + "//example.com", // protocol-relative + "file:///etc/passwd", + "ftp://example.com", + "javascript:alert(1)", + "data:text/html,x", + "https:///path", // empty host + "https://exa mple.com", + ] { + assert!(Href::from_str(bad).is_err(), "{bad:?} should fail"); + } + } + + #[test] + fn wikidata_and_commons_ids() { + assert!(WikidataId::from_str("Q130798").is_ok()); + assert!(WikidataId::from_str("q130798").is_err()); // uppercase Q only + assert!(WikidataId::from_str("Q").is_err()); + assert!(WikidataId::from_str("Q12x").is_err()); + assert!(CommonsFilename::from_str("File:X.jpg").is_ok()); + assert!(CommonsFilename::from_str("").is_err()); + } +} diff --git a/tools/parse-plays/src/net.rs b/tools/parse-plays/src/net.rs index 019c90d..ea961fb 100644 --- a/tools/parse-plays/src/net.rs +++ b/tools/parse-plays/src/net.rs @@ -1,4 +1,6 @@ -//! Shared retry/backoff for external HTTP lookups. +//! Shared HTTP boundary: the single owner of "how we talk to the +//! network" — agent construction, retry policy, user agent, rate +//! limiting, and retryable-status classification. //! //! Every endpoint we touch (MusicBrainz, Cover Art Archive, Wikidata, //! Wikimedia Commons) is rate-limited and occasionally flakes. Retrying @@ -7,7 +9,18 @@ //! link from this build is now retried, and the jitter stops a herd of //! parallel workers from hammering the endpoint at identical instants. -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use std::sync::Mutex; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +pub const USER_AGENT: &str = "karitham-blog/0.1.0 (https://karitham.dev)"; + +/// 3 attempts / 4s max: transient storms cost at most ~1.5s of backoff +/// before we give up and (thanks to negative caching) don't repeat. +pub const RETRY_POLICY: RetryPolicy = RetryPolicy { + max_attempts: 3, + base_delay: Duration::from_millis(500), + max_delay: Duration::from_secs(4), +}; /// How hard to try, and how long to wait between attempts. #[derive(Clone, Copy)] @@ -81,6 +94,135 @@ pub fn retry_after(resp: &ureq::Response) -> Option { .and_then(|v| v.trim().parse().ok()) } +/// HTTP status codes worth retrying: rate limiting and server errors. +pub fn is_retryable(code: u16) -> bool { + code == 429 || code == 500 || code == 503 +} + +/// Global rate limiter shared by workers: at most `rate_per_sec` +/// acquisitions per second across all threads. +pub struct Limiter { + min_interval: Duration, + last: Mutex, +} + +impl Limiter { + pub fn new(rate_per_sec: f64) -> Self { + Self { + min_interval: Duration::from_secs_f64(1.0 / rate_per_sec), + last: Mutex::new(Instant::now()), + } + } + + pub fn acquire(&self) { + let mut last = self.last.lock().unwrap(); + let since = Instant::now().duration_since(*last); + if since < self.min_interval { + std::thread::sleep(self.min_interval - since); + } + *last = Instant::now(); + } +} + +/// One agent, one policy, one user agent — the covers/images copies +/// collapse here. Methods own their retry/backoff internally. +pub struct HttpClient { + agent: ureq::Agent, + policy: RetryPolicy, +} + +impl HttpClient { + pub fn new() -> Self { + Self { + agent: ureq::AgentBuilder::new() + .user_agent(USER_AGENT) + .timeout(Duration::from_secs(15)) + .build(), + policy: RETRY_POLICY, + } + } + + /// GET a JSON API with query params. A 200 that fails to parse is a + /// permanent miss — the endpoint answered, the shape just isn't + /// what we expected. + pub fn get_json(&self, url: &str, params: &[(&str, &str)]) -> Option { + retry(&self.policy, || { + let mut req = self.agent.get(url); + for (key, value) in params { + req = req.query(key, value); + } + match req.call() { + Ok(resp) if resp.status() == 200 => match resp.into_string() { + Ok(body) => match serde_json::from_str(&body) { + Ok(value) => Attempt::Done(value), + Err(_) => Attempt::Stop, + }, + Err(_) => Attempt::Stop, + }, + Ok(_) => Attempt::Stop, + Err(ureq::Error::Status(code, resp)) if is_retryable(code) => { + Attempt::Again(retry_after(&resp)) + } + // Non-retryable status (404, 400...) is a permanent + // miss — ureq surfaces it as `Error::Status`, and + // retrying it just burns backoff. + Err(ureq::Error::Status(_, _)) => Attempt::Stop, + // Transport-level failures (timeouts, resets, DNS) are + // transient — retry rather than dropping the link for + // this whole build. + Err(_) => Attempt::Again(None), + } + }) + } + + /// GET a body, capped at `max` bytes, returning (content-type, + /// body). Callers check the length — an oversized response must + /// not become a cached artifact. + pub fn get_bytes(&self, url: &str, max: usize) -> Option<(String, Vec)> { + retry(&self.policy, || match self.agent.get(url).call() { + Ok(resp) if resp.status() == 200 => { + let ct = resp + .header("content-type") + .map(str::to_string) + .unwrap_or_default(); + use std::io::Read; + let mut buf = Vec::new(); + match resp + .into_reader() + .take(max as u64 + 1) + .read_to_end(&mut buf) + { + Ok(_) => Attempt::Done((ct, buf)), + Err(_) => Attempt::Stop, + } + } + Ok(_) => Attempt::Stop, + Err(ureq::Error::Status(code, resp)) if is_retryable(code) => { + Attempt::Again(retry_after(&resp)) + } + // Non-retryable status (404, 400...) is a permanent miss. + Err(ureq::Error::Status(_, _)) => Attempt::Stop, + Err(_) => Attempt::Again(None), + }) + } + + /// Probe a URL for a 200 (e.g. Cover Art Archive's front-500 + /// endpoint). A non-2xx response (404 — no art) is a permanent + /// miss, not worth retrying. + pub fn check(&self, url: &str) -> bool { + retry(&self.policy, || match self.agent.get(url).call() { + Ok(resp) if resp.status() == 200 => Attempt::Done(true), + Ok(_) => Attempt::Stop, + Err(ureq::Error::Status(code, resp)) if is_retryable(code) => { + Attempt::Again(retry_after(&resp)) + } + Err(ureq::Error::Status(_, _)) => Attempt::Stop, + Err(_) => Attempt::Again(None), + }) + .unwrap_or(false) + } +} + #[cfg(test)] mod tests { use super::*; @@ -147,4 +289,13 @@ mod tests { assert!(j < Duration::from_millis(1500)); } } + + #[test] + fn retryable_status_classification() { + assert!(is_retryable(429)); + assert!(is_retryable(500)); + assert!(is_retryable(503)); + assert!(!is_retryable(404)); + assert!(!is_retryable(200)); + } } diff --git a/tools/parse-plays/src/resolve.rs b/tools/parse-plays/src/resolve.rs new file mode 100644 index 0000000..323c9c9 --- /dev/null +++ b/tools/parse-plays/src/resolve.rs @@ -0,0 +1,558 @@ +//! Pure resolution core: decide what the cache can't satisfy, and turn +//! raw query outcomes into resolutions + cache writes. No I/O — time +//! is passed in as data so TTL decisions are deterministic. + +use crate::cache::{Cache, CacheKey, CacheValue}; +use crate::model::{AlbumRef, ArtistKey, Href, MusicBrainzId, ResolvedStats, TrackRef}; +use crate::stats::Needs; +use std::time::{SystemTime, UNIX_EPOCH}; + +/// A lookup the cache cannot satisfy from a fresh entry. Carries the +/// display names queries are built from and (for albums) the MBID from +/// the play records, which skips the MusicBrainz search. +#[derive(Clone, Debug, PartialEq)] +pub enum Query { + AlbumCover { + album: AlbumRef, + artist_display: String, + album_display: String, + mbid: Option, + /// The MBID came from the play record: if its art is missing, + /// retry once with a name search (the fallback round). + provided: bool, + }, + AlbumUrl { + album: AlbumRef, + artist_display: String, + album_display: String, + mbid: Option, + }, + ArtistImage { + artist: ArtistKey, + artist_display: String, + }, + ArtistUrl { + artist: ArtistKey, + artist_display: String, + }, + TrackUrl { + track: TrackRef, + artist_display: String, + track_display: String, + }, +} + +/// A found value, tagged with which output map it belongs in. +#[derive(Clone, Debug, PartialEq)] +pub enum Resolution { + AlbumCover { album: AlbumRef, url: Href }, + AlbumUrl { album: AlbumRef, url: Href }, + ArtistImage { artist: ArtistKey, url: Href }, + ArtistUrl { artist: ArtistKey, url: Href }, + TrackUrl { track: TrackRef, url: Href }, +} + +/// What one refresh needs: cache hits (no network) + the queries to +/// fetch. +pub struct ResolvePlan { + pub resolved: Vec, + pub queries: Vec, +} + +/// The outcome of running the queries: resolutions, staged cache +/// writes, and the fallback round of queries (usually empty). +pub struct Finalized { + pub resolutions: Vec, + pub writes: Vec<(CacheKey, CacheValue)>, + pub pending: Vec, +} + +/// Decide what to fetch: cache hits become resolutions, fresh negative +/// entries are skipped, everything else becomes a query. +pub fn plan(needs: &Needs, cache: &Cache, now: SystemTime) -> ResolvePlan { + let mut resolved = Vec::new(); + let mut queries = Vec::new(); + + for need in &needs.albums { + match cache.lookup(&CacheKey::AlbumCover(need.album.clone())) { + Some(CacheValue::Found(url)) => resolved.push(Resolution::AlbumCover { + album: need.album.clone(), + url: url.clone(), + }), + Some(value) if value.is_fresh_miss(now) => {} + _ => queries.push(Query::AlbumCover { + album: need.album.clone(), + artist_display: need.artist_display.clone(), + album_display: need.album_display.clone(), + mbid: need.mbid.clone(), + provided: need.mbid.is_some(), + }), + } + match cache.lookup(&CacheKey::AlbumUrl(need.album.clone())) { + Some(CacheValue::Found(url)) => resolved.push(Resolution::AlbumUrl { + album: need.album.clone(), + url: url.clone(), + }), + Some(value) if value.is_fresh_miss(now) => {} + _ => queries.push(Query::AlbumUrl { + album: need.album.clone(), + artist_display: need.artist_display.clone(), + album_display: need.album_display.clone(), + mbid: need.mbid.clone(), + }), + } + } + + for need in &needs.artists { + match cache.lookup(&CacheKey::ArtistImage(need.key.clone())) { + Some(CacheValue::Found(url)) => resolved.push(Resolution::ArtistImage { + artist: need.key.clone(), + url: url.clone(), + }), + Some(value) if value.is_fresh_miss(now) => {} + _ => queries.push(Query::ArtistImage { + artist: need.key.clone(), + artist_display: need.display.clone(), + }), + } + match cache.lookup(&CacheKey::ArtistUrl(need.key.clone())) { + Some(CacheValue::Found(url)) => resolved.push(Resolution::ArtistUrl { + artist: need.key.clone(), + url: url.clone(), + }), + Some(value) if value.is_fresh_miss(now) => {} + _ => queries.push(Query::ArtistUrl { + artist: need.key.clone(), + artist_display: need.display.clone(), + }), + } + } + + for need in &needs.tracks { + match cache.lookup(&CacheKey::TrackUrl(need.track.clone())) { + Some(CacheValue::Found(url)) => resolved.push(Resolution::TrackUrl { + track: need.track.clone(), + url: url.clone(), + }), + Some(value) if value.is_fresh_miss(now) => {} + _ => queries.push(Query::TrackUrl { + track: need.track.clone(), + artist_display: need.artist_display.clone(), + track_display: need.track_display.clone(), + }), + } + } + + ResolvePlan { resolved, queries } +} + +/// Turn raw query outcomes into resolutions + staged cache writes. A +/// `None` outcome books a negative entry; a provided-MBID album with +/// no art gets one more chance via a name-search query. +pub fn finalize( + plan: &ResolvePlan, + results: Vec<(Query, Option)>, + now: SystemTime, +) -> Finalized { + let since = now + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + let mut resolutions = Vec::new(); + let mut writes = Vec::new(); + let mut pending = Vec::new(); + + for (query, result) in results { + match (query, result) { + (Query::AlbumCover { album, .. }, Some(url)) => { + resolutions.push(Resolution::AlbumCover { + album: album.clone(), + url: url.clone(), + }); + writes.push((CacheKey::AlbumCover(album), CacheValue::Found(url))); + } + ( + Query::AlbumCover { + album, + artist_display, + album_display, + provided: true, + .. + }, + None, + ) => { + writes.push(( + CacheKey::AlbumCover(album.clone()), + CacheValue::Missing { since }, + )); + // Name-search fallback: usually lands on the canonical + // release that has art. Not `provided`, so this never + // cascades into another fallback round. + pending.push(Query::AlbumCover { + album, + artist_display, + album_display, + mbid: None, + provided: false, + }); + } + (Query::AlbumCover { album, .. }, None) => { + writes.push((CacheKey::AlbumCover(album), CacheValue::Missing { since })); + } + (Query::AlbumUrl { album, .. }, Some(url)) => { + resolutions.push(Resolution::AlbumUrl { + album: album.clone(), + url: url.clone(), + }); + writes.push((CacheKey::AlbumUrl(album), CacheValue::Found(url))); + } + (Query::AlbumUrl { album, .. }, None) => { + writes.push((CacheKey::AlbumUrl(album), CacheValue::Missing { since })); + } + (Query::ArtistImage { artist, .. }, Some(url)) => { + resolutions.push(Resolution::ArtistImage { + artist: artist.clone(), + url: url.clone(), + }); + writes.push((CacheKey::ArtistImage(artist), CacheValue::Found(url))); + } + (Query::ArtistImage { artist, .. }, None) => { + writes.push((CacheKey::ArtistImage(artist), CacheValue::Missing { since })); + } + (Query::ArtistUrl { artist, .. }, Some(url)) => { + resolutions.push(Resolution::ArtistUrl { + artist: artist.clone(), + url: url.clone(), + }); + writes.push((CacheKey::ArtistUrl(artist), CacheValue::Found(url))); + } + (Query::ArtistUrl { artist, .. }, None) => { + writes.push((CacheKey::ArtistUrl(artist), CacheValue::Missing { since })); + } + (Query::TrackUrl { track, .. }, Some(url)) => { + resolutions.push(Resolution::TrackUrl { + track: track.clone(), + url: url.clone(), + }); + writes.push((CacheKey::TrackUrl(track), CacheValue::Found(url))); + } + (Query::TrackUrl { track, .. }, None) => { + writes.push((CacheKey::TrackUrl(track), CacheValue::Missing { since })); + } + } + } + + let _ = plan; + Finalized { + resolutions, + writes, + pending, + } +} + +/// Fold cache hits and query results into the maps `build_ranges` +/// looks up. +pub fn collect(plan: &ResolvePlan, fin: &Finalized) -> ResolvedStats { + let mut stats = ResolvedStats::default(); + for res in plan.resolved.iter().chain(fin.resolutions.iter()) { + match res { + Resolution::AlbumCover { album, url } => { + stats.cover.insert(album.clone(), url.clone()); + } + Resolution::AlbumUrl { album, url } => { + stats.album_url.insert(album.clone(), url.clone()); + } + Resolution::ArtistImage { artist, url } => { + stats.artist_cover.insert(artist.clone(), url.clone()); + } + Resolution::ArtistUrl { artist, url } => { + stats.artist_url.insert(artist.clone(), url.clone()); + } + Resolution::TrackUrl { track, url } => { + stats.track_url.insert(track.clone(), url.clone()); + } + } + } + stats +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::{AlbumKey, ArtistKey, TrackKey}; + use crate::stats::AlbumNeed; + use std::str::FromStr; + use std::time::Duration; + + fn now() -> SystemTime { + SystemTime::UNIX_EPOCH + Duration::from_secs(1_800_000_000) + } + + fn album_ref(artist: &str, album: &str) -> AlbumRef { + AlbumRef { + artist: ArtistKey::from(artist), + album: AlbumKey::from(album), + } + } + + fn href(s: &str) -> Href { + Href::from_str(s).unwrap() + } + + #[test] + fn plan_serves_hits_skips_fresh_misses_and_requeues() { + let mut cache = Cache::default(); + // Album 1: cached URL → resolution. + cache.merge(vec![( + CacheKey::AlbumCover(album_ref("A", "Hit")), + CacheValue::Found(href("https://example.com/hit.jpg")), + )]); + // Album 2: fresh miss → skipped entirely. + let now = now(); + let since = now.duration_since(UNIX_EPOCH).unwrap().as_secs(); + cache.merge(vec![( + CacheKey::AlbumCover(album_ref("A", "Fresh Miss")), + CacheValue::Missing { since }, + )]); + // Album 3: stale miss → requeued. + cache.merge(vec![( + CacheKey::AlbumCover(album_ref("A", "Stale Miss")), + CacheValue::Missing { since: 0 }, + )]); + + let needs = Needs { + albums: vec![ + AlbumNeed { + album: album_ref("A", "Hit"), + artist_display: "A".into(), + album_display: "Hit".into(), + mbid: None, + }, + AlbumNeed { + album: album_ref("A", "Fresh Miss"), + artist_display: "A".into(), + album_display: "Fresh Miss".into(), + mbid: None, + }, + AlbumNeed { + album: album_ref("A", "Stale Miss"), + artist_display: "A".into(), + album_display: "Stale Miss".into(), + mbid: None, + }, + AlbumNeed { + album: album_ref("A", "Absent"), + artist_display: "A".into(), + album_display: "Absent".into(), + mbid: None, + }, + ], + ..Needs::default() + }; + + let plan = plan(&needs, &cache, now); + assert_eq!(plan.resolved.len(), 1); + assert!(matches!( + &plan.resolved[0], + Resolution::AlbumCover { album, .. } if *album == album_ref("A", "Hit") + )); + // Each album needs a cover query AND an album-url query; the + // fresh miss is skipped entirely. + assert_eq!(plan.queries.len(), 6); + let covers: Vec<_> = plan + .queries + .iter() + .filter_map(|q| match q { + Query::AlbumCover { album, .. } => Some(album.clone()), + _ => None, + }) + .collect(); + assert_eq!(covers.len(), 2); + assert!(covers.contains(&album_ref("A", "Stale Miss"))); + assert!(covers.contains(&album_ref("A", "Absent"))); + assert!(!covers.contains(&album_ref("A", "Fresh Miss"))); + assert!(!covers.contains(&album_ref("A", "Hit"))); + } + + #[test] + fn plan_preserves_provided_mbid() { + let mbid = MusicBrainzId::from_str("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee").unwrap(); + let needs = Needs { + albums: vec![AlbumNeed { + album: album_ref("A", "B"), + artist_display: "A".into(), + album_display: "B".into(), + mbid: Some(mbid.clone()), + }], + ..Needs::default() + }; + let plan = plan(&needs, &Cache::default(), now()); + assert_eq!(plan.queries.len(), 2); // cover + album url + match &plan.queries[0] { + Query::AlbumCover { + mbid: Some(m), + provided: true, + .. + } => assert_eq!(m, &mbid), + other => panic!("unexpected query: {other:?}"), + } + } + + #[test] + fn finalize_emits_resolution_and_found_write() { + let query = Query::ArtistImage { + artist: ArtistKey::from("A"), + artist_display: "A".into(), + }; + let url = href("https://example.com/a.jpg"); + let fin = finalize( + &ResolvePlan { + resolved: vec![], + queries: vec![query.clone()], + }, + vec![(query.clone(), Some(url.clone()))], + now(), + ); + assert_eq!(fin.pending.len(), 0); + assert_eq!(fin.resolutions.len(), 1); + assert_eq!( + fin.resolutions[0], + Resolution::ArtistImage { + artist: ArtistKey::from("A"), + url: url.clone() + } + ); + assert!(fin.writes.contains(&( + CacheKey::ArtistImage(ArtistKey::from("A")), + CacheValue::Found(url) + ))); + } + + #[test] + fn finalize_records_miss_with_now_timestamp() { + let query = Query::TrackUrl { + track: TrackRef { + artist: ArtistKey::from("A"), + track: TrackKey::from("T"), + }, + artist_display: "A".into(), + track_display: "T".into(), + }; + let now = now(); + let fin = finalize( + &ResolvePlan { + resolved: vec![], + queries: vec![query.clone()], + }, + vec![(query, None)], + now, + ); + assert!(fin.resolutions.is_empty()); + assert!(fin.pending.is_empty()); + let since = now.duration_since(UNIX_EPOCH).unwrap().as_secs(); + assert!(fin.writes.contains(&( + CacheKey::TrackUrl(TrackRef { + artist: ArtistKey::from("A"), + track: TrackKey::from("T") + }), + CacheValue::Missing { since }, + ))); + } + + #[test] + fn finalize_fallback_only_for_provided_mbids() { + let now = now(); + // Provided MBID with no art → pending name-search query. + let provided = Query::AlbumCover { + album: album_ref("A", "B"), + artist_display: "A".into(), + album_display: "B".into(), + mbid: Some(MusicBrainzId::from_str("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee").unwrap()), + provided: true, + }; + let fin = finalize( + &ResolvePlan { + resolved: vec![], + queries: vec![provided.clone()], + }, + vec![(provided, None)], + now, + ); + assert_eq!(fin.pending.len(), 1); + match &fin.pending[0] { + Query::AlbumCover { + mbid: None, + provided: false, + album, + .. + } => { + assert_eq!(*album, album_ref("A", "B")); + } + other => panic!("unexpected pending query: {other:?}"), + } + + // Not provided → no pending (no infinite fallback loop). + let searched = Query::AlbumCover { + album: album_ref("A", "B"), + artist_display: "A".into(), + album_display: "B".into(), + mbid: None, + provided: false, + }; + let fin = finalize( + &ResolvePlan { + resolved: vec![], + queries: vec![searched.clone()], + }, + vec![(searched, None)], + now, + ); + assert!(fin.pending.is_empty()); + + // Other query kinds never pend either. + let artist = Query::ArtistImage { + artist: ArtistKey::from("A"), + artist_display: "A".into(), + }; + let fin = finalize( + &ResolvePlan { + resolved: vec![], + queries: vec![artist.clone()], + }, + vec![(artist, None)], + now, + ); + assert!(fin.pending.is_empty()); + } + + #[test] + fn collect_merges_hits_and_results() { + let cache_hit = Resolution::AlbumCover { + album: album_ref("A", "Cached"), + url: href("https://example.com/cached.jpg"), + }; + let query_result = Resolution::ArtistImage { + artist: ArtistKey::from("A"), + url: href("https://example.com/artist.jpg"), + }; + let plan = ResolvePlan { + resolved: vec![cache_hit.clone()], + queries: vec![], + }; + let fin = Finalized { + resolutions: vec![query_result.clone()], + writes: vec![], + pending: vec![], + }; + let stats = collect(&plan, &fin); + assert_eq!( + stats.cover.get(&album_ref("A", "Cached")), + Some(&href("https://example.com/cached.jpg")) + ); + assert_eq!( + stats.artist_cover.get(&ArtistKey::from("A")), + Some(&href("https://example.com/artist.jpg")) + ); + assert!(stats.track_url.is_empty()); + } +} diff --git a/tools/parse-plays/src/sources.rs b/tools/parse-plays/src/sources.rs new file mode 100644 index 0000000..0354fff --- /dev/null +++ b/tools/parse-plays/src/sources.rs @@ -0,0 +1,600 @@ +//! Impure gather: MusicBrainz / Cover Art Archive / Wikidata / Commons +//! queries. One deep module owning the endpoint URLs, rate limits, and +//! JSON-shape parsing — the rest of the pipeline never knows a hostname. +//! +//! The JSON→typed parsers are pure and tested with fixtures; the +//! endpoint round-trips are tested against a localhost stub server. + +use crate::model::{CommonsFilename, Href, MusicBrainzId, WikidataId}; +use crate::net::{HttpClient, Limiter}; +use crate::resolve::{Query, ResolvePlan}; +use rayon::prelude::*; +use serde_json::Value; +use std::str::FromStr; + +/// Where each API lives. Production hits the real hosts; tests point +/// every endpoint at one localhost stub. +pub struct EndpointBases { + pub musicbrainz: String, + pub coverart: String, + pub wikidata: String, + pub commons: String, +} + +impl EndpointBases { + pub fn production() -> Self { + Self { + musicbrainz: "https://musicbrainz.org".to_string(), + coverart: "https://coverartarchive.org".to_string(), + wikidata: "https://www.wikidata.org".to_string(), + commons: "https://commons.wikimedia.org".to_string(), + } + } + + #[cfg(test)] + pub fn localhost(port: u16) -> Self { + let base = format!("http://127.0.0.1:{port}"); + Self { + musicbrainz: base.clone(), + coverart: base.clone(), + wikidata: base.clone(), + commons: base, + } + } +} + +/// Per-endpoint rate limits, single-sourced here. MusicBrainz and +/// Cover Art Archive document 1 req/s and enforce it; Wikimedia image +/// serving tolerates more. +pub struct RateLimits { + pub musicbrainz: Limiter, + pub coverart: Limiter, + pub wikidata: Limiter, + pub commons: Limiter, + pub coverart_img: Limiter, +} + +impl RateLimits { + pub fn production() -> Self { + Self { + musicbrainz: Limiter::new(1.0), + coverart: Limiter::new(1.0), + wikidata: Limiter::new(1.0), + commons: Limiter::new(4.0), + coverart_img: Limiter::new(1.0), + } + } + + /// Tests only: no sleeping between requests. + #[cfg(test)] + pub fn unthrottled() -> Self { + Self { + musicbrainz: Limiter::new(1_000_000.0), + coverart: Limiter::new(1_000_000.0), + wikidata: Limiter::new(1_000_000.0), + commons: Limiter::new(1_000_000.0), + coverart_img: Limiter::new(1_000_000.0), + } + } +} + +/// The impure gather boundary. Owns no cache — returns raw outcomes. +pub struct MusicSources { + pub(crate) client: HttpClient, + pub(crate) limits: RateLimits, + bases: EndpointBases, +} + +impl MusicSources { + pub fn new() -> Self { + Self { + client: HttpClient::new(), + limits: RateLimits::production(), + bases: EndpointBases::production(), + } + } + + /// Tests only: all endpoints against one localhost stub, no rate + /// limiting. + #[cfg(test)] + pub fn for_tests(port: u16) -> Self { + Self { + client: HttpClient::new(), + limits: RateLimits::unthrottled(), + bases: EndpointBases::localhost(port), + } + } + + /// Run every query in the plan (parallel, per-endpoint limiters), + /// returning per-query outcomes. + pub fn run_queries(&self, plan: &ResolvePlan) -> Vec<(Query, Option)> { + plan.queries + .par_iter() + .map(|query| { + let outcome = match query { + Query::AlbumCover { + artist_display, + album_display, + mbid, + .. + } => { + let mbid = match mbid { + Some(mbid) => Some(mbid.clone()), + None => self.release_id(artist_display, album_display), + }; + mbid.and_then(|mbid| self.cover_art_url(&mbid)) + } + Query::AlbumUrl { + artist_display, + album_display, + mbid, + .. + } => { + let mbid = match mbid { + Some(mbid) => Some(mbid.clone()), + None => self.release_id(artist_display, album_display), + }; + mbid.and_then(|mbid| { + Href::from_str(&format!( + "{}/release/{}", + self.bases.musicbrainz, + mbid.as_ref() + )) + .ok() + }) + } + Query::ArtistImage { artist_display, .. } => self.artist_image(artist_display), + Query::ArtistUrl { artist_display, .. } => { + self.artist_id(artist_display).and_then(|mbid| { + Href::from_str(&format!( + "{}/artist/{}", + self.bases.musicbrainz, + mbid.as_ref() + )) + .ok() + }) + } + Query::TrackUrl { + artist_display, + track_display, + .. + } => self + .recording_id(artist_display, track_display) + .and_then(|mbid| { + Href::from_str(&format!( + "{}/recording/{}", + self.bases.musicbrainz, + mbid.as_ref() + )) + .ok() + }), + }; + (query.clone(), outcome) + }) + .collect() + } + + // ---------------------------------------------------- endpoints + + fn release_id(&self, artist: &str, album: &str) -> Option { + self.limits.musicbrainz.acquire(); + let query = format!( + "release:\"{}\" AND artist:\"{}\"", + clean_query(album), + clean_query(artist) + ); + let body = self.client.get_json( + &format!("{}/ws/2/release", self.bases.musicbrainz), + &[("query", &query), ("fmt", "json"), ("limit", "1")], + )?; + release_id_from_json(&body) + } + + /// Cover Art Archive front image for a release, 500px. A non-2xx + /// response (404 — no front art) is a permanent miss. + fn cover_art_url(&self, mbid: &MusicBrainzId) -> Option { + self.limits.coverart.acquire(); + let url = format!( + "{}/release/{}/front-500", + self.bases.coverart, + mbid.as_ref() + ); + self.client + .check(&url) + .then(|| Href::from_str(&url).ok()) + .flatten() + } + + fn artist_id(&self, name: &str) -> Option { + self.limits.musicbrainz.acquire(); + let query = format!("artist:\"{}\"", clean_query(name)); + let body = self.client.get_json( + &format!("{}/ws/2/artist", self.bases.musicbrainz), + &[("query", &query), ("fmt", "json"), ("limit", "1")], + )?; + artist_id_from_json(&body) + } + + /// MB artist lookup with URL relations → the artist's Wikidata QID. + fn wikidata_qid(&self, mbid: &MusicBrainzId) -> Option { + self.limits.musicbrainz.acquire(); + let body = self.client.get_json( + &format!("{}/ws/2/artist/{}", self.bases.musicbrainz, mbid.as_ref()), + &[("inc", "url-rels"), ("fmt", "json")], + )?; + wikidata_qid_from_json(&body) + } + + /// Wikidata `P18` (image) claim → Commons filename. + fn wikidata_p18(&self, qid: &WikidataId) -> Option { + self.limits.wikidata.acquire(); + let body = self.client.get_json( + &format!("{}/w/api.php", self.bases.wikidata), + &[ + ("action", "wbgetclaims"), + ("property", "P18"), + ("entity", qid.as_ref()), + ("format", "json"), + ], + )?; + p18_filename_from_json(&body) + } + + /// Commons image URL for a filename, resized to 600px. + fn commons_url(&self, filename: &CommonsFilename) -> Option { + self.limits.commons.acquire(); + Href::from_str(&format!( + "{}/wiki/Special:FilePath/{}?width=600", + self.bases.commons, + percent_encode(filename.as_ref()) + )) + .ok() + } + + /// Artist photo via the Wikidata chain: MB artist search → + /// url-rels → P18 → Commons URL. + fn artist_image(&self, artist_display: &str) -> Option { + let mbid = self.artist_id(artist_display)?; + let qid = self.wikidata_qid(&mbid)?; + let filename = self.wikidata_p18(&qid)?; + self.commons_url(&filename) + } + + fn recording_id(&self, artist: &str, track: &str) -> Option { + self.limits.musicbrainz.acquire(); + let query = format!( + "recording:\"{}\" AND artist:\"{}\"", + clean_query(track), + clean_query(artist) + ); + let body = self.client.get_json( + &format!("{}/ws/2/recording", self.bases.musicbrainz), + &[("query", &query), ("fmt", "json"), ("limit", "1")], + )?; + recording_id_from_json(&body) + } +} + +// ---------------------------------------------------- pure parsers + +/// MusicBrainz release search response → best-guess release MBID. +pub(crate) fn release_id_from_json(body: &Value) -> Option { + let id = body["releases"].as_array()?.first()?.get("id")?.as_str()?; + MusicBrainzId::from_str(id).ok() +} + +/// MusicBrainz artist search response → best-guess artist MBID. +pub(crate) fn artist_id_from_json(body: &Value) -> Option { + let id = body["artists"].as_array()?.first()?.get("id")?.as_str()?; + MusicBrainzId::from_str(id).ok() +} + +/// MusicBrainz recording search response → best-guess recording MBID. +pub(crate) fn recording_id_from_json(body: &Value) -> Option { + let id = body["recordings"] + .as_array()? + .first()? + .get("id")? + .as_str()?; + MusicBrainzId::from_str(id).ok() +} + +/// MB artist url-rels response → the artist's Wikidata QID. +pub(crate) fn wikidata_qid_from_json(body: &Value) -> Option { + for relation in body["relations"].as_array()? { + if relation["type"].as_str() == Some("wikidata") { + let url = relation["url"]["resource"].as_str()?; + // https://www.wikidata.org/wiki/Q130798 + return url + .rsplit('/') + .next() + .and_then(|q| WikidataId::from_str(q).ok()); + } + } + None +} + +/// Wikidata `wbgetclaims` response → the P18 Commons filename. +pub(crate) fn p18_filename_from_json(body: &Value) -> Option { + let name = body["claims"]["P18"] + .as_array()? + .first()? + .get("mainsnak")? + .get("datavalue")? + .get("value")? + .as_str()?; + CommonsFilename::from_str(name).ok() +} + +fn clean_query(s: &str) -> String { + s.replace(['"', '\\'], "") +} + +/// Minimal percent-encoding for the path component of a Commons URL. +fn percent_encode(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for b in s.bytes() { + match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + out.push(b as char) + } + _ => out.push_str(&format!("%{b:02X}")), + } + } + out +} + +// ---------------------------------------------------------------- tests + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::{AlbumKey, AlbumRef, ArtistKey, TrackKey, TrackRef}; + use serde_json::json; + use std::io::{Read, Write}; + use std::net::TcpListener; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + // -------------------------------------------- pure parsers + + #[test] + fn parsers_extract_ids_from_json() { + let release = + json!({ "releases": [{ "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", "title": "X" }] }); + assert_eq!( + release_id_from_json(&release).unwrap().as_ref(), + "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + ); + // Empty result → None, not a panic. + assert!(release_id_from_json(&json!({ "releases": [] })).is_none()); + assert!(release_id_from_json(&json!({})).is_none()); + // Invalid ID shape → None. + assert!(release_id_from_json(&json!({ "releases": [{ "id": "bogus" }] })).is_none()); + + let artist = json!({ "artists": [{ "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" }] }); + assert!(artist_id_from_json(&artist).is_some()); + assert!(artist_id_from_json(&json!({ "artists": [] })).is_none()); + + let recording = json!({ "recordings": [{ "id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" }] }); + assert!(recording_id_from_json(&recording).is_some()); + assert!(recording_id_from_json(&json!({ "recordings": [] })).is_none()); + } + + #[test] + fn qid_and_p18_parsers() { + let rels = json!({ + "relations": [ + { "type": "allmusic", "url": { "resource": "https://www.allmusic.com/artist/x" } }, + { "type": "wikidata", "url": { "resource": "https://www.wikidata.org/wiki/Q130798" } }, + ] + }); + assert_eq!(wikidata_qid_from_json(&rels).unwrap().as_ref(), "Q130798"); + assert!(wikidata_qid_from_json(&json!({ "relations": [] })).is_none()); + assert!(wikidata_qid_from_json( + &json!({ "relations": [{ "type": "allmusic", "url": { "resource": "https://x" } }] }) + ) + .is_none()); + + let claims = json!({ + "claims": { + "P18": [{ + "mainsnak": { "datavalue": { "value": "Mac Miller 2017.jpg" } } + }] + } + }); + assert_eq!( + p18_filename_from_json(&claims).unwrap().as_ref(), + "Mac Miller 2017.jpg" + ); + assert!(p18_filename_from_json(&json!({ "claims": {} })).is_none()); + assert!(p18_filename_from_json(&json!({ "claims": { "P18": [] } })).is_none()); + } + + // -------------------------------------------- stub server + + /// Serve canned responses, recording request lines. One request + /// per test keeps ordering trivially correct; a sequence reuses + /// the last response when exhausted. The thread owns a cloned + /// listener; the struct keeps the original so the port stays bound + /// for the test's lifetime. + struct Stub { + _listener: TcpListener, + requests: Arc, + } + + impl Stub { + fn serve(response: Vec) -> (Self, u16) { + Self::serve_sequence(vec![response]) + } + + fn serve_sequence(responses: Vec>) -> (Self, u16) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + let thread_listener = listener.try_clone().unwrap(); + let requests = Arc::new(AtomicUsize::new(0)); + let reqs = requests.clone(); + std::thread::spawn(move || { + for i in 0..16 { + let Ok((mut stream, _)) = thread_listener.accept() else { + return; + }; + reqs.fetch_add(1, Ordering::SeqCst); + let mut buf = [0u8; 4096]; + let _ = stream.read(&mut buf); + let response = &responses[i.min(responses.len() - 1)]; + let _ = stream.write_all(response); + } + }); + ( + Stub { + _listener: listener, + requests, + }, + port, + ) + } + + fn count(&self) -> usize { + self.requests.load(Ordering::SeqCst) + } + } + + fn http_response(status: &str, content_type: &str, body: &'static [u8]) -> Vec { + let mut resp = format!( + "HTTP/1.1 {status}\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .into_bytes(); + resp.extend_from_slice(body); + resp + } + + const OK_JSON: &[u8] = + br#"{"releases":[{"id":"aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee","title":"Flute"}]}"#; + + #[test] + fn roundtrip_release_search_and_cover_art() { + let (stub, port) = Stub::serve(http_response("200 OK", "application/json", OK_JSON)); + let sources = MusicSources::for_tests(port); + let plan = ResolvePlan { + resolved: vec![], + queries: vec![Query::AlbumCover { + album: AlbumRef { + artist: ArtistKey::from("a"), + album: AlbumKey::from("b"), + }, + artist_display: "New World Sound".into(), + album_display: "Flute".into(), + mbid: None, + provided: false, + }], + }; + let results = sources.run_queries(&plan); + assert_eq!(stub.count(), 2); // release search + cover art probe + let url = results[0].1.as_ref().unwrap().as_ref(); + assert!( + url.contains("/release/aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee/front-500"), + "{url}" + ); + } + + #[test] + fn roundtrip_cover_art_404_is_permanent_miss() { + let (stub, port) = Stub::serve(http_response("404 Not Found", "text/plain", b"nope")); + let sources = MusicSources::for_tests(port); + let plan = ResolvePlan { + resolved: vec![], + queries: vec![Query::AlbumCover { + album: AlbumRef { + artist: ArtistKey::from("a"), + album: AlbumKey::from("b"), + }, + artist_display: "New World Sound".into(), + album_display: "Flute".into(), + mbid: Some( + MusicBrainzId::from_str("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee").unwrap(), + ), + provided: true, + }], + }; + let results = sources.run_queries(&plan); + // Provided MBID → single cover-art probe, no search, no retries. + assert_eq!(stub.count(), 1); + assert!(results[0].1.is_none()); + } + + #[test] + fn roundtrip_artist_image_chain() { + let artist_json = br#"{"artists":[{"id":"bbbbbbbb-cccc-dddd-eeee-ffffffffffff"}]}"#; + let rels_json = br#"{"relations":[{"type":"wikidata","url":{"resource":"https://www.wikidata.org/wiki/Q130798"}}]}"#; + let claims_json = + br#"{"claims":{"P18":[{"mainsnak":{"datavalue":{"value":"Mac Miller 2017.jpg"}}}]}}"#; + let (stub, port) = Stub::serve_sequence(vec![ + http_response("200 OK", "application/json", artist_json), + http_response("200 OK", "application/json", rels_json), + http_response("200 OK", "application/json", claims_json), + ]); + let sources = MusicSources::for_tests(port); + let plan = ResolvePlan { + resolved: vec![], + queries: vec![Query::ArtistImage { + artist: ArtistKey::from("a"), + artist_display: "Mac Miller".into(), + }], + }; + let results = sources.run_queries(&plan); + assert_eq!(stub.count(), 3); + let url = results[0].1.as_ref().unwrap().as_ref(); + assert!( + url.contains("Special:FilePath/Mac%20Miller%202017.jpg?width=600"), + "{url}" + ); + } + + #[test] + fn roundtrip_recording_and_artist_urls() { + let recording_json = br#"{"recordings":[{"id":"cccccccc-dddd-eeee-ffff-000000000000"}]}"#; + let artist_json = br#"{"artists":[{"id":"dddddddd-eeee-ffff-0000-111111111111"}]}"#; + let (stub, port) = Stub::serve_sequence(vec![ + http_response("200 OK", "application/json", recording_json), + http_response("200 OK", "application/json", artist_json), + ]); + let sources = MusicSources::for_tests(port); + let plan = ResolvePlan { + resolved: vec![], + queries: vec![ + Query::TrackUrl { + track: TrackRef { + artist: ArtistKey::from("a"), + track: TrackKey::from("t"), + }, + artist_display: "New World Sound".into(), + track_display: "Flute".into(), + }, + Query::ArtistUrl { + artist: ArtistKey::from("a"), + artist_display: "New World Sound".into(), + }, + ], + }; + let results = sources.run_queries(&plan); + assert_eq!(stub.count(), 2); + assert!( + results[0] + .1 + .as_ref() + .unwrap() + .as_ref() + .contains("/recording/cccccccc-dddd-eeee-ffff-000000000000") + ); + assert!( + results[1] + .1 + .as_ref() + .unwrap() + .as_ref() + .contains("/artist/dddddddd-eeee-ffff-0000-111111111111") + ); + } +} diff --git a/tools/parse-plays/src/stats.rs b/tools/parse-plays/src/stats.rs index 6593e31..e8777da 100644 --- a/tools/parse-plays/src/stats.rs +++ b/tools/parse-plays/src/stats.rs @@ -2,12 +2,18 @@ //! //! No I/O here — everything is deterministic and testable with plain //! values. The imperative shell in `main` calls `aggregate`, then -//! `needed_pairs` (to know which covers to fetch), then `build_ranges` -//! with a cover lookup function supplied from outside. - +//! `needed` (to know which covers/links to resolve), then +//! `build_ranges` with the resolved data. Keys are typed newtypes +//! (`ArtistKey`/`AlbumRef`/`TrackRef`), so display names can never be +//! mistaken for keys. + +use crate::model::{ + AlbumKey, AlbumRef, ArtistKey, Href, MusicBrainzId, ResolvedStats, TrackKey, TrackRef, +}; use serde::Serialize; use serde_json::json; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; +use std::str::FromStr; pub const TOP_N: usize = 9; @@ -62,17 +68,21 @@ struct Counter { #[derive(Default)] pub struct Aggregated { /// range -> artist_key -> (counter, display name) - artists: HashMap>, - /// range -> (artist_key, album_key) -> album entry - albums: HashMap>, - /// range -> (artist_key, track_key) -> (counter, artist display, track display, album_key) - tracks: HashMap>, + artists: HashMap>, + /// range -> album_ref -> album entry + albums: HashMap>, + /// range -> track_ref -> (counter, artist display, track display, album key) + tracks: HashMap>, /// artist_key -> (top album key, plays) — used to proxy artist cover art. - top_albums: HashMap, + top_albums: HashMap, /// artist_key -> display name - artist_names: HashMap, + artist_names: HashMap, } +/// One track's accumulated stats: the counter, the display names, and +/// which album (if any) its cover falls back to. +type TrackCounter = (Counter, String, String, AlbumKey); + /// One album's accumulated stats. `release_mbid` is carried through /// from the play records (piper/lazuli emit it on recent plays) so /// cover resolution can hit Cover Art Archive directly instead of @@ -82,26 +92,48 @@ pub struct AlbumEntry { counter: Counter, artist_display: String, album_display: String, - release_mbid: Option, + release_mbid: Option, +} + +/// What the top-N grids actually reference: the keys resolution works +/// on, plus the display names queries are built from. +pub struct AlbumNeed { + pub album: AlbumRef, + pub artist_display: String, + pub album_display: String, + pub mbid: Option, +} + +pub struct ArtistNeed { + pub key: ArtistKey, + pub display: String, +} + +pub struct TrackNeed { + pub track: TrackRef, + pub artist_display: String, + pub track_display: String, } -/// Strip the `mbid:` prefix some clients add to MBID fields. -pub fn clean_mbid(s: &str) -> Option { - let id = s.rsplit(':').next()?.to_string(); - (!id.is_empty()).then_some(id) +/// Everything one refresh needs resolved, deduplicated across ranges. +#[derive(Default)] +pub struct Needs { + pub albums: Vec, + pub artists: Vec, + pub tracks: Vec, } #[derive(Serialize)] struct StatsItem { name: String, - #[serde(skip_serializing_if = "String::is_empty")] - artist: String, + #[serde(skip_serializing_if = "Option::is_none")] + artist: Option, plays: u64, ms_played: u64, - #[serde(skip_serializing_if = "String::is_empty")] - image: String, - #[serde(skip_serializing_if = "String::is_empty")] - url: String, + #[serde(skip_serializing_if = "Option::is_none")] + image: Option, + #[serde(skip_serializing_if = "Option::is_none")] + url: Option, } #[derive(Serialize)] @@ -111,67 +143,6 @@ struct RangeStats { tracks: Vec, } -// ---------------------------------------------------------------- helpers - -/// Sanitized grouping key for artist names: lowercase, punctuation -/// dropped, collaboration credit cut to the first artist, junction -/// words (feat/with/&/vs...) removed. Two spellings of the same -/// artist ("JAY-Z" vs "Jay Z", "A$AP" vs "ASAP") land in the same -/// bucket; display names keep their original spelling. -pub fn normalize(s: &str) -> String { - const JUNCTION_WORDS: [&str; 12] = [ - "and", - "with", - "feat", - "featuring", - "ft", - "vs", - "versus", - "present", - "presents", - "pres", - "b2b", - "x", - ]; - - let lower = s.to_lowercase(); - // Collab credit strings: keep only the part before the first - // comma/ampersand/plus — "tofubeats, HITOMITOI" groups as tofubeats. - let head = lower - .split(['&', '+', ',']) - .next() - .unwrap_or(lower.as_str()); - head - // Any run of non-alphanumerics is a word boundary ("JAY-Z", - // "A$AP", "Café" -> "caf", accents are dropped with them). - .split(|c: char| !c.is_alphanumeric()) - .take_while(|word| !JUNCTION_WORDS.contains(word)) - .filter(|word| !word.is_empty()) - .collect::>() - .join(" ") -} - -/// Plain key for album/track name components: trim + lowercase only. -fn normalize_name(s: &str) -> String { - s.trim().to_lowercase() -} - -/// Resolve the display name of an album from the all-time album map. -fn album_display_for(agg: &Aggregated, artist_key: &str, album_key: &str) -> Option { - agg.albums - .get(&Range::AllTime)? - .get(&(artist_key.to_string(), album_key.to_string())) - .map(|entry| entry.album_display.clone()) -} - -/// The most recent MusicBrainz release ID for an album, if any. -fn album_mbid_for(agg: &Aggregated, artist_key: &str, album_key: &str) -> Option { - agg.albums - .get(&Range::AllTime)? - .get(&(artist_key.to_string(), album_key.to_string())) - .and_then(|entry| entry.release_mbid.clone()) -} - // ---------------------------------------------------------------- aggregate /// Aggregate all plays into per-range counters. Plays that fail to @@ -198,17 +169,19 @@ pub fn aggregate(plays: &[serde_json::Value], today: chrono::NaiveDate) -> Aggre continue; } - let artist_key = normalize(artist_display); - let track_key = normalize_name(track_display); + let artist_key = ArtistKey::from(artist_display); + let track_key = TrackKey::from(track_display); let ms_played = play["msPlayed"].as_u64().unwrap_or(0); let (album_key, album_display) = match play["releaseName"].as_str() { Some(name) if !name.trim().is_empty() => { - (normalize_name(name), name.trim().to_string()) + (AlbumKey::from(name), name.trim().to_string()) } - _ => (String::new(), String::new()), + _ => (AlbumKey::from(""), String::new()), }; - let release_mbid = play["releaseMbId"].as_str().and_then(clean_mbid); + let release_mbid = play["releaseMbId"] + .as_str() + .and_then(|s| MusicBrainzId::from_str(s).ok()); // Bucket by age; unparseable dates count as all-time only. let age_days = play["playedTime"] @@ -239,12 +212,15 @@ pub fn aggregate(plays: &[serde_json::Value], today: chrono::NaiveDate) -> Aggre artist.0.plays += 1; artist.0.ms_played += ms_played; - if !album_key.is_empty() { + if !album_key.as_ref().is_empty() { let album = agg .albums .entry(range) .or_default() - .entry((artist_key.clone(), album_key.clone())) + .entry(AlbumRef { + artist: artist_key.clone(), + album: album_key.clone(), + }) .or_insert_with(|| AlbumEntry { counter: Counter::default(), artist_display: artist_display.to_string(), @@ -259,12 +235,15 @@ pub fn aggregate(plays: &[serde_json::Value], today: chrono::NaiveDate) -> Aggre } } - if !track_key.is_empty() { + if !track_key.as_ref().is_empty() { let track = agg .tracks .entry(range) .or_default() - .entry((artist_key.clone(), track_key.clone())) + .entry(TrackRef { + artist: artist_key.clone(), + track: track_key.clone(), + }) .or_insert_with(|| { ( Counter::default(), @@ -282,13 +261,13 @@ pub fn aggregate(plays: &[serde_json::Value], today: chrono::NaiveDate) -> Aggre // Per artist, remember the album with the most plays — its cover // proxies the artist tile. if let Some(map) = agg.albums.get(&Range::AllTime) { - for ((artist_key, album_key), entry) in map { + for (album_ref, entry) in map { let record = agg .top_albums - .entry(artist_key.clone()) - .or_insert_with(|| (album_key.clone(), 0)); + .entry(album_ref.artist.clone()) + .or_insert_with(|| (album_ref.album.clone(), 0)); if entry.counter.plays > record.1 { - *record = (album_key.clone(), entry.counter.plays); + *record = (album_ref.album.clone(), entry.counter.plays); } } } @@ -317,119 +296,151 @@ where } /// Top-N keys by (plays, ms_played), for picking which covers we need. +/// The key itself breaks ties so the selection is deterministic — +/// HashMap iteration order is random per process, and at a tied +/// top-N boundary it would otherwise pick a different set every run. fn top_keys<'a, T>(entries: impl Iterator) -> Vec where - T: Clone, + T: Clone + Ord, { let mut list: Vec<(&Counter, T)> = entries.map(|(key, counter)| (counter, key)).collect(); list.sort_by(|a, b| { b.0.plays .cmp(&a.0.plays) .then_with(|| b.0.ms_played.cmp(&a.0.ms_played)) + .then_with(|| a.1.cmp(&b.1)) }); list.into_iter().take(TOP_N).map(|(_, key)| key).collect() } -/// Top-N artist display names across all ranges — the set we need -/// real artist images for. Deduplicated. -pub fn needed_artists(agg: &Aggregated) -> Vec { - use std::collections::HashSet; +// ---------------------------------------------------------------- needs + +/// The (deduplicated) set of keys the top-N grids reference — what the +/// resolution pipeline must fetch. Display names and MBIDs are carried +/// for building queries; `album`/`artist`/`track` are the typed keys +/// the resolved maps are keyed by. +pub fn needed(agg: &Aggregated) -> Needs { + let mut album_seen: HashSet = HashSet::new(); + let mut artist_seen: HashSet = HashSet::new(); + let mut track_seen: HashSet = HashSet::new(); - let mut seen: HashSet = HashSet::new(); for range in Range::ALL { + if let Some(map) = agg.albums.get(&range) { + for key in top_keys(map.iter().map(|(k, v)| (k.clone(), &v.counter))) { + album_seen.insert(key); + } + } + if let Some(map) = agg.tracks.get(&range) { + for key in top_keys(map.iter().map(|(k, v)| (k.clone(), &v.0))) { + if let Some((_, _, _, album_key)) = map.get(&key) + && !album_key.as_ref().is_empty() + { + album_seen.insert(AlbumRef { + artist: key.artist.clone(), + album: album_key.clone(), + }); + } + track_seen.insert(key); + } + } if let Some(map) = agg.artists.get(&range) { - for artist_key in top_keys(map.iter().map(|(key, value)| (key, &value.0))) { - if let Some(display) = agg.artist_names.get(artist_key) { - seen.insert(display.clone()); + for artist_key in top_keys(map.iter().map(|(k, v)| (k.clone(), &v.0))) { + if let Some((album_key, _)) = agg.top_albums.get(&artist_key) { + album_seen.insert(AlbumRef { + artist: artist_key.clone(), + album: album_key.clone(), + }); } + artist_seen.insert(artist_key); } } } - seen.into_iter().collect() -} -/// Top-N (artist, track) display-name pairs across all ranges — the -/// set we need MusicBrainz recording links for. Deduplicated. -pub fn needed_tracks(agg: &Aggregated) -> Vec<(String, String)> { - use std::collections::HashSet; + let all_albums = agg.albums.get(&Range::AllTime); + let mut albums = Vec::new(); + for album_ref in album_seen { + let entry = all_albums.and_then(|m| m.get(&album_ref)); + if let (Some(artist_display), Some(entry)) = + (agg.artist_names.get(&album_ref.artist), entry) + { + albums.push(AlbumNeed { + album: album_ref, + artist_display: artist_display.clone(), + album_display: entry.album_display.clone(), + mbid: entry.release_mbid.clone(), + }); + } + } - let mut seen: HashSet<(String, String)> = HashSet::new(); - for range in Range::ALL { - if let Some(map) = agg.tracks.get(&range) { - for (artist_key, track_key) in top_keys(map.iter().map(|(key, value)| (key, &value.0))) - { - seen.insert((artist_key.clone(), track_key.clone())); - } + let mut artists = Vec::new(); + for key in artist_seen { + if let Some(display) = agg.artist_names.get(&key) { + artists.push(ArtistNeed { + key, + display: display.clone(), + }); } } // Display names live in the all-time map, which contains every // track ever played (so any range top is present there). let all_tracks = agg.tracks.get(&Range::AllTime); - let mut out = Vec::new(); - for (artist_key, track_key) in seen { + let mut tracks = Vec::new(); + for track_ref in track_seen { if let Some((_, artist_display, track_display, _)) = - all_tracks.and_then(|map| map.get(&(artist_key.clone(), track_key.clone()))) + all_tracks.and_then(|m| m.get(&track_ref)) { - out.push((artist_display.clone(), track_display.clone())); + tracks.push(TrackNeed { + track: track_ref, + artist_display: artist_display.clone(), + track_display: track_display.clone(), + }); } } - out + + Needs { + albums, + artists, + tracks, + } } -// ---------------------------------------------------------------- covers +// ---------------------------------------------------------------- serialize /// Cover art URL for an artist tile: the artist's own image when -/// resolved (via `artist_cover`), falling back to the cover of its -/// most-played album. +/// resolved, falling back to the cover of its most-played album. fn artist_image( agg: &Aggregated, - artist_key: &str, - cover: &dyn Fn(&str, &str) -> String, - artist_cover: &dyn Fn(&str) -> String, -) -> String { - let Some(artist_display) = agg.artist_names.get(artist_key) else { - return String::new(); - }; - let direct = artist_cover(artist_display); - if !direct.is_empty() { - return direct; + artist_key: &ArtistKey, + resolved: &ResolvedStats, +) -> Option { + if let Some(url) = resolved.artist_cover.get(artist_key) { + return Some(url.clone()); } - let Some((album_key, _)) = agg.top_albums.get(artist_key) else { - return String::new(); - }; - let Some(album_display) = album_display_for(agg, artist_key, album_key) else { - return String::new(); - }; - cover(artist_display, &album_display) -} - -/// Lookup closures handed to `build_ranges`: image URLs and page -/// links for the top-N entries. Keys are normalized display names, so -/// the callers resolve case/spelling variants the same way the -/// aggregation does. -pub struct Lookups<'a> { - pub cover: &'a dyn Fn(&str, &str) -> String, - pub artist_cover: &'a dyn Fn(&str) -> String, - pub album_url: &'a dyn Fn(&str, &str) -> String, - pub artist_url: &'a dyn Fn(&str) -> String, - pub track_url: &'a dyn Fn(&str, &str) -> String, + let (album_key, _) = agg.top_albums.get(artist_key)?; + resolved + .cover + .get(&AlbumRef { + artist: artist_key.clone(), + album: album_key.clone(), + }) + .cloned() } /// Build the top-N lists for one range. -fn build_range_stats(agg: &Aggregated, range: Range, lookups: &Lookups) -> RangeStats { +fn build_range_stats(agg: &Aggregated, range: Range, resolved: &ResolvedStats) -> RangeStats { let artists = agg.artists.get(&range).map_or(vec![], |map| { sort_top( map.iter().map(|(key, value)| (key, &value.0)), |artist_key| { - let (counter, display) = &map[&artist_key.clone()]; + let (counter, display) = &map[artist_key]; StatsItem { name: display.clone(), - artist: String::new(), + artist: None, plays: counter.plays, ms_played: counter.ms_played, - image: artist_image(agg, artist_key, lookups.cover, lookups.artist_cover), - url: (lookups.artist_url)(display), + image: artist_image(agg, artist_key, resolved), + url: resolved.artist_url.get(artist_key).cloned(), } }, ) @@ -438,15 +449,15 @@ fn build_range_stats(agg: &Aggregated, range: Range, lookups: &Lookups) -> Range let albums = agg.albums.get(&range).map_or(vec![], |map| { sort_top( map.iter().map(|(key, value)| (key, &value.counter)), - |(artist_key, album_key)| { - let entry = &map[&(artist_key.clone(), album_key.clone())]; + |album_ref| { + let entry = &map[album_ref]; StatsItem { name: entry.album_display.clone(), - artist: entry.artist_display.clone(), + artist: Some(entry.artist_display.clone()), plays: entry.counter.plays, ms_played: entry.counter.ms_played, - image: (lookups.cover)(&entry.artist_display, &entry.album_display), - url: (lookups.album_url)(&entry.artist_display, &entry.album_display), + image: resolved.cover.get(album_ref).cloned(), + url: resolved.album_url.get(album_ref).cloned(), } }, ) @@ -455,22 +466,26 @@ fn build_range_stats(agg: &Aggregated, range: Range, lookups: &Lookups) -> Range let tracks = agg.tracks.get(&range).map_or(vec![], |map| { sort_top( map.iter().map(|(key, value)| (key, &value.0)), - |(artist_key, track_key)| { - let (counter, artist_display, track_display, album_key) = - &map[&(artist_key.clone(), track_key.clone())]; - let image = if album_key.is_empty() { - String::new() + |track_ref| { + let (counter, artist_display, track_display, album_key) = &map[track_ref]; + let image = if album_key.as_ref().is_empty() { + None } else { - album_display_for(agg, artist_key, album_key) - .map_or(String::new(), |ad| (lookups.cover)(artist_display, &ad)) + resolved + .cover + .get(&AlbumRef { + artist: track_ref.artist.clone(), + album: album_key.clone(), + }) + .cloned() }; StatsItem { name: track_display.clone(), - artist: artist_display.clone(), + artist: Some(artist_display.clone()), plays: counter.plays, ms_played: counter.ms_played, image, - url: (lookups.track_url)(artist_display, track_display), + url: resolved.track_url.get(track_ref).cloned(), } }, ) @@ -483,64 +498,13 @@ fn build_range_stats(agg: &Aggregated, range: Range, lookups: &Lookups) -> Range } } -/// (artist, album, optional MusicBrainz release ID) triples whose -/// covers the top-N grids actually reference — derived from the -/// trimmed top-N of each range, never the full history. The MBID is -/// carried through so cover resolution can skip the search. -pub fn needed_pairs(agg: &Aggregated) -> Vec<(String, String, Option)> { - use std::collections::HashSet; - - let mut seen: HashSet<(String, String)> = HashSet::new(); - - for range in Range::ALL { - if let Some(map) = agg.albums.get(&range) { - for (artist_key, album_key) in - top_keys(map.iter().map(|(key, value)| (key, &value.counter))) - { - seen.insert((artist_key.clone(), album_key.clone())); - } - } - if let Some(map) = agg.tracks.get(&range) { - for (artist_key, track_key) in top_keys(map.iter().map(|(key, value)| (key, &value.0))) - { - if let Some((_, _, _, album_key)) = - map.get(&(artist_key.clone(), track_key.clone())) - { - if !album_key.is_empty() { - seen.insert((artist_key.clone(), album_key.clone())); - } - } - } - } - if let Some(map) = agg.artists.get(&range) { - for artist_key in top_keys(map.iter().map(|(key, value)| (key, &value.0))) { - if let Some((album_key, _)) = agg.top_albums.get(artist_key) { - seen.insert((artist_key.clone(), album_key.clone())); - } - } - } - } - - let mut pairs = Vec::new(); - for (artist_key, album_key) in seen { - if let Some(artist_display) = agg.artist_names.get(&artist_key) { - if let Some(album_display) = album_display_for(agg, &artist_key, &album_key) { - let mbid = album_mbid_for(agg, &artist_key, &album_key); - pairs.push((artist_display.clone(), album_display, mbid)); - } - } - } - pairs -} - -// ---------------------------------------------------------------- serialize - -/// Serialize every range's top-N grids. `lookups` maps display names -/// to cover art, artist images, and page links (or "" when none). -pub fn build_ranges(agg: &Aggregated, lookups: &Lookups) -> serde_json::Value { +/// Serialize every range's top-N grids. `resolved` carries the cover +/// art, artist images, and page links (or `None` when absent — the +/// serialized JSON omits those keys entirely). +pub fn build_ranges(agg: &Aggregated, resolved: &ResolvedStats) -> serde_json::Value { let mut ranges = serde_json::Map::new(); for range in Range::ALL { - let rs = build_range_stats(agg, range, lookups); + let rs = build_range_stats(agg, range, resolved); ranges.insert( range.key().to_string(), serde_json::to_value(rs).expect("serialize range stats"), @@ -554,6 +518,7 @@ pub fn build_ranges(agg: &Aggregated, lookups: &Lookups) -> serde_json::Value { #[cfg(test)] mod tests { use super::*; + use crate::model::{AlbumKey, ArtistKey, normalize, normalize_name}; fn play(track: &str, artist: &str, album: &str, date: &str, ms: u64) -> serde_json::Value { json!({ @@ -565,25 +530,6 @@ mod tests { }) } - #[test] - fn normalize_collapses_spelling_variants() { - assert_eq!(normalize("JAY-Z"), "jay z"); - assert_eq!(normalize("Jay Z"), "jay z"); - // Punctuation is a word boundary: A$AP splits, but every - // spelling of "A$AP ..." still lands in the same bucket. - assert_eq!(normalize("A$AP Rocky"), "a ap rocky"); - assert_eq!(normalize("A$AP Rocky"), normalize("a$ap-rocky")); - assert_eq!(normalize("US3"), "us3"); - assert_eq!(normalize("us3"), "us3"); - // Collaboration credit cuts to the first artist... - assert_eq!(normalize("JAY-Z feat. Beyoncé"), "jay z"); - assert_eq!(normalize("tofubeats, HITOMITOI"), "tofubeats"); - assert_eq!(normalize("Drake & 21 Savage"), "drake"); - assert_eq!(normalize("Freddie Gibbs vs Madlib"), "freddie gibbs"); - // Single-token names pass through untouched. - assert_eq!(normalize("XXXTENTACION"), "xxxtentacion"); - } - #[test] fn aggregates_and_buckets_by_range() { let today = chrono::NaiveDate::from_ymd_opt(2026, 8, 3).unwrap(); @@ -597,24 +543,30 @@ mod tests { let agg = aggregate(&plays, today); let all = agg.artists.get(&Range::AllTime).unwrap(); - assert_eq!(all.get("artist a").unwrap().0.plays, 3); - assert_eq!(all.get("artist b").unwrap().0.plays, 1); - assert_eq!(all.get("artist c").unwrap().0.plays, 1); + assert_eq!(all.get(&ArtistKey::from("Artist A")).unwrap().0.plays, 3); + assert_eq!(all.get(&ArtistKey::from("Artist B")).unwrap().0.plays, 1); + assert_eq!(all.get(&ArtistKey::from("Artist C")).unwrap().0.plays, 1); // 1m: only Artist A (plays within 30 days of Aug 3). let one_m = agg.artists.get(&Range::OneMonth).unwrap(); assert_eq!(one_m.len(), 1); - assert_eq!(one_m.get("artist a").unwrap().0.plays, 2); + assert_eq!(one_m.get(&ArtistKey::from("Artist A")).unwrap().0.plays, 2); // 6m: A (2 in-range) + B (1). let six_m = agg.artists.get(&Range::SixMonths).unwrap(); assert_eq!(six_m.len(), 2); // Album with most plays per artist. - assert_eq!(agg.top_albums.get("artist a").unwrap().0, "album a"); + assert_eq!( + agg.top_albums.get(&ArtistKey::from("Artist A")).unwrap().0, + AlbumKey::from("Album A") + ); // msPlayed accumulates. - assert_eq!(all.get("artist a").unwrap().0.ms_played, 7000); + assert_eq!( + all.get(&ArtistKey::from("Artist A")).unwrap().0.ms_played, + 7000 + ); } #[test] @@ -632,18 +584,113 @@ mod tests { }) .collect(); let agg = aggregate(&plays, today); - let empty = |_: &str| String::new(); - let empty_pair = |_: &str, _: &str| String::new(); - let lookups = Lookups { - cover: &empty_pair, - artist_cover: &empty, - album_url: &empty_pair, - artist_url: &empty, - track_url: &empty_pair, - }; - let rs = build_range_stats(&agg, Range::OneMonth, &lookups); + let resolved = ResolvedStats::default(); + let rs = build_range_stats(&agg, Range::OneMonth, &resolved); assert_eq!(rs.artists.len(), TOP_N); assert_eq!(rs.albums.len(), TOP_N); assert_eq!(rs.tracks.len(), TOP_N); } + + #[test] + fn build_ranges_emits_exact_json_with_optional_fields() { + let today = chrono::NaiveDate::from_ymd_opt(2026, 8, 3).unwrap(); + let plays = vec![ + play("Alpha", "Artist A", "Album A", "2026-07-20T10:00:00Z", 1000), + play("Beta", "Artist B", "Album B", "2026-02-10T10:00:00Z", 3000), + ]; + let agg = aggregate(&plays, today); + + let mut resolved = ResolvedStats::default(); + resolved.cover.insert( + AlbumRef { + artist: ArtistKey::from("Artist A"), + album: AlbumKey::from("Album A"), + }, + Href::from_str("https://example.com/cover.jpg").unwrap(), + ); + resolved.artist_url.insert( + ArtistKey::from("Artist A"), + Href::from_str("https://musicbrainz.org/artist/abc").unwrap(), + ); + + let value = build_ranges(&agg, &resolved); + + // Artist tile: no `artist` field; no direct artist image, so + // the tile falls back to its top album's cover; url present. + let artist = value["all"]["artists"] + .as_array() + .unwrap() + .iter() + .find(|a| a["name"] == "Artist A") + .unwrap(); + assert_eq!(artist["name"], "Artist A"); + assert!(artist.get("artist").is_none()); + assert_eq!(artist["url"], "https://musicbrainz.org/artist/abc"); + assert_eq!(artist["image"], "https://example.com/cover.jpg"); + + // Album tile: artist present, cover present, no url. + let album = value["all"]["albums"] + .as_array() + .unwrap() + .iter() + .find(|a| a["name"] == "Album A") + .unwrap(); + assert_eq!(album["artist"], "Artist A"); + assert_eq!(album["image"], "https://example.com/cover.jpg"); + assert!(album.get("url").is_none()); + + // The unresolved artist's tile carries no image/url keys at all. + let artist_b = value["all"]["artists"] + .as_array() + .unwrap() + .iter() + .find(|a| a["name"] == "Artist B") + .unwrap(); + assert_eq!(artist_b["name"], "Artist B"); + assert!(artist_b.get("image").is_none()); + assert!(artist_b.get("url").is_none()); + } + + #[test] + fn needed_dedupes_across_ranges_and_carries_mbids() { + let today = chrono::NaiveDate::from_ymd_opt(2026, 8, 3).unwrap(); + let mut plays = vec![ + play("Alpha", "Artist A", "Album A", "2026-07-20T10:00:00Z", 1000), + play("Alpha", "Artist A", "Album A", "2026-01-20T10:00:00Z", 1000), + play("Beta", "Artist B", "Album B", "2026-01-20T10:00:00Z", 1000), + ]; + // Give Artist A's album an MBID on the most recent play. + plays[0]["releaseMbId"] = json!("mbid:aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"); + let agg = aggregate(&plays, today); + + let needs = needed(&agg); + // Album A (with MBID) and Album B, each once. + assert_eq!(needs.albums.len(), 2); + let album_a = needs + .albums + .iter() + .find(|n| n.album_display == "Album A") + .unwrap(); + assert_eq!( + album_a.mbid.as_ref().unwrap().as_ref(), + "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + ); + // Artists and tracks deduplicated. + assert_eq!(needs.artists.len(), 2); + assert_eq!(needs.tracks.len(), 2); + let artist_a = needs + .artists + .iter() + .find(|n| n.display == "Artist A") + .unwrap(); + assert_eq!(artist_a.key, ArtistKey::from("Artist A")); + } + + #[test] + fn normalize_collapses_spelling_variants() { + // The derivation lives in model; a thin smoke check that stats + // and keys share it. + assert_eq!(normalize("JAY-Z"), normalize("Jay Z")); + assert_eq!(normalize_name(" Animals "), "animals"); + } }