diff --git a/.dockerignore b/.dockerignore --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,5 @@ .git +.jj .DS_Store ._.DS_Store .direnv/ @@ -37,3 +38,6 @@ docker-compose.*.yml **/Dockerfile **/*.Dockerfile + +knot2/pack-spike +**/src/_lex diff --git a/Cargo.lock b/Cargo.lock --- a/Cargo.lock +++ b/Cargo.lock @@ -9,6 +9,43 @@ checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] +name = "aead" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" +dependencies = [ + "crypto-common 0.2.2", + "inout", +] + +[[package]] +name = "aes" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1fc76eaeac4c9164506c466d4ffdd8ec9d0c5bf57ee97177c4d8eceb3a0e138" +dependencies = [ + "cipher", + "cpubits", + "cpufeatures 0.3.0", + "zeroize", +] + +[[package]] +name = "aes-gcm" +version = "0.11.0-rc.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da8c919c118108f144adecad74b425b804ad075580d605d9b33c2d6d1c62a2f8" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", + "zeroize", +] + +[[package]] name = "ahash" version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -31,10 +68,19 @@ ] [[package]] -name = "aliasable" -version = "0.1.3" +name = "alloc-no-stdlib" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] [[package]] name = "alloca" @@ -149,6 +195,91 @@ ] [[package]] +name = "argon2" +version = "0.6.0-rc.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7af50940b73bf4e16c15c448a2b121c63f2d68e3e54b6a8731673cb4aa0cdff5" +dependencies = [ + "base64ct", + "blake2", + "cpufeatures 0.3.0", + "password-hash", +] + +[[package]] +name = "arrayvec" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f02882884d3e1bc524fb12c79f107f6ad0e1cfd498c536ffb494301740995dfe" + +[[package]] +name = "asn1-rs" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5493c3bedbacf7fd7382c6346bbd66687d12bbaad3a89a2d2c303ee6cf20b048" +dependencies = [ + "asn1-rs-derive 0.5.1", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive 0.6.0", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", + "synstructure", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] name = "assert-json-diff" version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -156,6 +287,18 @@ dependencies = [ "serde", "serde_json", +] + +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", ] [[package]] @@ -171,6 +314,65 @@ ] [[package]] +name = "async-http-codec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "096146020b08dbc4587685b0730a7ba905625af13c65f8028035cdfd69573c91" +dependencies = [ + "anyhow", + "futures", + "http", + "httparse", + "log", +] + +[[package]] +name = "async-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" +dependencies = [ + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-net" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b948000fad4873c1c9339d60f2623323a0cfd3816e5181033c6a5cb68b2accf7" +dependencies = [ + "async-io", + "blocking", + "futures-lite", +] + +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + +[[package]] name = "async-trait" version = "0.1.89" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -178,7 +380,26 @@ dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", +] + +[[package]] +name = "async-web-client" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8caf502b44d6d4be6154ac33af012cbb5fef11e6066edcfb42834217fbaf501b" +dependencies = [ + "async-http-codec", + "async-net", + "futures", + "futures-rustls", + "http", + "lazy_static", + "log", + "rustls-pki-types", + "serde", + "thiserror 1.0.69", + "webpki-roots 0.26.11", ] [[package]] @@ -203,12 +424,36 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] +name = "aws-lc-rs" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +dependencies = [ + "aws-lc-sys", + "untrusted 0.7.1", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + +[[package]] name = "axum" version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ "axum-core", + "base64", "bytes", "form_urlencoded", "futures-util", @@ -227,8 +472,10 @@ "serde_json", "serde_path_to_error", "serde_urlencoded", + "sha1 0.10.6", "sync_wrapper", "tokio", + "tokio-tungstenite 0.29.0", "tower", "tower-layer", "tower-service", @@ -267,6 +514,12 @@ checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" [[package]] +name = "base16ct" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd307490d624467aa6f74b0eabb77633d1f758a7b25f12bceb0b22e08d9726f6" + +[[package]] name = "base256emoji" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -275,6 +528,12 @@ "const-str", "match-lookup", ] + +[[package]] +name = "base32" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "022dfe9eb35f19ebbcb51e0b40a5ab759f46ad60cadf7297e0bd085afb50e076" [[package]] name = "base64" @@ -289,6 +548,17 @@ checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" [[package]] +name = "bcrypt-pbkdf" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "144e573728da132683b9488acd528274c790e07fc06ff81ee29f9d8f8b1041e0" +dependencies = [ + "blowfish", + "pbkdf2", + "sha2 0.11.0", +] + +[[package]] name = "better_any" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -300,7 +570,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" dependencies = [ - "bit-vec", + "bit-vec 0.8.0", ] [[package]] @@ -310,10 +580,25 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] -name = "bitflags" -version = "2.11.1" +name = "bit-vec" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +dependencies = [ + "serde", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" [[package]] name = "bitpacking" @@ -325,12 +610,63 @@ ] [[package]] +name = "blake2" +version = "0.11.0-rc.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "061f1a09225e328e1ffbb378d2d49923c0ca5fee19fb5ac1cc9c1e9d52b93690" +dependencies = [ + "digest 0.11.3", +] + +[[package]] name = "block-buffer" version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "generic-array", + "generic-array 0.14.9", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", + "zeroize", +] + +[[package]] +name = "block-padding" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "710f1dd022ef4e93f8a438b4ba958de7f64308434fa6a87104481645cc30068b" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "blocking" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21" +dependencies = [ + "async-channel", + "async-task", + "futures-io", + "futures-lite", + "piper", +] + +[[package]] +name = "blowfish" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62ce3946557b35e71d1bbe07ec385073ce9eda05043f95de134eb578fcf1a298" +dependencies = [ + "byteorder", + "cipher", ] [[package]] @@ -421,7 +757,7 @@ "futures", "http", "jacquard-common", - "reqwest", + "reqwest 0.13.1", "serde", "serde_json", "thiserror 2.0.18", @@ -441,7 +777,7 @@ "futures", "http", "jacquard-common", - "reqwest", + "reqwest 0.13.1", "scc", "thiserror 2.0.18", "tokio", @@ -483,9 +819,9 @@ "ahash", "bytes", "futures", - "getrandom 0.3.4", + "getrandom 0.4.3", "http", - "reqwest", + "reqwest 0.13.1", "thiserror 2.0.18", "tokio", "tokio-tungstenite 0.29.0", @@ -542,7 +878,7 @@ "futures", "http", "jacquard-common", - "reqwest", + "reqwest 0.13.1", "serde", "serde_json", "thiserror 2.0.18", @@ -591,7 +927,7 @@ "thiserror 2.0.18", "tokio", "tower", - "tower-http", + "tower-http 0.7.0", "tracing", "url", "wiremock", @@ -599,9 +935,9 @@ [[package]] name = "bon" -version = "3.9.1" +version = "3.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f47dbe92550676ee653353c310dfb9cf6ba17ee70396e1f7cf0a2020ad49b2fe" +checksum = "a602c73c7b0148ec6d12af6fd5cc7a46e2eacc8878271a999abac56eed12f561" dependencies = [ "bon-macros", "rustversion", @@ -609,9 +945,9 @@ [[package]] name = "bon-macros" -version = "3.9.1" +version = "3.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c" +checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" dependencies = [ "darling", "ident_case", @@ -619,7 +955,7 @@ "proc-macro2", "quote", "rustversion", - "syn", + "syn 2.0.118", ] [[package]] @@ -630,12 +966,33 @@ [[package]] name = "borsh" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd1e3f8955a5d7de9fab72fc8373fade9fb8a703968cb200ae3dc6cf08e185a" +checksum = "2f3f6da4992df95bbcd9af42a6c7dcb994498fc9048230405f3b36ff7cd3f145" dependencies = [ "bytes", "cfg_aliases", +] + +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", ] [[package]] @@ -645,6 +1002,17 @@ checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" dependencies = [ "tinyvec", +] + +[[package]] +name = "bstr" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +dependencies = [ + "memchr", + "regex-automata", + "serde", ] [[package]] @@ -661,18 +1029,33 @@ [[package]] name = "bytes" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" dependencies = [ "serde", ] + +[[package]] +name = "bytesize" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49e78e506b9d7633710dab98996f22f95f3d0f488e8f1aa162830556ed9fc14d" [[package]] name = "cast" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cbc" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2dc9ee5f88d11e0beb842c88b33c8a5cf0d1329c4b19494af42b07dbfe8896" +dependencies = [ + "cipher", +] [[package]] name = "cbor4ii" @@ -685,9 +1068,9 @@ [[package]] name = "cc" -version = "1.2.62" +version = "1.2.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" dependencies = [ "find-msvc-tools", "jobserver", @@ -720,6 +1103,12 @@ checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0" [[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + +[[package]] name = "cfg-if" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -732,10 +1121,23 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] -name = "chrono" -version = "0.4.44" +name = "chacha20" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.3.0", + "rand_core 0.10.1", + "zeroize", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "js-sys", @@ -786,6 +1188,18 @@ ] [[package]] +name = "cipher" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", + "inout", + "zeroize", +] + +[[package]] name = "clap" version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -805,6 +1219,7 @@ "anstyle", "clap_lex", "strsim", + "terminal_size", ] [[package]] @@ -813,10 +1228,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" dependencies = [ - "heck 0.5.0", + "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -824,6 +1239,30 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "clru" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "197fd99cb113a8d5d9b6376f3aa817f32c1078f2343b714fff7d2ca44fdf67d5" +dependencies = [ + "hashbrown 0.16.1", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" [[package]] name = "cobs" @@ -841,14 +1280,27 @@ checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] name = "compression-codecs" version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" dependencies = [ + "brotli", "compression-core", "flate2", "memchr", + "zstd", + "zstd-safe", ] [[package]] @@ -856,6 +1308,21 @@ version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "condtype" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf0a07a401f374238ab8e2f11a104d2851bf9ce711ec69804834de8af45c7af" [[package]] name = "confique" @@ -874,10 +1341,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4d1754680cd218e7bcb4c960cc9bae3444b5197d64563dccccfdf83cab9e1a7" dependencies = [ - "heck 0.5.0", + "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -885,6 +1352,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" [[package]] name = "const-str" @@ -929,10 +1402,25 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + +[[package]] name = "cpufeatures" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" dependencies = [ "libc", ] @@ -996,9 +1484,9 @@ [[package]] name = "crossbeam-deque" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -1031,8 +1519,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ - "generic-array", + "generic-array 0.14.9", "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-bigint" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97bb4a855e3b10f84c4e7e895a7de01db7f9a7b7eb7f73ed9773fd52ac686451" +dependencies = [ + "cpubits", + "ctutils", + "getrandom 0.4.3", + "hybrid-array", + "num-traits", + "rand_core 0.10.1", + "serdect", "subtle", "zeroize", ] @@ -1043,8 +1548,75 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ - "generic-array", + "generic-array 0.14.9", "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "getrandom 0.4.3", + "hybrid-array", + "rand_core 0.10.1", +] + +[[package]] +name = "crypto-primes" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3633a51a39c69ebbaa4feaa694bd83d241e4093901c84a0963b19d9bb3f0cf8f" +dependencies = [ + "crypto-bigint 0.7.4", + "rand_core 0.10.1", +] + +[[package]] +name = "ctr" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21" +dependencies = [ + "cipher", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", + "subtle", +] + +[[package]] +name = "curve25519-dalek" +version = "5.0.0-rc.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f359e08ca85e7bd759e1fd933ff2bccd81864c60a8fba0e259c7f822b0924bf" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "curve25519-dalek-derive", + "digest 0.11.3", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] @@ -1067,7 +1639,7 @@ "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.118", ] [[package]] @@ -1078,7 +1650,7 @@ dependencies = [ "darling_core", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1118,7 +1690,7 @@ checksum = "ccc2776f0c61eca1ca32528f85548abd1a4be8fb53d1b21c013e4f18da1e7090" dependencies = [ "data-encoding", - "syn", + "syn 2.0.118", ] [[package]] @@ -1146,14 +1718,96 @@ checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" [[package]] +name = "defmt" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6e524506490a1953d237cb87b1cfc1e46f88c18f10a22dfe0f507dc6bfc7f7f" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0a27770e9c8f719a79d8b638281f4d828f77d8fd61e0bd94451b9b85e576a0b" +dependencies = [ + "defmt-parser", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.18", +] + +[[package]] +name = "delegate" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "780eb241654bf097afb00fc5f054a09b687dad862e485fdcf8399bb056565370" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] name = "der" version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", - "pem-rfc7468", + "const-oid 0.9.6", + "pem-rfc7468 0.7.0", "zeroize", +] + +[[package]] +name = "der" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71fd89660b2dc699704064e59e9dba0147b903e85319429e131620d022be411b" +dependencies = [ + "const-oid 0.10.2", + "pem-rfc7468 1.0.0", + "zeroize", +] + +[[package]] +name = "der-parser" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" +dependencies = [ + "asn1-rs 0.6.2", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs 0.7.2", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", ] [[package]] @@ -1162,7 +1816,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ - "powerfmt", "serde_core", ] @@ -1183,8 +1836,17 @@ dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "unicode-xid", +] + +[[package]] +name = "des" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "916a94e407b54f9034d71dd748234cd1e516ced6284009906ae246f177eafe5a" +dependencies = [ + "cipher", ] [[package]] @@ -1199,10 +1861,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.6", "subtle", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", + "ctutils", ] [[package]] @@ -1213,7 +1887,32 @@ dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", +] + +[[package]] +name = "divan" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a405457ec78b8fe08b0e32b4a3570ab5dff6dd16eb9e76a5ee0a9d9cbd898933" +dependencies = [ + "cfg-if", + "clap", + "condtype", + "divan-macros", + "libc", + "regex-lite", +] + +[[package]] +name = "divan-macros" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9556bc800956545d6420a640173e5ba7dfa82f38d3ea5a167eb555bc69ac3323" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] @@ -1223,24 +1922,71 @@ checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" [[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] name = "ecdsa" version = "0.16.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ - "der", - "digest", - "elliptic-curve", - "rfc6979", - "signature", - "spki", + "der 0.7.10", + "digest 0.10.7", + "elliptic-curve 0.13.8", + "rfc6979 0.4.0", + "signature 2.2.0", + "spki 0.7.3", +] + +[[package]] +name = "ecdsa" +version = "0.17.0-rc.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54fb064faabbee66e1fc8e5c5a9458d4269dc2d8b638fe86a425adb2510d1a96" +dependencies = [ + "der 0.8.0", + "digest 0.11.3", + "elliptic-curve 0.14.0-rc.33", + "rfc6979 0.5.0", + "signature 3.0.0", + "spki 0.8.0", + "zeroize", +] + +[[package]] +name = "ed25519" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29fcf32e6c73d1079f83ab4d782de2d81620346a5f38c6237a86a22f8368980a" +dependencies = [ + "pkcs8 0.11.0", + "signature 3.0.0", +] + +[[package]] +name = "ed25519-dalek" +version = "3.0.0-rc.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b011170fe4f04665565b4110afef66774fe9ffff278f3eb5b81cc73d26e27d60" +dependencies = [ + "curve25519-dalek", + "ed25519", + "rand_core 0.10.1", + "serde", + "sha2 0.11.0", + "signature 3.0.0", + "subtle", + "zeroize", ] [[package]] name = "either" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" [[package]] name = "elliptic-curve" @@ -1248,16 +1994,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ - "base16ct", - "crypto-bigint", - "digest", - "ff", - "generic-array", - "group", - "pem-rfc7468", - "pkcs8", + "base16ct 0.2.0", + "crypto-bigint 0.5.5", + "digest 0.10.7", + "ff 0.13.1", + "generic-array 0.14.9", + "group 0.13.0", + "pem-rfc7468 0.7.0", + "pkcs8 0.10.2", "rand_core 0.6.4", - "sec1", + "sec1 0.7.3", + "subtle", + "zeroize", +] + +[[package]] +name = "elliptic-curve" +version = "0.14.0-rc.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "102d3643d30dd8b559613c5cced68317199597fffb278cdc88daa2ef7fafc935" +dependencies = [ + "base16ct 1.0.0", + "crypto-bigint 0.7.4", + "crypto-common 0.2.2", + "digest 0.11.3", + "ff 0.14.0", + "group 0.14.0", + "hkdf", + "hybrid-array", + "once_cell", + "pem-rfc7468 1.0.0", + "pkcs8 0.11.0", + "rand_core 0.10.1", + "sec1 0.8.1", "subtle", "zeroize", ] @@ -1281,6 +2050,30 @@ checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" dependencies = [ "cfg-if", +] + +[[package]] +name = "enum-as-inner" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "enum_dispatch" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" +dependencies = [ + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] @@ -1311,10 +2104,53 @@ ] [[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] name = "fastdivide" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471" + +[[package]] +name = "faster-hex" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7223ae2d2f179b803433d9c830478527e92b8117eab39460edae7f1614d9fb73" +dependencies = [ + "heapless 0.8.0", + "serde", +] [[package]] name = "fastrand" @@ -1330,6 +2166,32 @@ dependencies = [ "rand_core 0.6.4", "subtle", +] + +[[package]] +name = "ff" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f686ab92a9fb0eaf188f6c6c87b89490baa6fdb0db4544ba4dc47f7942489f" +dependencies = [ + "rand_core 0.10.1", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", ] [[package]] @@ -1352,6 +2214,7 @@ dependencies = [ "crc32fast", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -1393,6 +2256,16 @@ ] [[package]] +name = "forwarded-header-value" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8835f84f38484cc86f110a805655697908257fb9a7af005234060891557198e9" +dependencies = [ + "nonempty 0.7.0", + "thiserror 1.0.69", +] + +[[package]] name = "fs4" version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1401,6 +2274,12 @@ "rustix", "windows-sys 0.59.0", ] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures" @@ -1484,7 +2363,18 @@ dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", +] + +[[package]] +name = "futures-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f2f12607f92c69b12ed746fabf9ca4f5c482cba46679c1a75b874ed7c26adb" +dependencies = [ + "futures-io", + "rustls", + "rustls-pki-types", ] [[package]] @@ -1498,6 +2388,12 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-timer" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" [[package]] name = "futures-util" @@ -1518,9 +2414,9 @@ [[package]] name = "generator" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52f04ae4152da20c76fe800fa48659201d5cf627c5149ca0b707b69d7eef6cf9" +checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" dependencies = [ "cc", "cfg-if", @@ -1540,6 +2436,32 @@ "typenum", "version_check", "zeroize", +] + +[[package]] +name = "generic-array" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2e55f16dcf0e9c00efbe2e655ffe45fc98e7066b52bc92f8a79e64060a79351" +dependencies = [ + "generic-array 0.14.9", + "rustversion", + "typenum", +] + +[[package]] +name = "gengo-language" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f4ac35e0d9289625d1266dd236eca53fa8cb0450dea944fe3a99a85d664c82" +dependencies = [ + "glob", + "indexmap", + "proc-macro2", + "quote", + "regex", + "serde", + "serde_yaml", ] [[package]] @@ -1571,15 +2493,966 @@ [[package]] name = "getrandom" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", - "wasip2", - "wasip3", + "rand_core 0.10.1", + "wasm-bindgen", +] + +[[package]] +name = "ghash" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5" +dependencies = [ + "polyval", +] + +[[package]] +name = "gix" +version = "0.84.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae54ae0ebd1a5a3c3f8d95dd3b5ca6e63f4fed9bfd585e13801a97d7bde8f9ce" +dependencies = [ + "gix-actor", + "gix-archive", + "gix-attributes", + "gix-blame", + "gix-command", + "gix-commitgraph", + "gix-config", + "gix-credentials", + "gix-date", + "gix-diff", + "gix-dir", + "gix-discover", + "gix-error", + "gix-features", + "gix-filter", + "gix-fs", + "gix-glob", + "gix-hash", + "gix-hashtable", + "gix-ignore", + "gix-index", + "gix-lock", + "gix-mailmap", + "gix-negotiate", + "gix-object", + "gix-odb", + "gix-pack", + "gix-path", + "gix-pathspec", + "gix-prompt", + "gix-protocol", + "gix-ref", + "gix-refspec", + "gix-revision", + "gix-revwalk", + "gix-sec", + "gix-shallow", + "gix-status", + "gix-submodule", + "gix-tempfile", + "gix-trace", + "gix-traverse", + "gix-url", + "gix-utils", + "gix-validate", + "gix-worktree", + "gix-worktree-state", + "gix-worktree-stream", + "nonempty 0.12.0", + "parking_lot", + "regex", + "signal-hook", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-actor" +version = "0.41.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bc998b8f746dda8565450d08a63b792ced9165d8c27a1ed3f02799ec6a7820f" +dependencies = [ + "bstr", + "gix-date", + "gix-error", +] + +[[package]] +name = "gix-archive" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16909cacc78936ab96f6c3be08379d0a2e88bfa3a7527972d2ed75c7517ef31e" +dependencies = [ + "bstr", + "flate2", + "gix-date", + "gix-error", + "gix-object", + "gix-path", + "gix-worktree-stream", + "rawzip", + "tar", +] + +[[package]] +name = "gix-attributes" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d43f12e246d3bf7ec624c8fc15ac4a4b62b7c4c6f586cb82be6c90bf84c9d02" +dependencies = [ + "bstr", + "gix-glob", + "gix-path", + "gix-quote", + "gix-trace", + "kstring", + "smallvec", + "thiserror 2.0.18", + "unicode-bom", +] + +[[package]] +name = "gix-bitmap" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ebef0c26ad305747649e727bbcd56a7b7910754eb7cea88f6dff6f93c51283" +dependencies = [ + "gix-error", +] + +[[package]] +name = "gix-blame" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d39a0c14af94c2edaa5eefe06d5ef2cdea55316ae9a9321314288e3f55fa4c0" +dependencies = [ + "gix-commitgraph", + "gix-date", + "gix-diff", + "gix-error", + "gix-hash", + "gix-object", + "gix-revwalk", + "gix-trace", + "gix-traverse", + "gix-worktree", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-chunk" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9faee47943b638e58ddd5e275a4906ad3e4b6c8584f1d41bd18ab9032ec52afb" +dependencies = [ + "gix-error", +] + +[[package]] +name = "gix-command" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00706d4fef135ef4b01680d5218c6ee40cda8baf697b864296cbc887d19118f6" +dependencies = [ + "bstr", + "gix-path", + "gix-quote", + "gix-trace", + "shell-words", +] + +[[package]] +name = "gix-commitgraph" +version = "0.37.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f675d0df484a7f6a47e64bd6f311af489d947c0323b0564f36d14f3d7762abb" +dependencies = [ + "bstr", + "gix-chunk", + "gix-error", + "gix-hash", + "memmap2", + "nonempty 0.12.0", +] + +[[package]] +name = "gix-config" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2372d4b49ca28431e7d150cab9d25edc1890f0184bd57eb0e917c7799e63de" +dependencies = [ + "bstr", + "gix-config-value", + "gix-features", + "gix-glob", + "gix-path", + "gix-ref", + "gix-sec", + "smallvec", + "thiserror 2.0.18", + "unicode-bom", +] + +[[package]] +name = "gix-config-value" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed42168329552f6c2e5df09665c104199d45d84bedb53683738a49b57fe1baab" +dependencies = [ + "bitflags 2.13.0", + "bstr", + "gix-path", + "libc", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-credentials" +version = "0.38.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40cd22f0dd71988be12d6e78b1709de2370e1957c5f107ff31e56caeba3745d" +dependencies = [ + "bstr", + "gix-command", + "gix-config-value", + "gix-date", + "gix-path", + "gix-prompt", + "gix-sec", + "gix-trace", + "gix-url", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-date" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3ecab64a98bbac9f8e02990a9ea5e3c974a7d49b95f2bd70ad94ad22fa6b48c" +dependencies = [ + "bstr", + "gix-error", + "itoa", + "jiff", +] + +[[package]] +name = "gix-diff" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b6d9528f32d94cef2edf39a1ac01fe5a0fc44ddbb18d9e44099936047c3302b" +dependencies = [ + "bstr", + "gix-attributes", + "gix-command", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-imara-diff", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-tempfile", + "gix-trace", + "gix-traverse", + "gix-worktree", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-dir" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21bb2a53a6fd917ec499ed0bfb5b6887de7a15bd79197dcea7c987938749a9f1" +dependencies = [ + "bstr", + "gix-discover", + "gix-fs", + "gix-ignore", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-trace", + "gix-utils", + "gix-worktree", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-discover" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77bacdd12b7879d2178a80c58c2f319995e4654e1a7a23e3181e5c8a12b824f7" +dependencies = [ + "bstr", + "dunce", + "gix-fs", + "gix-path", + "gix-ref", + "gix-sec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-error" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57831e199be480af90dcd7e459abed8a174c09ec9a6e2cc8f7ca6c54598b06b" +dependencies = [ + "bstr", +] + +[[package]] +name = "gix-features" +version = "0.48.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1849ae154d38bc403185be14fa871e38e3c93ee606875d94e207fdb9fba52dbc" +dependencies = [ + "bytes", + "bytesize", + "crc32fast", + "crossbeam-channel", + "gix-path", + "gix-trace", + "gix-utils", + "libc", + "once_cell", + "parking_lot", + "prodash", + "thiserror 2.0.18", + "walkdir", + "zlib-rs", +] + +[[package]] +name = "gix-filter" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecf74b7d16f6694ce4a3049074c41be0c7987105743674f1671807bd6dce09fa" +dependencies = [ + "bstr", + "encoding_rs", + "gix-attributes", + "gix-command", + "gix-hash", + "gix-object", + "gix-packetline", + "gix-path", + "gix-quote", + "gix-trace", + "gix-utils", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-fs" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cdff46db8798e47e2f727d84b9379aac5add3dd3d9d0b07bb4d7d5d640771fe" +dependencies = [ + "bstr", + "fastrand", + "gix-features", + "gix-path", + "gix-utils", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-glob" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1fcb8ef5b16bcf874abe9b68d8abb3c0493c876d367ab824151f30a0f3f3756" +dependencies = [ + "bitflags 2.13.0", + "bstr", + "gix-features", + "gix-path", +] + +[[package]] +name = "gix-hash" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0926d3819c837750b4e03c7754901e73f68b8c9b690753a6372a1bed4eedce" +dependencies = [ + "faster-hex", + "gix-features", + "sha1-checked", + "sha2 0.11.0", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-hashtable" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0e30b93eea8718baf7d8153fcb938e2926175bbf18097c09f1c01b6f0be0563" +dependencies = [ + "gix-hash", + "hashbrown 0.17.1", + "parking_lot", +] + +[[package]] +name = "gix-ignore" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d491bab9bf2c9f341dc754f425c31d5d3f63aca615312167b82e1deeaca97d8d" +dependencies = [ + "bstr", + "gix-glob", + "gix-path", + "gix-trace", + "unicode-bom", +] + +[[package]] +name = "gix-imara-diff" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19753d40da53d0ec41604750eeb969097a90fb2d7f7992730d904541c04e2c19" +dependencies = [ + "bstr", + "hashbrown 0.17.1", +] + +[[package]] +name = "gix-index" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e6b28cc592dc753adb58302bb14a64e412ee591a3bec77aa4df87bff74fa80d" +dependencies = [ + "bitflags 2.13.0", + "bstr", + "filetime", + "fnv", + "gix-bitmap", + "gix-features", + "gix-fs", + "gix-hash", + "gix-lock", + "gix-object", + "gix-traverse", + "gix-utils", + "gix-validate", + "hashbrown 0.17.1", + "itoa", + "libc", + "memmap2", + "rustix", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-lock" +version = "23.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9dedd9e90b0d47624d2ed241d394e09294118364e87b9b7e5f1fe755f3c2c" +dependencies = [ + "gix-tempfile", + "gix-utils", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-mailmap" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "195fd20808055824531be2fd0d34136d900e5fbca3ffb0a3c07e8beeefb9c828" +dependencies = [ + "bstr", + "gix-actor", + "gix-date", + "gix-error", +] + +[[package]] +name = "gix-negotiate" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "890c936a215bae25818c076cb881cb2e54d2c66ba947ba58b8dd47cff921bf55" +dependencies = [ + "bitflags 2.13.0", + "gix-commitgraph", + "gix-date", + "gix-hash", + "gix-object", + "gix-revwalk", +] + +[[package]] +name = "gix-object" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5cd857e29429c7213bdef3f5aef83f8cc124774fe8ae0d27b1607d218d6d525" +dependencies = [ + "bstr", + "gix-actor", + "gix-date", + "gix-features", + "gix-hash", + "gix-hashtable", + "gix-utils", + "gix-validate", + "itoa", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-odb" +version = "0.81.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d004c32858b1556f2d7874405edb3c97dc78fc09beaa87d57bb077ee2858a7d" +dependencies = [ + "arc-swap", + "gix-features", + "gix-fs", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-pack", + "gix-path", + "gix-quote", + "memmap2", + "parking_lot", + "tempfile", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-pack" +version = "0.71.0" +dependencies = [ + "clru", + "gix-chunk", + "gix-diff", + "gix-error", + "gix-features", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-path", + "gix-tempfile", + "gix-traverse", + "parking_lot", + "smallvec", + "thiserror 2.0.18", + "uluru", +] + +[[package]] +name = "gix-packetline" +version = "0.21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b217dd0ee0c4021ecf169a4a519b1b4f80d15e3f3765f3dc466223dc0ac891d7" +dependencies = [ + "bstr", + "faster-hex", + "gix-trace", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-path" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afa6ac14cd14939ea94a496ce7460daa6511c09f5b84757e9cfc6f9c8d0f93a6" +dependencies = [ + "bstr", + "gix-trace", + "gix-validate", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-pathspec" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3050783b41ee11511e1e8fb35623df81806194f4030395f14f48ea37c2798c9f" +dependencies = [ + "bitflags 2.13.0", + "bstr", + "gix-attributes", + "gix-config-value", + "gix-glob", + "gix-path", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-prompt" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee604d7746080ae7e1023bf47204bcc2c5f307bfbe2306a3c90b1bfd1a2c6d8" +dependencies = [ + "gix-command", + "gix-config-value", + "parking_lot", + "rustix", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-protocol" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51dea3acb390707ab868f1f9584f18449eb95d869deffae96768e47d303595ee" +dependencies = [ + "bstr", + "gix-date", + "gix-features", + "gix-hash", + "gix-ref", + "gix-shallow", + "gix-transport", + "gix-utils", + "maybe-async", + "nonempty 0.12.0", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-quote" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6e541fc33cc2b783b7979040d445a0c86a2eca747c8faea4ca84230d06ae6ef" +dependencies = [ + "bstr", + "gix-error", + "gix-utils", +] + +[[package]] +name = "gix-ref" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c04f64c37eb7e6feb73c7060f8dc6f381cc5de5d53249bfd450bc48a86b2e8b" +dependencies = [ + "gix-actor", + "gix-features", + "gix-fs", + "gix-hash", + "gix-lock", + "gix-object", + "gix-path", + "gix-tempfile", + "gix-utils", + "gix-validate", + "memmap2", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-refspec" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b216ae06ec74b5f24ad0142026a997fb0a935b7410eaf9c1616fc3f0e6c5a6d3" +dependencies = [ + "bstr", + "gix-error", + "gix-glob", + "gix-hash", + "gix-revision", + "gix-validate", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-revision" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b47c88884dd3c1a19a39da19d10211fcdea2809aadc86869b6e824a1774340f" +dependencies = [ + "bitflags 2.13.0", + "bstr", + "gix-commitgraph", + "gix-date", + "gix-error", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-revwalk", + "gix-trace", + "nonempty 0.12.0", +] + +[[package]] +name = "gix-revwalk" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85f5756abffe0917827aac683b13684ed99875bc398fa1f9b8f479b0681ef9e6" +dependencies = [ + "gix-commitgraph", + "gix-date", + "gix-error", + "gix-hash", + "gix-hashtable", + "gix-object", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-sec" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab8519976e4c7e486270740a5400369f37940779b80bd1377d94cfa1125d01b3" +dependencies = [ + "bitflags 2.13.0", + "gix-path", + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "gix-shallow" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a292fc2fe548c5dfa575479d16b445b0ddf1dd2f56f1fec6aed386f82553cd97" +dependencies = [ + "bstr", + "gix-hash", + "gix-lock", + "nonempty 0.12.0", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-status" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22042e385d28a34275e029d98f4970285045be14b9073658ca897923f2ed8700" +dependencies = [ + "bstr", + "filetime", + "gix-diff", + "gix-dir", + "gix-features", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-worktree", + "portable-atomic", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-submodule" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3059890ef054066c22a94bfc6a3eaba0d806aedcd630a0bc9e5783fd88884781" +dependencies = [ + "bstr", + "gix-config", + "gix-path", + "gix-pathspec", + "gix-refspec", + "gix-url", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-tempfile" +version = "23.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27850097e1ff9515f46a0dad0f5f9c9d020e972727772dabab9450690c4adb22" +dependencies = [ + "dashmap", + "gix-fs", + "libc", + "parking_lot", + "signal-hook", + "signal-hook-registry", + "tempfile", +] + +[[package]] +name = "gix-trace" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44dc45eae785c0eb14173e0f152e6e224dcf4d45b6a6999a3aed22af541ad678" + +[[package]] +name = "gix-transport" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd0e34995b1aab0fa8dff2af8db726a0bfad3e119c89302604463264046e7ff" +dependencies = [ + "bstr", + "gix-command", + "gix-features", + "gix-packetline", + "gix-quote", + "gix-sec", + "gix-url", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-traverse" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8de590ecc86a3b2870665f2288324fa9f7f8672c7fc2d4e020fdd81cd1f7aed" +dependencies = [ + "bitflags 2.13.0", + "gix-commitgraph", + "gix-date", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-revwalk", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-url" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bb01ec69d55e82ccb7a19e264501ead4e6aac38463a8cebfdd81e22bb67ab2" +dependencies = [ + "bstr", + "gix-path", + "percent-encoding", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-utils" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66c50966184123caf580ffa64e28031a878597f1c7fceb8fe19566c38eb1b771" +dependencies = [ + "bstr", + "fastrand", + "unicode-normalization", +] + +[[package]] +name = "gix-validate" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bc6fc771c4063ba7cd2f47b91fb6076251c6a823b64b7fe7b8874b0fe4afae3" +dependencies = [ + "bstr", +] + +[[package]] +name = "gix-worktree" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef414ed275e8407cd5d53d301e83be19700b0dd3f859d2434417b58f454a2d1" +dependencies = [ + "bstr", + "gix-attributes", + "gix-fs", + "gix-glob", + "gix-hash", + "gix-ignore", + "gix-index", + "gix-object", + "gix-path", + "gix-validate", +] + +[[package]] +name = "gix-worktree-state" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bffae8b3ca258fdd50370cd51f06deb4c76a3b43db3868bc28dde45ffa77d69" +dependencies = [ + "bstr", + "gix-features", + "gix-filter", + "gix-fs", + "gix-index", + "gix-object", + "gix-path", + "gix-worktree", + "io-close", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-worktree-stream" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25e9ed30100c63f7590bc581c225e53f731a53e06aa79a245739c07f7dcc557" +dependencies = [ + "gix-attributes", + "gix-error", + "gix-features", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-object", + "gix-path", + "gix-traverse", + "parking_lot", +] + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "globset" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52dfc19153a48bde0cbd630453615c8151bce3a5adfac7a0aebfbf0a1e1f57e3" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "governor" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9efcab3c1958580ff1f25a2a41be1668f7603d849bb63af523b208a3cc1223b8" +dependencies = [ + "cfg-if", + "dashmap", + "futures-sink", + "futures-timer", + "futures-util", + "getrandom 0.3.4", + "hashbrown 0.16.1", + "nonzero_ext", + "parking_lot", + "portable-atomic", + "quanta", + "rand 0.9.4", + "smallvec", + "spinning_top", + "web-time", ] [[package]] @@ -1588,16 +3461,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ - "ff", + "ff 0.13.1", "rand_core 0.6.4", "subtle", ] [[package]] -name = "h2" -version = "0.4.14" +name = "group" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +checksum = "7fd1a1c7a5206c5b7a3f5a0d7ccd3ff85d0c8f5133d62a02680255b0004af5f4" +dependencies = [ + "ff 0.14.0", + "rand_core 0.10.1", + "subtle", +] + +[[package]] +name = "h2" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" dependencies = [ "atomic-waker", "bytes", @@ -1610,6 +3494,34 @@ "tokio", "tokio-util", "tracing", +] + +[[package]] +name = "h3" +version = "0.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10872b55cfb02a821b69dc7cf8dc6a71d6af25eb9a79662bec4a9d016056b3be" +dependencies = [ + "bytes", + "fastrand", + "futures-util", + "http", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "h3-quinn" +version = "0.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2e732c8d91a74731663ac8479ab505042fbf547b9a207213ab7fbcbfc4f8b4" +dependencies = [ + "bytes", + "futures", + "h3", + "quinn", + "tokio", + "tokio-util", ] [[package]] @@ -1628,6 +3540,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" dependencies = [ "byteorder", ] @@ -1669,6 +3590,20 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hashlink" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824e001ac4f3012dd16a264bec811403a67ca9deb6c102fc5049b32c4574b35f" +dependencies = [ + "hashbrown 0.16.1", +] [[package]] name = "heapless" @@ -1677,7 +3612,7 @@ checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" dependencies = [ "atomic-polyfill", - "hash32", + "hash32 0.2.1", "rustc_version", "serde", "spin 0.9.8", @@ -1685,10 +3620,14 @@ ] [[package]] -name = "heck" -version = "0.4.1" +name = "heapless" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32 0.3.1", + "stable_deref_trait", +] [[package]] name = "heck" @@ -1709,12 +3648,81 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] +name = "hex-literal" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e712f64ec3850b98572bffac52e2c6f282b29fe6c5fa6d42334b30be438d95c1" + +[[package]] +name = "hickory-proto" +version = "0.24.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92652067c9ce6f66ce53cc38d1169daa36e6e7eb7dd3b63b5103bd9d97117248" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "enum-as-inner", + "futures-channel", + "futures-io", + "futures-util", + "idna", + "ipnet", + "once_cell", + "rand 0.8.6", + "thiserror 1.0.69", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "hickory-resolver" +version = "0.24.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbb117a1ca520e111743ab2f6688eddee69db4e0ea242545a604dce8a66fd22e" +dependencies = [ + "cfg-if", + "futures-util", + "hickory-proto", + "ipconfig", + "lru-cache", + "once_cell", + "parking_lot", + "rand 0.8.6", + "resolv-conf", + "smallvec", + "thiserror 1.0.69", + "tokio", + "tracing", +] + +[[package]] +name = "hkdf" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018" +dependencies = [ + "hmac 0.13.0", +] + +[[package]] name = "hmac" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", ] [[package]] @@ -1725,9 +3733,9 @@ [[package]] name = "http" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" dependencies = [ "bytes", "itoa", @@ -1757,6 +3765,12 @@ ] [[package]] +name = "http-range-header" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9171a2ea8a68358193d15dd5d70c1c10a2afc3e7e4c5bc92bc9f025cebd7359c" + +[[package]] name = "httparse" version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1769,10 +3783,28 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] -name = "hyper" -version = "1.10.0" +name = "human_format" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb92f162bf56536459fc83c79b974bb12837acfed43d6bc370a7916d0ae15ecc" +checksum = "eaec953f16e5bcf6b8a3cb3aa959b17e5577dbd2693e94554c462c08be22624b" + +[[package]] +name = "hybrid-array" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +dependencies = [ + "ctutils", + "subtle", + "typenum", + "zeroize", +] + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" dependencies = [ "atomic-waker", "bytes", @@ -1803,7 +3835,7 @@ "tokio", "tokio-rustls", "tower-service", - "webpki-roots 1.0.7", + "webpki-roots 1.0.8", ] [[package]] @@ -1938,12 +3970,6 @@ ] [[package]] -name = "id-arena" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" - -[[package]] name = "ident_case" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1983,12 +4009,57 @@ ] [[package]] +name = "inout" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" +dependencies = [ + "block-padding", + "hybrid-array", +] + +[[package]] +name = "internal-russh-num-bigint" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae8e22120c32fb4d19ec55fba35015f57095cd95a2e3b732e44457f5915b2ee8" +dependencies = [ + "num-integer", + "num-traits", + "rand 0.10.1", + "rand_core 0.10.1", +] + +[[package]] name = "inventory" version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" dependencies = [ "rustversion", +] + +[[package]] +name = "io-close" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cadcf447f06744f8ce713d2d6239bb5bde2c357a452397a9ed90c625da390bc" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "ipconfig" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" +dependencies = [ + "socket2", + "widestring", + "windows-registry", + "windows-result", + "windows-sys 0.61.2", ] [[package]] @@ -2040,9 +4111,9 @@ [[package]] name = "jacquard-common" -version = "0.12.0-beta.2" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e830579811d60e29209c9466d034225d5e045ecdc2b3c55282709bd07da97869" +checksum = "fec8b6661e6fcfb4d8b9eb1132503be71bc3e3c6f6ff15fa5500ec62655f6da7" dependencies = [ "base64", "bon", @@ -2064,23 +4135,22 @@ "multibase", "multihash", "n0-future", - "ouroboros", "oxilangtag", - "p256", + "p256 0.13.2", "phf", "postcard", "rand 0.9.4", "regex", "regex-automata", "regex-lite", - "reqwest", + "reqwest 0.12.28", "rustversion", "serde", "serde_bytes", "serde_html_form", "serde_ipld_dagcbor", "serde_json", - "signature", + "signature 2.2.0", "smol_str", "spin 0.10.0", "thiserror 2.0.18", @@ -2093,26 +4163,26 @@ [[package]] name = "jacquard-derive" -version = "0.12.0-beta.2" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f83b8049e4e7916e0f6764c3deaf5e55a7ffbab26c379415e9b1d4d645d957" +checksum = "e41b82a9a72c0d2b907a5e0734e0692615e717cda903502c6c034ae6f3ddddf7" dependencies = [ - "heck 0.5.0", + "heck", "jacquard-lexicon", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] name = "jacquard-lexicon" -version = "0.12.0-beta.2" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64935ef85dd24f60f467082c21ad52f739a02dd402a2adf40e5794e3de949e1f" +checksum = "160049c269e3d7ec56f130736d25d61953fc6216da660c8b5e102f6b052dd3a1" dependencies = [ "cid", "dashmap", - "heck 0.5.0", + "heck", "inventory", "jacquard-common", "miette", @@ -2126,10 +4196,96 @@ "serde_path_to_error", "serde_repr", "serde_with", - "sha2", - "syn", + "sha2 0.10.9", + "syn 2.0.118", "thiserror 2.0.18", "unicode-segmentation", +] + +[[package]] +name = "jiff" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34f877a98676d2fb664698d74cc6a51ce6c484ce8c770f05d0108ec9090aeb46" +dependencies = [ + "defmt", + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link", +] + +[[package]] +name = "jiff-static" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0666b5ab5ecaca213fc2a85b8c0083d9004e84ee2d5f9a7e0017aaf50986f25f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c900ef84826f1338a557697dc8fc601df9ca9af4ac137c7fb61d4c6f2dfd3076" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.118", ] [[package]] @@ -2144,13 +4300,12 @@ [[package]] name = "js-sys" -version = "0.3.99" +version = "0.3.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" dependencies = [ "cfg-if", "futures-util", - "once_cell", "wasm-bindgen", ] @@ -2161,9 +4316,676 @@ checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" dependencies = [ "cfg-if", - "ecdsa", - "elliptic-curve", - "sha2", + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", + "once_cell", + "sha2 0.10.9", + "signature 2.2.0", +] + +[[package]] +name = "keccak" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e24a010dd405bd7ed803e5253182815b41bf2e6a80cc3bfc066658e03a198aa" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", +] + +[[package]] +name = "kem" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01737161ba802849cfd486b5bd209d38ba4943494c249a8126005170c7621edd" +dependencies = [ + "crypto-common 0.2.2", + "rand_core 0.10.1", +] + +[[package]] +name = "knot-acl" +version = "2.0.0" +dependencies = [ + "knot-cob", + "knot-cobs", + "knot-git", + "knot-index", + "knot-runtime", + "knot-types", + "tempfile", +] + +[[package]] +name = "knot-atproto" +version = "2.0.0" +dependencies = [ + "base32", + "base64", + "bs58", + "bytes", + "futures", + "http", + "k256", + "knot-cache", + "knot-lexicons", + "knot-runtime", + "knot-types", + "proptest", + "serde", + "serde_ipld_dagcbor", + "serde_json", + "sha2 0.11.0", + "thiserror 2.0.18", + "tokio", + "url", +] + +[[package]] +name = "knot-bench" +version = "2.0.0" +dependencies = [ + "divan", + "gix", + "knot-cob", + "knot-cobs", + "knot-git", + "knot-index", + "knot-pack", + "knot-runtime", + "knot-types", + "tempfile", +] + +[[package]] +name = "knot-cache" +version = "2.0.0" +dependencies = [ + "knot-runtime", + "knot-types", + "moka", +] + +[[package]] +name = "knot-cob" +version = "2.0.0" +dependencies = [ + "gix", + "gix-hash", + "k256", + "knot-git", + "knot-resource", + "knot-runtime", + "knot-types", + "proptest", + "serde", + "serde_ipld_dagcbor", + "tempfile", + "thiserror 2.0.18", + "tracing", +] + +[[package]] +name = "knot-cobs" +version = "2.0.0" +dependencies = [ + "gix", + "knot-cob", + "knot-git", + "knot-runtime", + "knot-types", + "proptest", + "serde", + "serde_ipld_dagcbor", + "tempfile", + "thiserror 2.0.18", +] + +[[package]] +name = "knot-config" +version = "2.0.0" +dependencies = [ + "base64", + "confique", + "knot-messages", + "knot-runtime", + "knot-types", + "tempfile", + "thiserror 2.0.18", + "url", +] + +[[package]] +name = "knot-edge" +version = "2.0.0" +dependencies = [ + "arc-swap", + "async-trait", + "axum", + "base64", + "bytes", + "futures", + "governor", + "h3", + "h3-quinn", + "http", + "http-body", + "hyper", + "hyper-util", + "knot-types", + "proptest", + "quinn", + "rcgen 0.14.8", + "rustls", + "rustls-acme", + "rustls-pemfile", + "sha2 0.11.0", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http 0.7.0", + "tower_governor", + "tracing", + "x509-parser 0.18.1", +] + +[[package]] +name = "knot-events" +version = "2.0.0" +dependencies = [ + "knot-runtime", + "knot-types", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "knot-fixtures" +version = "2.0.0" +dependencies = [ + "tempfile", +] + +[[package]] +name = "knot-git" +version = "2.0.0" +dependencies = [ + "base64", + "flate2", + "gix", + "gix-archive", + "gix-bitmap", + "gix-hash", + "gix-pack", + "knot-cache", + "knot-fixtures", + "knot-resource", + "knot-types", + "proptest", + "scc", + "tempfile", + "thiserror 2.0.18", + "walkdir", +] + +[[package]] +name = "knot-index" +version = "2.0.0" +dependencies = [ + "knot-cache", + "knot-cob", + "knot-cobs", + "knot-git", + "knot-runtime", + "knot-types", + "lasso", + "proptest", + "scc", + "serde", + "tempfile", + "thiserror 2.0.18", +] + +[[package]] +name = "knot-langs" +version = "2.0.0" +dependencies = [ + "gengo-language", + "knot-git", + "knot-types", + "regex", +] + +[[package]] +name = "knot-lexicons" +version = "2.0.0" +dependencies = [ + "anyhow", + "bytes", + "cid", + "jacquard-common", + "jacquard-derive", + "jacquard-lexicon", + "miette", + "serde", + "serde_json", + "thiserror 2.0.18", + "url", + "walkdir", +] + +[[package]] +name = "knot-lfs" +version = "2.0.0" +dependencies = [ + "gix-packetline", + "knot-git", + "knot-messages", + "knot-resource", + "knot-runtime", + "knot-types", + "proptest", + "serde", + "serde_json", + "sha2 0.11.0", + "tempfile", + "thiserror 2.0.18", + "tracing", + "url", +] + +[[package]] +name = "knot-maintenance" +version = "2.0.0" +dependencies = [ + "gix", + "gix-hash", + "gix-pack", + "knot-config", + "knot-fixtures", + "knot-git", + "knot-lfs", + "knot-pack", + "knot-resource", + "knot-runtime", + "knot-types", + "sha2 0.11.0", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tracing", + "walkdir", +] + +[[package]] +name = "knot-messages" +version = "2.0.0" +dependencies = [ + "confique", + "thiserror 2.0.18", +] + +[[package]] +name = "knot-migrate" +version = "2.0.0" +dependencies = [ + "base64", + "chrono", + "knot-cob", + "knot-cobs", + "knot-config", + "knot-git", + "knot-index", + "knot-runtime", + "knot-secrets", + "knot-types", + "rusqlite", + "rustix", + "serde", + "serde_json", + "ssh-key", + "tempfile", + "thiserror 2.0.18", + "url", + "walkdir", + "zeroize", +] + +[[package]] +name = "knot-pack" +version = "2.0.0" +dependencies = [ + "axum", + "bytes", + "flate2", + "gix", + "gix-hash", + "gix-pack", + "gix-packetline", + "h3", + "h3-quinn", + "http", + "http-body-util", + "knot-bench", + "knot-cache", + "knot-cob", + "knot-cobs", + "knot-edge", + "knot-fixtures", + "knot-git", + "knot-index", + "knot-maintenance", + "knot-messages", + "knot-resource", + "knot-runtime", + "knot-types", + "proptest", + "quinn", + "rcgen 0.14.8", + "rustls", + "scc", + "tempfile", + "thiserror 2.0.18", + "tikv-jemallocator", + "tokio", + "tokio-stream", + "tokio-util", + "tower", + "tracing-subscriber", + "url", + "walkdir", +] + +[[package]] +name = "knot-postreceive" +version = "2.0.0" +dependencies = [ + "knot-events", + "knot-fixtures", + "knot-git", + "knot-langs", + "knot-messages", + "knot-runtime", + "knot-types", + "knot-workflow", + "serde_json", + "tempfile", + "tracing", + "url", +] + +[[package]] +name = "knot-receive" +version = "2.0.0" +dependencies = [ + "knot-atproto", + "knot-cob", + "knot-events", + "knot-git", + "knot-index", + "knot-maintenance", + "knot-messages", + "knot-pack", + "knot-postreceive", + "knot-resource", + "knot-runtime", + "knot-types", + "tokio", + "tracing", +] + +[[package]] +name = "knot-resource" +version = "2.0.0" +dependencies = [ + "knot-types", + "rustix", + "tempfile", + "tokio", +] + +[[package]] +name = "knot-runtime" +version = "2.0.0" +dependencies = [ + "bytes", + "futures", + "getrandom 0.4.3", + "hickory-resolver", + "http", + "k256", + "knot-types", + "reqwest 0.13.1", + "serde", + "thiserror 2.0.18", + "tokio", + "url", +] + +[[package]] +name = "knot-secrets" +version = "2.0.0" +dependencies = [ + "aes-gcm", + "base64", + "hkdf", + "k256", + "knot-resource", + "knot-runtime", + "knot-types", + "serde", + "serde_json", + "sha2 0.11.0", + "tempfile", + "thiserror 2.0.18", + "zeroize", +] + +[[package]] +name = "knot-server" +version = "2.0.0" +dependencies = [ + "anyhow", + "axum", + "base64", + "confique", + "http", + "knot-atproto", + "knot-cache", + "knot-config", + "knot-edge", + "knot-events", + "knot-git", + "knot-index", + "knot-lfs", + "knot-maintenance", + "knot-messages", + "knot-pack", + "knot-postreceive", + "knot-resource", + "knot-runtime", + "knot-secrets", + "knot-ssh", + "knot-types", + "knot-xrpc", + "rustix", + "tempfile", + "tikv-jemalloc-ctl", + "tikv-jemallocator", + "tokio", + "tokio-util", + "tower-http 0.7.0", + "tracing", + "tracing-subscriber", + "walkdir", + "zeroize", +] + +[[package]] +name = "knot-sim" +version = "2.0.0" +dependencies = [ + "axum", + "base64", + "bytes", + "futures", + "h3", + "h3-quinn", + "http", + "knot-atproto", + "knot-cob", + "knot-cobs", + "knot-config", + "knot-edge", + "knot-events", + "knot-fixtures", + "knot-git", + "knot-index", + "knot-lfs", + "knot-maintenance", + "knot-messages", + "knot-pack", + "knot-resource", + "knot-runtime", + "knot-secrets", + "knot-ssh", + "knot-types", + "knot-xrpc", + "quinn", + "rcgen 0.14.8", + "rustls", + "serde", + "serde_json", + "sha2 0.11.0", + "tempfile", + "tokio", + "tokio-util", + "tower", + "url", +] + +[[package]] +name = "knot-ssh" +version = "2.0.0" +dependencies = [ + "bytes", + "futures", + "http", + "knot-acl", + "knot-atproto", + "knot-cob", + "knot-cobs", + "knot-events", + "knot-fixtures", + "knot-git", + "knot-index", + "knot-lfs", + "knot-maintenance", + "knot-messages", + "knot-pack", + "knot-postreceive", + "knot-receive", + "knot-resource", + "knot-runtime", + "knot-types", + "russh", + "serde_json", + "sha2 0.11.0", + "tempfile", + "thiserror 2.0.18", + "tikv-jemalloc-ctl", + "tikv-jemallocator", + "tokio", + "tokio-util", + "tracing", + "url", +] + +[[package]] +name = "knot-types" +version = "2.0.0" +dependencies = [ + "gix-hash", + "http", + "jacquard-common", + "proptest", + "serde", + "serde_json", + "thiserror 2.0.18", + "url", +] + +[[package]] +name = "knot-workflow" +version = "2.0.0" +dependencies = [ + "globset", + "knot-types", + "serde", + "serde_json", + "serde_norway", +] + +[[package]] +name = "knot-xrpc" +version = "2.0.0" +dependencies = [ + "axum", + "base64", + "bytes", + "chrono", + "futures", + "http", + "http-body", + "httpdate", + "k256", + "knot-acl", + "knot-atproto", + "knot-cache", + "knot-cob", + "knot-cobs", + "knot-config", + "knot-events", + "knot-fixtures", + "knot-git", + "knot-index", + "knot-langs", + "knot-lfs", + "knot-maintenance", + "knot-messages", + "knot-pack", + "knot-postreceive", + "knot-receive", + "knot-resource", + "knot-runtime", + "knot-secrets", + "knot-types", + "serde", + "serde_json", + "sha2 0.11.0", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tokio-tungstenite 0.29.0", + "tokio-util", + "tower", + "tower-http 0.7.0", + "tracing", + "url", +] + +[[package]] +name = "kstring" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "558bf9508a558512042d3095138b1f7b8fe90c5467d94f9f1da28b3731c5dbd1" +dependencies = [ + "static_assertions", ] [[package]] @@ -2183,12 +5005,6 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] -name = "leb128fmt" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" - -[[package]] name = "levenshtein_automata" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2199,6 +5015,23 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libsqlite3-sys" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95b4103cffefa72eb8428cb6b47d6627161e51c2739fc5e3b734584157bc642a" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" @@ -2223,9 +5056,9 @@ [[package]] name = "log" -version = "0.4.30" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "loom" @@ -2247,6 +5080,15 @@ checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" dependencies = [ "hashbrown 0.16.1", +] + +[[package]] +name = "lru-cache" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" +dependencies = [ + "linked-hash-map", ] [[package]] @@ -2282,7 +5124,7 @@ dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2301,6 +5143,23 @@ checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" [[package]] +name = "maybe-async" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "746873a384ad60adc5db74471dfaba74bd278afbdcfd81db93fafcdfc8b5ca0c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "md5" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0" + +[[package]] name = "measure_time" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2311,9 +5170,9 @@ [[package]] name = "memchr" -version = "2.8.1" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" [[package]] name = "memmap2" @@ -2352,7 +5211,7 @@ dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2360,6 +5219,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] [[package]] name = "minimal-lexical" @@ -2379,13 +5248,58 @@ [[package]] name = "mio" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", "wasi", "windows-sys 0.61.2", +] + +[[package]] +name = "ml-kem" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e15f3e5b957493873e396a66914e83e616b6afe335cdef7efe5c6e1216aba66" +dependencies = [ + "hybrid-array", + "kem", + "module-lattice", + "pkcs8 0.11.0", + "rand_core 0.10.1", + "sha3", +] + +[[package]] +name = "module-lattice" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c61b87c9683ab7cb1c6871d261ad5479b6b10ceb52c4352aaca3b5d35a8febe" +dependencies = [ + "ctutils", + "hybrid-array", + "num-traits", +] + +[[package]] +name = "moka" +version = "0.12.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "event-listener", + "futures-util", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", ] [[package]] @@ -2464,7 +5378,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ - "bitflags", + "bitflags 2.13.0", "cfg-if", "cfg_aliases", "libc", @@ -2482,6 +5396,24 @@ ] [[package]] +name = "nonempty" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9e591e719385e6ebaeb5ce5d3887f7d5676fceca6411d1925ccc95745f3d6f7" + +[[package]] +name = "nonempty" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9737e026353e5cd0736f98eddae28665118eb6f6600902a7f50db585621fecb6" + +[[package]] +name = "nonzero_ext" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" + +[[package]] name = "nu-ansi-term" version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2491,10 +5423,29 @@ ] [[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] name = "num-conv" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] [[package]] name = "num-traits" @@ -2513,6 +5464,24 @@ dependencies = [ "hermit-abi", "libc", +] + +[[package]] +name = "oid-registry" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d8034d9489cdaf79228eb9f6a3b8d7bb32ba00d6645ebd48eef4077ceb5bd9" +dependencies = [ + "asn1-rs 0.6.2", +] + +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs 0.7.2", ] [[package]] @@ -2564,30 +5533,6 @@ ] [[package]] -name = "ouroboros" -version = "0.18.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0f050db9c44b97a94723127e6be766ac5c340c48f2c4bb3ffa11713744be59" -dependencies = [ - "aliasable", - "ouroboros_macro", - "static_assertions", -] - -[[package]] -name = "ouroboros_macro" -version = "0.18.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c7028bdd3d43083f6d8d4d5187680d0d3560d54df4cc9d752005268b41e64d0" -dependencies = [ - "heck 0.4.1", - "proc-macro2", - "proc-macro2-diagnostics", - "quote", - "syn", -] - -[[package]] name = "ownedbytes" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2611,10 +5556,51 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" dependencies = [ - "ecdsa", - "elliptic-curve", - "primeorder", - "sha2", + "ecdsa 0.16.9", + "elliptic-curve 0.13.8", + "primeorder 0.13.6", + "sha2 0.10.9", +] + +[[package]] +name = "p256" +version = "0.14.0-rc.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41adc63effe99d48837a8cc0e6d7a77e32ae6a07f6000df466178dbc2193093e" +dependencies = [ + "ecdsa 0.17.0-rc.18", + "elliptic-curve 0.14.0-rc.33", + "primefield", + "primeorder 0.14.0-rc.10", + "sha2 0.11.0", +] + +[[package]] +name = "p384" +version = "0.14.0-rc.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bd5333afa5ae0347f39e6a0f2c9c155da431583fd71fe5555bd0521b4ccaf02" +dependencies = [ + "ecdsa 0.17.0-rc.18", + "elliptic-curve 0.14.0-rc.33", + "fiat-crypto", + "primefield", + "primeorder 0.14.0-rc.10", + "sha2 0.11.0", +] + +[[package]] +name = "p521" +version = "0.14.0-rc.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3a5297f53dc16d35909060ba3032cff7867e8809f01e273ff325579d5f0ceae" +dependencies = [ + "base16ct 1.0.0", + "ecdsa 0.17.0-rc.18", + "elliptic-curve 0.14.0-rc.33", + "primefield", + "primeorder 0.14.0-rc.10", + "sha2 0.11.0", ] [[package]] @@ -2625,6 +5611,26 @@ dependencies = [ "libc", "winapi", +] + +[[package]] +name = "pageant" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f3a5ae18f65a85c67a77d18d42d3606c07948e3c17c1e5f74852b26589e88a5" +dependencies = [ + "base16ct 1.0.0", + "byteorder", + "bytes", + "delegate", + "futures", + "log", + "rand 0.10.1", + "sha2 0.11.0", + "thiserror 2.0.18", + "tokio", + "windows", + "windows-strings", ] [[package]] @@ -2657,16 +5663,54 @@ ] [[package]] +name = "password-hash" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aab41826031698d6ffcd9cff78ef56ef998e39dc7e5067cdfebe373842d4723b" +dependencies = [ + "phc", +] + +[[package]] name = "paste" version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] +name = "pbkdf2" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629" +dependencies = [ + "digest 0.11.3", + "hmac 0.13.0", +] + +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64", + "serde_core", +] + +[[package]] name = "pem-rfc7468" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "pem-rfc7468" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" dependencies = [ "base64ct", ] @@ -2686,6 +5730,16 @@ "fixedbitset", "hashbrown 0.15.5", "indexmap", +] + +[[package]] +name = "phc" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44dc769b75f93afdddd8c7fa12d685292ddeff1e66f7f0f3a234cf1818afe892" +dependencies = [ + "base64ct", + "ctutils", ] [[package]] @@ -2718,7 +5772,7 @@ "phf_shared", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2747,7 +5801,7 @@ dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2757,13 +5811,62 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] +name = "piper" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1" +dependencies = [ + "atomic-waker", + "fastrand", + "futures-io", +] + +[[package]] +name = "pkcs1" +version = "0.8.0-rc.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "986d2e952779af96ea048f160fd9194e1751b4faea78bcf3ceb456efe008088e" +dependencies = [ + "der 0.8.0", + "spki 0.8.0", +] + +[[package]] +name = "pkcs5" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "279a91971a1d8eb1260a30938eae3be9cb67b472dffecb222fbbbe2fd2dc1453" +dependencies = [ + "aes", + "cbc", + "der 0.8.0", + "pbkdf2", + "rand_core 0.10.1", + "scrypt", + "sha2 0.11.0", + "spki 0.8.0", +] + +[[package]] name = "pkcs8" version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", - "spki", + "der 0.7.10", + "spki 0.7.3", +] + +[[package]] +name = "pkcs8" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" +dependencies = [ + "der 0.8.0", + "pkcs5", + "rand_core 0.10.1", + "spki 0.8.0", ] [[package]] @@ -2773,10 +5876,55 @@ checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "poly1305" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00baa632505d05512f48a963e16051c54fda9a95cc9acea1a4e3c90991c4a2e" +dependencies = [ + "cpufeatures 0.3.0", + "universal-hash", + "zeroize", +] + +[[package]] +name = "polyval" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dfc63250416fea14f5749b90725916a6c903f599d51cb635aa7a52bfd03eede" +dependencies = [ + "cpubits", + "cpufeatures 0.3.0", + "universal-hash", +] + +[[package]] name = "portable-atomic" version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] [[package]] name = "postcard" @@ -2787,7 +5935,7 @@ "cobs", "embedded-io 0.4.0", "embedded-io 0.6.1", - "heapless", + "heapless 0.7.17", "serde", ] @@ -2822,7 +5970,21 @@ checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.118", +] + +[[package]] +name = "primefield" +version = "0.14.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8675564771a62f69a0af716b03e89b917b963c7b173b5855575e84fd4f605ca0" +dependencies = [ + "crypto-bigint 0.7.4", + "crypto-common 0.2.2", + "ff 0.14.0", + "rand_core 0.10.1", + "subtle", + "zeroize", ] [[package]] @@ -2831,7 +5993,38 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" dependencies = [ - "elliptic-curve", + "elliptic-curve 0.13.8", +] + +[[package]] +name = "primeorder" +version = "0.14.0-rc.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d2793f22b9b6fd11ef3ac1d59bf003c2573593e4968702341605c2748fd90bf" +dependencies = [ + "elliptic-curve 0.14.0-rc.33", +] + +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] @@ -2844,23 +6037,40 @@ ] [[package]] -name = "proc-macro2-diagnostics" -version = "0.10.1" +name = "prodash" +version = "31.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +checksum = "962200e2d7d551451297d9fdce85138374019ada198e30ea9ede38034e27604c" dependencies = [ - "proc-macro2", - "quote", - "syn", - "version_check", - "yansi", + "bytesize", + "human_format", + "parking_lot", +] + +[[package]] +name = "proptest" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" +dependencies = [ + "bit-set", + "bit-vec 0.8.0", + "bitflags 2.13.0", + "num-traits", + "rand 0.9.4", + "rand_chacha 0.9.0", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", ] [[package]] name = "prost" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" dependencies = [ "bytes", "prost-derive", @@ -2868,11 +6078,11 @@ [[package]] name = "prost-build" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" +checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ - "heck 0.5.0", + "heck", "itertools 0.14.0", "log", "multimap", @@ -2881,21 +6091,21 @@ "prost", "prost-types", "regex", - "syn", + "syn 2.0.118", "tempfile", ] [[package]] name = "prost-derive" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", "itertools 0.14.0", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2932,9 +6142,9 @@ [[package]] name = "prost-reflect" -version = "0.16.4" +version = "0.16.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "590aa145fee8f7a26b5a6055365e7c5e89a5c1caae9869de76ec0ee73181a2f9" +checksum = "01b80ea363c31af2de2b92e3c07ed1156628f7838c4afb4df75ee78a37fedbd1" dependencies = [ "base64", "prost", @@ -2946,9 +6156,9 @@ [[package]] name = "prost-reflect-build" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8214ae2c30bbac390db0134d08300e770ef89b6d4e5abf855e8d300eded87e28" +checksum = "95a9e8261adf6617d5dc2a5a9e75cce5ab9d546a007f6f870f809a1ad25386b6" dependencies = [ "prost-build", "prost-reflect", @@ -2956,29 +6166,50 @@ [[package]] name = "prost-reflect-derive" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b6d90e29fa6c0d13c2c19ba5e4b3fb0efbf5975d27bcf4e260b7b15455bcabe" +checksum = "30320eb03b43b7dfcaf9b361f808a4f1adad1e718ad219df1d7e4283e34e73f5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] name = "prost-types" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" dependencies = [ "prost", ] [[package]] -name = "quick_cache" -version = "0.6.22" +name = "quanta" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1c821816e9b928e20e92ed59bb3ac4aab321d16ca2316871c9fe7ca739cd477" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quick_cache" +version = "0.6.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9c6658afe513a3b484e3abfdaa0d03ef3c0bbf017542c178dd55f94eb3051f9" dependencies = [ "ahash", "equivalent", @@ -2994,6 +6225,7 @@ dependencies = [ "bytes", "cfg_aliases", + "futures-io", "pin-project-lite", "quinn-proto", "quinn-udp", @@ -3012,6 +6244,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ + "aws-lc-rs", "bytes", "getrandom 0.3.4", "lru-slab", @@ -3084,6 +6317,17 @@ ] [[package]] +name = "rand" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] name = "rand_chacha" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3122,6 +6366,36 @@ ] [[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core 0.9.5", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags 2.13.0", +] + +[[package]] +name = "rawzip" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d9575f44c8cf85bc843ad666dcdf20d05a7753772bef56eb2a5140282b32150" + +[[package]] name = "rayon" version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3142,12 +6416,39 @@ ] [[package]] +name = "rcgen" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" +dependencies = [ + "aws-lc-rs", + "pem", + "rustls-pki-types", + "time", + "yasna 0.5.2", +] + +[[package]] +name = "rcgen" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57f6d249aad744e274e682777a50283a225a32705394ee6d5fcc01efa25e4055" +dependencies = [ + "aws-lc-rs", + "pem", + "rustls-pki-types", + "time", + "x509-parser 0.18.1", + "yasna 0.6.0", +] + +[[package]] name = "redox_syscall" version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags", + "bitflags 2.13.0", ] [[package]] @@ -3167,14 +6468,14 @@ dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] name = "regex" -version = "1.12.3" +version = "1.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" dependencies = [ "aho-corasick", "memchr", @@ -3201,9 +6502,9 @@ [[package]] name = "regex-syntax" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "reqwest" @@ -3239,15 +6540,63 @@ "tokio-rustls", "tokio-util", "tower", - "tower-http", + "tower-http 0.6.11", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 1.0.7", + "webpki-roots 1.0.8", ] + +[[package]] +name = "reqwest" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04e9018c9d814e5f30cc16a0f03271aeab3571e609612d9fe78c1aa8d11c2f62" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http 0.6.11", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots 1.0.8", +] + +[[package]] +name = "resolv-conf" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" [[package]] name = "rfc6979" @@ -3255,7 +6604,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" dependencies = [ - "hmac", + "hmac 0.12.1", + "subtle", +] + +[[package]] +name = "rfc6979" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5236ce872cac07e0fb3969b0cbf468c7d2f37d432f1b627dcb7b8d34563fb0c3" +dependencies = [ + "hmac 0.13.0", "subtle", ] @@ -3269,8 +6628,148 @@ "cfg-if", "getrandom 0.2.17", "libc", - "untrusted", + "untrusted 0.9.0", "windows-sys 0.52.0", +] + +[[package]] +name = "rsa" +version = "0.10.0-rc.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b2aa4ba0d89f73d1e332df05be0eeab8840351c36ca5654341dfdb57bb3caf" +dependencies = [ + "const-oid 0.10.2", + "crypto-bigint 0.7.4", + "crypto-primes", + "digest 0.11.3", + "pkcs1", + "pkcs8 0.11.0", + "rand_core 0.10.1", + "sha2 0.11.0", + "signature 3.0.0", + "spki 0.8.0", + "zeroize", +] + +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror 2.0.18", +] + +[[package]] +name = "rusqlite" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1c93dd1c9683b438c392c492109cb702b8090b2bfc8fed6f6e4eb4523f17af3" +dependencies = [ + "bitflags 2.13.0", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", + "sqlite-wasm-rs", +] + +[[package]] +name = "russh" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbf893f64684e58da8a68d56a5e84d1cf0440226274c515770fe267707a7d0b0" +dependencies = [ + "aes", + "aws-lc-rs", + "bitflags 2.13.0", + "block-padding", + "byteorder", + "bytes", + "cbc", + "cipher", + "crypto-bigint 0.7.4", + "ctr", + "curve25519-dalek", + "data-encoding", + "delegate", + "der 0.8.0", + "digest 0.11.3", + "ecdsa 0.17.0-rc.18", + "ed25519-dalek", + "elliptic-curve 0.14.0-rc.33", + "enum_dispatch", + "flate2", + "futures", + "generic-array 1.4.3", + "getrandom 0.4.3", + "ghash", + "hex-literal", + "hmac 0.13.0", + "inout", + "internal-russh-num-bigint", + "keccak", + "log", + "md5", + "ml-kem", + "module-lattice", + "num-bigint", + "p256 0.14.0-rc.10", + "p384", + "p521", + "pageant", + "pbkdf2", + "pkcs1", + "pkcs5", + "pkcs8 0.11.0", + "polyval", + "rand 0.10.1", + "rand_core 0.10.1", + "rsa", + "russh-cryptovec", + "russh-util", + "salsa20", + "scrypt", + "sec1 0.8.1", + "sha1 0.11.0", + "sha2 0.11.0", + "sha3", + "signature 3.0.0", + "spki 0.8.0", + "ssh-encoding", + "ssh-key", + "subtle", + "thiserror 2.0.18", + "tokio", + "typenum", + "universal-hash", + "zeroize", +] + +[[package]] +name = "russh-cryptovec" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "443f6bbcfacb34a1aab2b12b99bf08e0c63abdc5a0db261901365df9d57fff51" +dependencies = [ + "log", + "nix", + "ssh-encoding", + "windows-sys 0.61.2", +] + +[[package]] +name = "russh-util" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668424a5dde0bcb45b55ba7de8476b93831b4aa2fa6947e145f3b053e22c60b6" +dependencies = [ + "chrono", + "tokio", + "wasm-bindgen", + "wasm-bindgen-futures", ] [[package]] @@ -3299,12 +6798,21 @@ ] [[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + +[[package]] name = "rustix" version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.13.0", "errno", "libc", "linux-raw-sys", @@ -3317,6 +6825,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ + "aws-lc-rs", + "log", "once_cell", "ring", "rustls-pki-types", @@ -3326,15 +6836,50 @@ ] [[package]] -name = "rustls-native-certs" -version = "0.8.3" +name = "rustls-acme" +version = "0.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +checksum = "b9c70a17ecb067d5067565a16a2e0f26a4a2ea0924f49739d558c45186facc75" +dependencies = [ + "async-io", + "async-trait", + "async-web-client", + "aws-lc-rs", + "base64", + "blocking", + "chrono", + "futures", + "futures-rustls", + "http", + "log", + "pem", + "rcgen 0.13.2", + "serde", + "serde_json", + "thiserror 2.0.18", + "webpki-roots 1.0.8", + "x509-parser 0.16.0", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", ] [[package]] @@ -3348,14 +6893,42 @@ ] [[package]] +name = "rustls-platform-verifier" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + +[[package]] name = "rustls-webpki" version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", - "untrusted", + "untrusted 0.9.0", ] [[package]] @@ -3363,6 +6936,18 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "rusty-fork" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] [[package]] name = "ryu" @@ -3377,6 +6962,16 @@ checksum = "68f5acb362a0e75c2a963532fa7fabf13dff81626dc494df16488d30befcbea0" [[package]] +name = "salsa20" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f874456e72520ff1375a06c588eaf074b0f01f9e9e1aada45bd9b7954a6e42c" +dependencies = [ + "cfg-if", + "cipher", +] + +[[package]] name = "same-file" version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3387,9 +6982,9 @@ [[package]] name = "scc" -version = "3.7.1" +version = "3.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bcd12b6caff5213cc3c03123cde8c3db5e413008a63b0c0ba35e6275825ea92" +checksum = "5581cd5dd2cb79cbe9d8137071d67f62f0db926e83a07b4d14561c5b7d423776" dependencies = [ "saa", "sdd", @@ -3417,6 +7012,18 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] +name = "scrypt" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d87af57419b594aa23fa95f09f0e06d80d84ba01c26148c43844cad6ff4485f0" +dependencies = [ + "cfg-if", + "pbkdf2", + "salsa20", + "sha2 0.11.0", +] + +[[package]] name = "sdd" version = "4.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3431,10 +7038,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ - "base16ct", - "der", - "generic-array", - "pkcs8", + "base16ct 0.2.0", + "der 0.7.10", + "generic-array 0.14.9", + "pkcs8 0.10.2", + "subtle", + "zeroize", +] + +[[package]] +name = "sec1" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d" +dependencies = [ + "base16ct 1.0.0", + "ctutils", + "der 0.8.0", + "hybrid-array", "subtle", "zeroize", ] @@ -3445,7 +7066,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags", + "bitflags 2.13.0", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -3521,7 +7142,7 @@ dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3562,6 +7183,19 @@ ] [[package]] +name = "serde_norway" +version = "0.9.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e408f29489b5fd500fab51ff1484fc859bb655f32c671f307dcd733b72e8168c" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml-norway", +] + +[[package]] name = "serde_path_to_error" version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3580,7 +7214,7 @@ dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3606,9 +7240,9 @@ [[package]] name = "serde_with" -version = "3.20.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" dependencies = [ "base64", "bs58", @@ -3622,14 +7256,37 @@ [[package]] name = "serde_with_macros" -version = "3.20.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b90c488738ecb4fb0262f41f43bc40efc5868d9fb744319ddf5f5317f417bfac" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" dependencies = [ "darling", "proc-macro2", "quote", - "syn", + "syn 2.0.118", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "serdect" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66cf8fedced2fcf12406bcb34223dffb92eaf34908ede12fed414c82b7f00b3e" +dependencies = [ + "base16ct 1.0.0", + "serde", ] [[package]] @@ -3639,8 +7296,29 @@ checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + +[[package]] +name = "sha1-checked" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89f599ac0c323ebb1c6082821a54962b839832b03984598375bff3975b804423" +dependencies = [ + "digest 0.10.7", + "sha1 0.10.6", ] [[package]] @@ -3650,8 +7328,29 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + +[[package]] +name = "sha3" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be176f1a57ce4e3d31c1a166222d9768de5954f811601fb7ca06fc8203905ce1" +dependencies = [ + "digest 0.11.3", + "keccak", ] [[package]] @@ -3664,10 +7363,16 @@ ] [[package]] -name = "shlex" -version = "1.3.0" +name = "shell-words" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "shuttle" @@ -3690,6 +7395,16 @@ ] [[package]] +name = "signal-hook" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a0c28ca5908dbdbcd52e6fdaa00358ab88637f8ab33e1f188dd510eb44b53d" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] name = "signal-hook-registry" version = "1.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3705,8 +7420,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "digest", + "digest 0.10.7", "rand_core 0.6.4", +] + +[[package]] +name = "signature" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" +dependencies = [ + "digest 0.11.3", + "rand_core 0.10.1", ] [[package]] @@ -3738,9 +7463,9 @@ [[package]] name = "smallvec" -version = "1.15.1" +version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "smol_str" @@ -3754,9 +7479,9 @@ [[package]] name = "socket2" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", "windows-sys 0.61.2", @@ -3778,13 +7503,105 @@ checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" [[package]] +name = "spinning_top" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d96d2d1d716fb500937168cc09353ffdc7a012be8475ac7308e1bdf0e3923300" +dependencies = [ + "lock_api", +] + +[[package]] name = "spki" version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der", + "der 0.7.10", +] + +[[package]] +name = "spki" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f" +dependencies = [ + "base64ct", + "der 0.8.0", +] + +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + +[[package]] +name = "ssh-cipher" +version = "0.3.0-rc.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10db6f219196a8528f9ec904d9d45cdad692d65b0e57e72be4dedd1c5fddce36" +dependencies = [ + "aead", + "aes", + "aes-gcm", + "cbc", + "chacha20", + "cipher", + "ctr", + "ctutils", + "des", + "poly1305", + "ssh-encoding", + "zeroize", +] + +[[package]] +name = "ssh-encoding" +version = "0.3.0-rc.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7abf34aa716da5d5b4c496936d042ea282ab392092cd68a72ef6a8863ff8c96a" +dependencies = [ + "base64ct", + "bytes", + "crypto-bigint 0.7.4", + "ctutils", + "digest 0.11.3", + "pem-rfc7468 1.0.0", + "zeroize", +] + +[[package]] +name = "ssh-key" +version = "0.7.0-rc.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45735ce3dea95690e4a9e414c4cfde7f79835063c3dcd35881df85a84118e74b" +dependencies = [ + "argon2", + "bcrypt-pbkdf", + "ctutils", + "ed25519-dalek", + "hex", + "hmac 0.13.0", + "p256 0.14.0-rc.10", + "p384", + "p521", + "rand_core 0.10.1", + "rsa", + "sec1 0.8.1", + "sha1 0.11.0", + "sha2 0.11.0", + "signature 3.0.0", + "ssh-cipher", + "ssh-encoding", + "zeroize", ] [[package]] @@ -3813,9 +7630,20 @@ [[package]] name = "syn" -version = "2.0.117" +version = "2.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" dependencies = [ "proc-macro2", "quote", @@ -3839,7 +7667,7 @@ dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3848,7 +7676,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags", + "bitflags 2.13.0", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -3862,6 +7690,12 @@ "core-foundation-sys", "libc", ] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" [[package]] name = "tantivy" @@ -4012,14 +7846,35 @@ ] [[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] name = "tempfile" version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.2", + "getrandom 0.4.3", "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "terminal_size" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" +dependencies = [ "rustix", "windows-sys 0.61.2", ] @@ -4050,7 +7905,7 @@ dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -4061,7 +7916,7 @@ dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -4106,12 +7961,11 @@ [[package]] name = "time" -version = "0.3.47" +version = "0.3.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +checksum = "711a53c2d47bbd818258c498c8dbfe186a2526c631495cfe7e078567f86b8469" dependencies = [ "deranged", - "itoa", "num-conv", "powerfmt", "serde_core", @@ -4121,15 +7975,15 @@ [[package]] name = "time-core" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.27" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +checksum = "71c652a3727a9cbb9a02f707f530b618ce00d0ccd762009c8c23bd191df3c17d" dependencies = [ "num-conv", "time-core", @@ -4194,7 +8048,7 @@ dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -4346,6 +8200,7 @@ "pin-project-lite", "sync_wrapper", "tokio", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -4358,7 +8213,7 @@ checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "async-compression", - "bitflags", + "bitflags 2.13.0", "bytes", "futures-core", "futures-util", @@ -4371,8 +8226,34 @@ "tower", "tower-layer", "tower-service", - "tracing", "url", +] + +[[package]] +name = "tower-http" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233" +dependencies = [ + "async-compression", + "bitflags 2.13.0", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "http-range-header", + "httpdate", + "mime", + "mime_guess", + "percent-encoding", + "pin-project-lite", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", ] [[package]] @@ -4386,6 +8267,22 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tower_governor" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44de9b94d849d3c46e06a883d72d408c2de6403367b39df2b1c9d9e7b6736fe6" +dependencies = [ + "axum", + "forwarded-header-value", + "governor", + "http", + "pin-project", + "thiserror 2.0.18", + "tower", + "tracing", +] [[package]] name = "tracing" @@ -4407,7 +8304,7 @@ dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -4470,7 +8367,7 @@ dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -4494,7 +8391,7 @@ "rand 0.8.6", "rustls", "rustls-pki-types", - "sha1", + "sha1 0.10.6", "thiserror 1.0.69", "utf-8", ] @@ -4513,7 +8410,7 @@ "rand 0.9.4", "rustls", "rustls-pki-types", - "sha1", + "sha1 0.10.6", "thiserror 2.0.18", ] @@ -4531,15 +8428,15 @@ [[package]] name = "typenum" -version = "1.20.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "typetag" -version = "0.2.22" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5a897b12c6c1151ad0b138b8db50252dc301f93bc3b027db05eec82aeed298c" +checksum = "c90e86058a30d42a1a928dfb4b49bb33c98c3a2b4909492e6b0881cd94798ec2" dependencies = [ "erased-serde", "inventory", @@ -4550,14 +8447,41 @@ [[package]] name = "typetag-impl" -version = "0.2.22" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf808357c6ed7e13ba0f3277ec8d8f21b2d501274895104263985330c726c1c5" +checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] + +[[package]] +name = "uluru" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c8a2469e56e6e5095c82ccd3afb98dad95f7af7929aab6d8ba8d6e0f73657da" +dependencies = [ + "arrayvec", +] + +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + +[[package]] +name = "unicode-bom" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eec5d1121208364f6793f7d2e222bf75a915c19557537745b195b253dd64217" [[package]] name = "unicode-ident" @@ -4566,10 +8490,19 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] -name = "unicode-segmentation" -version = "1.13.2" +name = "unicode-normalization" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" [[package]] name = "unicode-width" @@ -4584,10 +8517,38 @@ checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] +name = "universal-hash" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4987bdc12753382e0bec4a65c50738ffaabc998b9cdd1f952fb5f39b0048a96" +dependencies = [ + "crypto-common 0.2.2", + "ctutils", +] + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +[[package]] +name = "unsafe-libyaml-norway" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39abd59bf32521c7f2301b52d05a6a2c975b6003521cbd0c6dc1582f0a22104" + +[[package]] name = "unsigned-varint" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "untrusted" @@ -4634,11 +8595,11 @@ [[package]] name = "uuid" -version = "1.23.1" +version = "1.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" dependencies = [ - "getrandom 0.4.2", + "getrandom 0.4.3", "js-sys", "serde_core", "wasm-bindgen", @@ -4649,6 +8610,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "version_check" @@ -4664,6 +8631,15 @@ dependencies = [ "libc", "nix", +] + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", ] [[package]] @@ -4693,27 +8669,18 @@ [[package]] name = "wasip2" -version = "1.0.3+wasi-0.2.9" +version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ - "wit-bindgen 0.57.1", -] - -[[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" -dependencies = [ - "wit-bindgen 0.51.0", + "wit-bindgen", ] [[package]] name = "wasm-bindgen" -version = "0.2.122" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" dependencies = [ "cfg-if", "once_cell", @@ -4724,9 +8691,9 @@ [[package]] name = "wasm-bindgen-futures" -version = "0.4.72" +version = "0.4.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" +checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" dependencies = [ "js-sys", "wasm-bindgen", @@ -4734,9 +8701,9 @@ [[package]] name = "wasm-bindgen-macro" -version = "0.2.122" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4744,46 +8711,24 @@ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.122" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.118", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.122" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" dependencies = [ "unicode-ident", -] - -[[package]] -name = "wasm-encoder" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" -dependencies = [ - "leb128fmt", - "wasmparser", -] - -[[package]] -name = "wasm-metadata" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" -dependencies = [ - "anyhow", - "indexmap", - "wasm-encoder", - "wasmparser", ] [[package]] @@ -4800,22 +8745,10 @@ ] [[package]] -name = "wasmparser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" -dependencies = [ - "bitflags", - "hashbrown 0.15.5", - "indexmap", - "semver", -] - -[[package]] name = "web-sys" -version = "0.3.99" +version = "0.3.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436" +checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" dependencies = [ "js-sys", "wasm-bindgen", @@ -4832,22 +8765,37 @@ ] [[package]] +name = "webpki-root-certs" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" +dependencies = [ + "rustls-pki-types", +] + +[[package]] name = "webpki-roots" version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.7", + "webpki-roots 1.0.8", ] [[package]] name = "webpki-roots" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" dependencies = [ "rustls-pki-types", ] + +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" [[package]] name = "winapi" @@ -4881,6 +8829,27 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + +[[package]] name = "windows-core" version = "0.62.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -4894,6 +8863,17 @@ ] [[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + +[[package]] name = "windows-implement" version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -4901,7 +8881,7 @@ dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -4912,7 +8892,7 @@ dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -4920,6 +8900,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] [[package]] name = "windows-registry" @@ -4948,6 +8938,15 @@ checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" dependencies = [ "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", ] [[package]] @@ -4988,6 +8987,21 @@ [[package]] name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" @@ -5020,6 +9034,21 @@ ] [[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -5033,6 +9062,12 @@ [[package]] name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" @@ -5042,6 +9077,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" [[package]] name = "windows_i686_gnu" @@ -5069,6 +9110,12 @@ [[package]] name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" @@ -5078,6 +9125,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" [[package]] name = "windows_x86_64_gnu" @@ -5093,6 +9146,12 @@ [[package]] name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" @@ -5102,6 +9161,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" [[package]] name = "windows_x86_64_msvc" @@ -5152,97 +9217,9 @@ [[package]] name = "wit-bindgen" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" -dependencies = [ - "wit-bindgen-rust-macro", -] - -[[package]] -name = "wit-bindgen" version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" - -[[package]] -name = "wit-bindgen-core" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" -dependencies = [ - "anyhow", - "heck 0.5.0", - "wit-parser", -] - -[[package]] -name = "wit-bindgen-rust" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" -dependencies = [ - "anyhow", - "heck 0.5.0", - "indexmap", - "prettyplease", - "syn", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" -dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" -dependencies = [ - "anyhow", - "bitflags", - "indexmap", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder", - "wasm-metadata", - "wasmparser", - "wit-parser", -] - -[[package]] -name = "wit-parser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" -dependencies = [ - "anyhow", - "id-arena", - "indexmap", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", - "wasmparser", -] [[package]] name = "writeable" @@ -5251,16 +9228,74 @@ checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] -name = "yansi" -version = "1.0.1" +name = "x509-parser" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" +checksum = "fcbc162f30700d6f3f82a24bf7cc62ffe7caea42c0b2cba8bf7f3ae50cf51f69" +dependencies = [ + "asn1-rs 0.6.2", + "data-encoding", + "der-parser 9.0.0", + "lazy_static", + "nom", + "oid-registry 0.7.1", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs 0.7.2", + "aws-lc-rs", + "data-encoding", + "der-parser 10.0.0", + "lazy_static", + "nom", + "oid-registry 0.8.1", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + +[[package]] +name = "yasna" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" +dependencies = [ + "bit-vec 0.9.1", + "time", +] [[package]] name = "yoke" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -5275,28 +9310,28 @@ dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.49" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bce33a6288fa3f072a8c2c7d0f2fdbb90e28298f0135c1f99b96c3db2efcc60b" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.49" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd425244944f4ab65ccff928e7323354c5a018c75838362fdce749dfad2ee1e" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -5316,15 +9351,29 @@ dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] [[package]] name = "zeroize" -version = "1.8.2" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] [[package]] name = "zerotrie" @@ -5356,8 +9405,14 @@ dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] + +[[package]] +name = "zlib-rs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" [[package]] name = "zmij" diff --git a/Cargo.toml b/Cargo.toml --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] -resolver = "2" -members = ["bobbin/crates/*", "shuttle"] +resolver = "3" +members = ["bobbin/crates/*", "shuttle", "knot2/crates/*"] exclude = ["sites"] [workspace.package] @@ -41,24 +41,63 @@ bobbin-sim = { path = "bobbin/crates/bobbin-sim" } bobbin-xrpc = { path = "bobbin/crates/xrpc" } -jacquard-common = "0.12.0-beta.2" -jacquard-derive = "0.12.0-beta.2" -jacquard-lexicon = { version = "0.12.0-beta.2", default-features = false } +knot-fixtures = { path = "knot2/crates/knot-fixtures" } +knot-lexicons = { path = "knot2/crates/knot-lexicons" } +knot-types = { path = "knot2/crates/knot-types" } +knot-resource = { path = "knot2/crates/knot-resource" } +knot-config = { path = "knot2/crates/knot-config" } +knot-runtime = { path = "knot2/crates/knot-runtime" } +knot-git = { path = "knot2/crates/knot-git" } +knot-langs = { path = "knot2/crates/knot-langs" } +knot-lfs = { path = "knot2/crates/knot-lfs" } +knot-workflow = { path = "knot2/crates/knot-workflow" } +knot-pack = { path = "knot2/crates/knot-pack" } +knot-cob = { path = "knot2/crates/knot-cob" } +knot-cobs = { path = "knot2/crates/knot-cobs" } +knot-index = { path = "knot2/crates/knot-index" } +knot-cache = { path = "knot2/crates/knot-cache" } +knot-acl = { path = "knot2/crates/knot-acl" } +knot-atproto = { path = "knot2/crates/knot-atproto" } +knot-messages = { path = "knot2/crates/knot-messages" } +knot-postreceive = { path = "knot2/crates/knot-postreceive" } +knot-receive = { path = "knot2/crates/knot-receive" } +knot-ssh = { path = "knot2/crates/knot-ssh" } +knot-secrets = { path = "knot2/crates/knot-secrets" } +knot-events = { path = "knot2/crates/knot-events" } +knot-xrpc = { path = "knot2/crates/knot-xrpc" } +knot-bench = { path = "knot2/crates/knot-bench" } +knot-maintenance = { path = "knot2/crates/knot-maintenance" } +knot-sim = { path = "knot2/crates/knot-sim" } +knot-edge = { path = "knot2/crates/knot-edge" } + +jacquard-common = "0.12.1" +jacquard-derive = "0.12.1" +jacquard-lexicon = { version = "0.12.1", default-features = false } +jacquard-repo = "0.12.1" + +gix = { version = "0.84", features = ["parallel", "revision", "blob-diff", "worktree-archive", "tree-editor", "sha1", "sha256"] } +gix-pack = { version = "0.71", default-features = false, features = ["generate", "streaming-input", "sha1", "sha256"] } +gix-packetline = { version = "0.21", features = ["blocking-io"] } +gix-archive = "0.33" +gix-hash = { version = "0.25", features = ["sha1", "sha256"] } +flate2 = "1" anyhow = "1" miette = "7" walkdir = "2" -tokio = { version = "1.52", features = ["macros", "rt-multi-thread", "time", "signal", "io-util", "sync"] } -tokio-util = "0.7" +tokio = { version = "1.52", features = ["macros", "rt-multi-thread", "time", "signal", "io-util", "net", "sync"] } +tokio-util = { version = "0.7", features = ["rt"] } tokio-stream = "0.1" tokio-tungstenite = { version = "0.29", features = ["rustls-tls-webpki-roots"] } futures = "0.3" either = "1" +async-trait = "0.1" serde = { version = "1", features = ["derive"] } serde_json = { version = "1", features = ["raw_value"] } serde_ipld_dagcbor = "0.6" +serde_norway = "0.9" chrono = { version = "0.4", features = ["serde"] } cid = "0.11" @@ -69,18 +108,55 @@ roaring = "0.11" lasso = { version = "0.7", features = ["multi-threaded"] } quick_cache = "0.6" -getrandom = "0.3" +moka = { version = "0.12", features = ["sync", "future"] } +getrandom = "0.4" ahash = { version = "0.8", default-features = false, features = ["std"] } +arc-swap = "1" -reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-webpki-roots", "http2", "json", "gzip", "stream"] } +reqwest = { version = "0.13", default-features = false, features = ["rustls", "webpki-roots", "http2", "json", "gzip", "stream"] } axum = "0.8" -tower = { version = "0.5", features = ["util"] } -tower-http = { version = "0.6", features = ["trace"] } +hyper = { version = "1", features = ["server", "http1", "http2"] } +hyper-util = { version = "0.1", features = ["server", "server-auto", "tokio", "service"] } +tower = { version = "0.5", features = ["util", "limit", "load-shed"] } +tower-http = { version = "0.7", default-features = false } +tower_governor = { version = "0.8", default-features = false, features = ["axum"] } +governor = "0.10" http = "1" +http-body = "1" +httpdate = "1" + +rustls = { version = "0.23", features = ["aws_lc_rs", "prefer-post-quantum"] } +tokio-rustls = { version = "0.26", default-features = false, features = ["aws_lc_rs", "tls12", "logging"] } +rustls-pemfile = "2" +rustls-acme = { version = "0.15.3", default-features = false, features = ["aws-lc-rs", "tls12", "webpki-roots"] } +x509-parser = "0.18" +quinn = { version = "0.11.9", default-features = false, features = ["runtime-tokio", "rustls-aws-lc-rs", "log"] } +h3 = "0.0.8" +h3-quinn = "0.0.10" +rcgen = { version = "0.14", default-features = false, features = ["aws_lc_rs", "pem"] } +hickory-resolver = "0.24" + +k256 = { version = "0.13", features = ["ecdsa"] } +aes-gcm = "0.11.0-rc.4" +hkdf = "0.13" +sha2 = "0.11" +base32 = "0.5" +base64 = "0.22" +bs58 = "0.5" +zeroize = { version = "1", features = ["derive"] } wiremock = "0.6" +tempfile = "3" +proptest = "1" +divan = "0.1.21" tantivy = "0.26" +gengo-language = "0.14" +regex = "1" +globset = "0.4" +rustix = { version = "1.1", features = ["process"] } +tikv-jemallocator = "0.7" +tikv-jemalloc-ctl = "0.7" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt", "json"] } @@ -91,11 +167,16 @@ clap = { version = "4", features = ["derive", "env"] } toml = { version = "0.9", default-features = false, features = ["parse"] } +[patch.crates-io] +gix-pack = { path = "knot2/third_party/gix-pack" } + +[profile.dev] +opt-level = 1 + [profile.release] lto = "fat" strip = true codegen-units = 1 -panic = "abort" [profile.bench] debug = 1 diff --git a/flake.nix b/flake.nix --- a/flake.nix +++ b/flake.nix @@ -122,8 +122,10 @@ (fs.unions [ ./Cargo.toml ./Cargo.lock + ./lexicons ./shuttle ./bobbin + ./knot2 ]); }; buildGoApplication = diff --git a/knot2/.gitignore b/knot2/.gitignore new file mode 100644 --- /dev/null +++ b/knot2/.gitignore @@ -0,0 +1,11 @@ +/target +**/target + +crates/knot-lexicons/src/_lex/ + +crates/knot-pack/fuzz/corpus/ +crates/knot-pack/fuzz/artifacts/ + +config.toml + +*.swp diff --git a/knot2/Containerfile b/knot2/Containerfile new file mode 100644 --- /dev/null +++ b/knot2/Containerfile @@ -0,0 +1,21 @@ +FROM docker.io/library/rust:1.96-slim-trixie AS builder +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates build-essential cmake perl pkg-config clang mold \ + && rm -rf /var/lib/apt/lists/* +ENV RUSTFLAGS="-C linker=clang -C link-arg=-fuse-ld=mold" +WORKDIR /src +COPY Cargo.toml Cargo.lock rust-toolchain.toml ./ +COPY bobbin/crates ./bobbin/crates +COPY shuttle ./shuttle +COPY lexicons ./lexicons +COPY knot2/lexicons ./knot2/lexicons +COPY knot2/crates ./knot2/crates +COPY knot2/third_party ./knot2/third_party +RUN cargo build --release --package knot-server +RUN strip target/release/knot-server + +FROM gcr.io/distroless/cc-debian13:latest@sha256:1e3c6d9c255be500eb680cdea0ad07554f52ae92dfcbdf07043a2a435b4c1fe3 +COPY --from=builder /src/target/release/knot-server /usr/local/bin/knot-server +EXPOSE 5555 2222 +ENTRYPOINT ["/usr/local/bin/knot-server"] +CMD ["/etc/knot/config.toml"] diff --git a/knot2/README.md b/knot2/README.md new file mode 100644 --- /dev/null +++ b/knot2/README.md @@ -0,0 +1,332 @@ +# Knot 2 + +> Lewis 🦪 +> +> Not sure if this will be upstreamed by Tangled the company, but I'm mentioning that possibility if it concerns the reader. +> I work at Tangled after all! + +This is an alternate implementation of a Tangled knot server! + +In Tangled, a "knot" is simply a git server that does its auth layer over the AT Protocol. +Essentially, it tastes like this: + +1. Atproto users declare their public SSH key for themselves, in their PDS. +2. A knot admin (defined by atproto DID in the knot config) makes a request to the knot to allow membership to a given atproto user. +3. Said user can now create/update/delete a git repo on the knot, and do normal git things to that git repo over SSH. +4. Said user makes a request to the knot to allow more atproto users as collaborators on their specific git repo. + +The reader will notice this disconnect between making requests to the knot, vs. making requests to atproto (individual users' PDSes). The social side of git repos (PRs, issues, etc.) are "owned" by atproto at time of writing; in contrast, important ACL data (members, collaborators), lives on the knot as the source of truth. As time goes on we are re-assessing the idea of users owning what is "collaborative data" (issues, PRs, etc.) on their PDSes - soon may come the day that an issue also lives on the knot as a source of truth, with an accompanying pointer record on user PDS to attest that it's theirs. + +Back to knot 2 today: + +- **It has no database, or should I say, git is the only database, along with a secret-key-file.** +- It aims to be small, fast, and modern - for example: http3/quic support, SHA256 by default, and no need for unix `git` user, to name a couple of features. +- Config variables are in example.toml, just make a config.toml from that and go ham. + +Have fun! + +# Running a knot2 + +The following is how I actually run `knot.oyster.cafe`. Please treat it as one possible setup. + +The soon-to-be knot operator will need 3 things before beginning: +1. A remote computer, preferably one that stays online +1. A domain name directed at the remote computer +1. The operator's own atproto DID, which becomes the knot's admin + +If the operator doesn't know their DID, resolve the handle at [pds.ls](https://pds.ls) or with any atproto tool that does identity resolution. + +## Configuration + +`example.toml` at project root is generated from the code, +so it always ought to be up to date with what's possible on knot2. +Copy it and fill in the required values: + +```sh +cp example.toml config.toml +``` + +The most important values have no default and thus the knot won't start without them: + +- `server.hostname`: Public hostname, which will also be the knot's identity as `did:web:`, so it oughtta be a public, good, solid, representative name. +- `server.admins`: List of atproto DIDs, where the first one is the knot's "full owner", which is the DID the operator registers the knot under & what `sh.tangled.owner` returns. Every other admin has the same powers, just no claim to the fancy title, the way tangled is *currently* set up. Admin & minions. +- `server.ssh_host_key_file`: Path for the SSH host key. +- `repo.scan_path`: Path to the directory that gets the actual git repos. +- `secrets.sealed_key_file`: Path to the sealed key store. +- `secrets.master_key_env`: Name of the env var holding the master key. +- `atproto.plc_directory`: PLC directory URL, which is "normally" `https://plc.directory`. There is deliberately no default here because I don't want Bluesky-defaultism. The operator chooses their own if they please. + +Some of those can denote files that don't exist yet, because it's more like *where* to put them, since a knot can for sure create files but won't choose paths for the operator. The host key and the sealed store both get created on demand, parent directories and all. Directories are a little different in that `repo.scan_path` and `lfs.store_path` (if the operator sets that) have to already exist and be writable, or the knot won't start. After that, `scan_path` gets filled out as repos arrive. + +Every key outside the `[messages]` block can also come from an environment variable, as one can see in the `example.toml`. Environment variables win over the file if both are specified btw. + +## Master key + +The master key is for unsealing the per-repo signing keys. One has to generate 32 bytes of base64 and keep it out of the config file, for opsec: + +```sh +openssl rand -base64 32 +``` + +Put it in an env file that only root can read, then `secrets.master_key_env` at the variable name: + +```sh +# knot.env +KNOT_MASTER_KEY= +``` + +**Back this up somewhere separate from the server.** The sealed key store on disk is useless without it, and every repo-DID on a knot is derived from keys inside that store. If the operator loses the master key, the repos will of course stay readable as regular git, but the knot will no longer be able to prove ownership to atproto. + +## Ports, Taylor's version (ha ha, 22 joke) + +A knot has two listeners where HTTP defaults to `[::]:5555`, SSH defaults to `[::]:2222`. + +Thus far in Tangled in general, SSH is heavily used because the existing Tangled knot doesn't do HTTP pushing. Tangled generally auto-generates a URL for SSH git'ing that looks like this: + +``` +git clone git@knot.oyster.cafe:did:plc:barnacle +``` + +However notice that from Tangled's client, it builds a clone URL with no port number in, because it always assumes 22. If the knot's SSH listener is listening on another port, all of its users have gotta either rewrite the URL as `ssh://git@knot.oyster.cafe:2222/did:plc:barnacle`, or add a block to their `~/.ssh/config`. + +So **give the knot port 22 if one can**! It doesn't need unix `git` user / shell account, so the only thing in the way is the remote computer's own sshd, which is usually already taking port 22. + +If the reader is nodding along instead of saying "no Lewis I won't change the remote computer's sshd because I like not bricking it" then: + +Moving sshd is something that can lock one out of one's own server, so do it in this order: + +1. Open a second SSH session to the server and keep it open for this whole procedure. If step 4 goes wrong, having this session open might be the saving grace. +2. Edit `/etc/ssh/sshd_config` and set `Port 2200` or whatever free port one likes. Leave `Port 22` in place as well for now, so sshd listens on both. +3. Open the new port in the firewall if there is one. On ufw that's `ufw allow 2200/tcp` (I think!! Untested). On a cloud provider one will probably also have to deal with it / open it in their proprietary config. +4. Restart sshd, and **from the local computer, in a third terminal**, confirm `ssh -p 2200 root@the.server` works before continuing. +5. Once confirmed, only now remove `Port 22` from `sshd_config`, restart sshd once more, & give the port to the knot. When running the binary directly, that means `ssh_listen_addr = "[::]:22"`. Comparatively, in a container it entails publishing the container's 2222 as the host's 22, which is what the compose file example below does. + +If one would rather not move sshd at all, another cool option for having knot2 on port 22 is a second IP address on the remote computer. Bind sshd to one with `ListenAddress`, bind the knot to the other with `ssh_listen_addr = ":22"`, and just put the knot's DNS record on that second address. Leaving the knot on `[::]:22` would wildcard-bind every address on the box and would collide with sshd no matter which single IP that sshd listens on. + +HTTP can stay on 5555 behind a reverse proxy, or move to 443 if one wants the knot to terminate TLS itself. + +## Running with containers + +The `Containerfile` at project root builds a distroless image with just the `knot-server` binary in it: + +```sh +podman build -t knot-oyster:latest . +``` + +I personally run it with a composefile. This is the file from `knot.oyster.cafe` with a few opsec adjustments: + +```yaml +services: + knot: + image: localhost/knot-oyster:latest + container_name: knot-oyster + pull_policy: never + restart: unless-stopped + mem_limit: 2g + env_file: ./knot.env + ports: + - "0.0.0.0:22:2222" + - "[::]:22:2222" + volumes: + - ./config.toml:/etc/knot/config.toml:ro + - ./repos:/data/repos + - ./ssh:/data/ssh + - ./secrets:/data/secrets + - ./lfs:/data/lfs +``` + +The container keeps listening on 2222 internally and the host publishes that as 22, so the config file never has to change. My own instance publishes 2222 on the host because that computer already had sshd on 22 when I set it up, and my laziness has been regrettable until knot2 came with http pushing. + +Here's the corresponding config, with path specifying the mounted volumes: + +```toml +[server] +hostname = "knot.oyster.cafe" +admins = ["did:plc:nel"] # well obviously this isn't a real DID but one gets the picture +listen_addr = "[::]:5555" +ssh_listen_addr = "[::]:2222" +ssh_host_key_file = "/data/ssh/host_key" +appview_endpoint = "https://tangled.org" # this is *not* Tangled defaultism, it's cosmetic for git operation messaging + +[repo] +scan_path = "/data/repos" + +[secrets] +sealed_key_file = "/data/secrets/sealed.bin" +master_key_env = "KNOT_MASTER_KEY" + +[atproto] +plc_directory = "https://plc.directory" + +[xrpc] +trusted_proxy_header = "x-forwarded-for" + +[git] +object_format = "sha256" + +[lfs] +store_path = "/data/lfs" +free_space_floor_bytes = 32212254720 +``` + +Create the dirs, then let there be light I suppose: + +```sh +mkdir -p repos ssh secrets lfs +podman-compose up -d +``` + +The `mkdir` is necessary, since the knot won't start unless the repo and LFS directories are present/writable. Podman would create the bind-mount sources for the operator, but then they belong to whichever unix user podman has rather than to the operator. + +The knot creates the SSH host key on the first run at mode 600, aaand the sealed store on that same first run, because the knot's own signing key needs sealing before any repo exists. The knot purposefully won't load a host key that is group or other readable, so don't loosen those please. + +Note that this (my) config turns LFS on, since `lfs.store_path` is set. One can drop that whole `[lfs]` block if one doesn't want it. The floor of 30GiB is what I have judged for my disk (of 500GiB, doing other things at the same time), so pick something that suits one's own rather than copying mine. + +Speaking of LFS, I made the directory different in the first place so that we could specify a whole separate storage medium if wanted. For example, let's say I want my actual git repos to be wicked fast, so everything *else* is on an SSD, and *only* LFS is on a massive-but-relatively-cheap HDD cluster. Wouldn't want terabytes and terabytes of massive files taking up precious SSD space in this economy! + +## TLS + +My setup has Caddy in front, which gets me certificates for free and lets one machine serve several sites (which it does). The config is: + +``` +knot.oyster.cafe { + reverse_proxy knot-oyster:5555 +} +``` + +That talks to the knot by container name, which needs both containers on a single podman network. One creates such a network once with something like `podman network create tangled`, then add the network to the compose file above as an external one and to whatever runs the proxy. If one would rather not, publish `127.0.0.1:5555:5555` from the knot container and proxy to that instead. + +Set `xrpc.trusted_proxy_header = "x-forwarded-for"` when doing this, otherwise every client looks like it comes from the proxy and the ratelimiter wil treat them as one very busy mister. Only set it behind a proxy the operator controls, since a direct client can like, invent that header. + +The knot can also terminate TLS itself (and that's the only way to get its HTTP3 support) because a plain TCP frontend can't proxy QUIC. Using a certificate the operator already manages: + +```toml +[server] +listen_addr = "[::]:443" + +[tls] +cert_path = "/data/tls/fullchain.pem" +key_path = "/data/tls/privkey.pem" +``` + +Or let it fetch its own via ACME: + +```toml +[server] +listen_addr = "[::]:443" + +[tls] +acme_enabled = true +acme_cache_dir = "/data/acme" +acme_contact = "nel@oyster.cafe" +``` + +ACME here uses the TLS-ALPN-01 challenge, so the knot has to be the reciever (in the phone sense) answering on 443 for the configured hostname. Set `acme_staging = true` while testing such that a typo doesn't nuke the Let's Encrypt ratelimit. Leave `trusted_proxy_header` unset in this mode, and open 443/udp in the firewall if one wants HTTP3 to be reachable. + +## First run + +```sh +curl -s https://knot.oyster.cafe/xrpc/_health +curl -s https://knot.oyster.cafe/xrpc/sh.tangled.owner +curl -s https://knot.oyster.cafe/.well-known/did.json +``` + +`_health` reports the version, and if LFS is on it returns an error status when the LFS store isn't writable. `sh.tangled.owner` returns the first DID in `server.admins`, which is what the Tangled appview reads to confirm the operator is who they say they are when they register the knot. `did.json` is the knot's own `did:web` document, served at the hostname it was configured with. + +If any of the above endpoints doesn't pong, check out the stderr logs, `podman logs -f knot-oyster` in my case. + +## Letting people on + +Sharing is caring! + +If one wants the knot to show up nicely on Tangled the web app, register the knot on [tangled.org](https://tangled.org) with the same DID listed first in `server.admins`. + +Admission is closed by default, so an admin adds each member before they can create repos. Repo owners can then add their own collaborators without any admin involvement of course. If one wants a knot anyone can use: + +```toml +[acl] +admission = "open" +``` + +Note that open admission still respects the blocklist, naturally. + +Since ssh-pushing is so popular, users will usually authenticate with the SSH key they published on their PDS -> the most common support question the operator may get is that a user's ssh client agent offers five keys and gets rejected before it ever reaches the registered one. The fix on their side is: + +``` +Host knot.oyster.cafe + IdentityFile ~/.ssh/id_ed25519 + IdentitiesOnly yes +``` + +## Clone URLs + +A git repo can be pushed/pulled by its owner + name, or by its own repo-DID. The owner can be a DID or an atproto handle. + +Over SSH: + +```sh +git clone knot.oyster.cafe:nel.pet/squid +git clone knot.oyster.cafe:did:plc:nel/squid +git clone knot.oyster.cafe:did:plc:barnacle +``` + +Over HTTPS, the same: + +```sh +git clone https://knot.oyster.cafe/nel.pet/squid +git clone https://knot.oyster.cafe/did:plc:nel/squid +git clone https://knot.oyster.cafe/did:plc:barnacle +``` + +Push works over both, of course . For HTTP pushing, it is up to the user to find a good Tangled-CLI or something that can put the right things in the git credential helper such that a service auth token is minted and used on push. + +A trailing `.git` on the repo name is optional, so `did:plc:nel/squid.git` goes to the same repo as `did:plc:nel/squid`. That only applies to the repo name variant though - `did:plc:barnacle.git` is read as a DID rather than as a repo-DID with a suffix, and it won't resolve. This would be made better from better DID parsing, since a `did:plc` can't have dots, only a `did:web` can. + +## Things worth knowing before one commits (get it?) to a config + +### SHA-256 is the default + +Yeah, sorry, let's modernize. + +`git.object_format` defaults to `sha256`, and it applies to repos at creation time. A SHA-256 repo cannot be pushed to or fetched from a SHA-1 repo, so if one expects users mirroring in from elsewhere, set `object_format = "sha1"` before anyone creates anything. Changing it later only affects new repos. + +### LFS is off until one supplies a path + +LFS turns on for both transports when `lfs.store_path` is set. `free_space_floor_bytes` is the disk headroom below which the knot starts refusing uploads, defaulting to 1GiB, so set it to the amount of free space one actually wants to keep. + +### Resources tune themselves + +`resources.max_threads` and `resources.max_memory_bytes` are smart-ceilings, both `0` by default meaning "use the whole computer". Under a container memory limit the knot reads the cgroup and sizes itself to that, so a `mem_limit` on the container usually suffices. + +### Maintenance runs on its own + +Commit-graphs, multi-pack indexes, bitmaps, and geometric repacks happen every 6 hours by default. Turn it all off with `maintenance.enabled = false` if one would rather do it oneself. + +### The homepage is replaceable, please do replace it + +`homepage.path` serves an HTML file of one's choice at `/`, and `homepage.enabled = false` disables the homepage entirely. + +## Backups + +Back these things up or don't come cryin' to me! + +1. The master key, wherever one keeps it. +2. `secrets/sealed.bin`, the sealed key store. It's useless without the master key & the master key is useless without it, so treat them as a pair. +3. `repos/`, which is every repo plus the knot's own ACL data. Git is the database, so that one directory is the knot's entire state. +4. `lfs/`, if one enables it. +5. I guess the SSH host key, though much less dire if it is lost and has to be changed. + +## Updating + +// TODO: publish to ATCR, maybe nix something something. + +```sh +git pull +podman build -t knot-oyster:latest . +podman-compose up -d +``` + +The knot drains at `SIGTERM` time, so in-flight clones/pushes get up to 40s to finish before they're cut off. + +Happy knotting! + diff --git a/knot2/_typos.toml b/knot2/_typos.toml new file mode 100644 --- /dev/null +++ b/knot2/_typos.toml @@ -0,0 +1,4 @@ +[files] +extend-exclude = ["lexicons/", "**/_lex/", "Cargo.lock", "pack-spike/"] + +[default.extend-words] diff --git a/knot2/example.toml b/knot2/example.toml new file mode 100644 --- /dev/null +++ b/knot2/example.toml @@ -0,0 +1,600 @@ +[server] +# Can also be specified via environment variable `KNOT_HOSTNAME`. +# Required! This value must be specified. +#hostname = + +# Can also be specified via environment variable `KNOT_ADMINS`. +# Required! This value must be specified. +#admins = + +# Can also be specified via environment variable `KNOT_LISTEN_ADDR`. +# Default value: "[::]:5555" +#listen_addr = "[::]:5555" + +# Can also be specified via environment variable `KNOT_LISTEN_HEADER_TIMEOUT_MS`. +# Default value: 10000 +#listen_header_timeout_ms = 10000 + +# Can also be specified via environment variable `KNOT_LISTEN_IDLE_TIMEOUT_MS`. +# Default value: 60000 +#listen_idle_timeout_ms = 60000 + +# Can also be specified via environment variable `KNOT_LISTEN_MAX_CONNECTIONS`. +# Default value: 1024 +#listen_max_connections = 1024 + +# Can also be specified via environment variable `KNOT_LISTEN_RATE_LIMIT_PER_SECOND`. +# Default value: 50 +#listen_rate_limit_per_second = 50 + +# Can also be specified via environment variable `KNOT_LISTEN_RATE_LIMIT_BURST`. +# Default value: 200 +#listen_rate_limit_burst = 200 + +# Can also be specified via environment variable `KNOT_LISTEN_MAX_INFLIGHT_REQUESTS`. +# Default value: 1024 +#listen_max_inflight_requests = 1024 + +# Can also be specified via environment variable `KNOT_LISTEN_REQUEST_TIMEOUT_MS`. +# Default value: 60000 +#listen_request_timeout_ms = 60000 + +# Can also be specified via environment variable `KNOT_LISTEN_BODY_TIMEOUT_MS`. +# Default value: 30000 +#listen_body_timeout_ms = 30000 + +# Can also be specified via environment variable `KNOT_LISTEN_WRITE_REQUEST_TIMEOUT_MS`. +# Default value: 1800000 +#listen_write_request_timeout_ms = 1800000 + +# Can also be specified via environment variable `KNOT_INTERNAL_LISTEN_ADDR`. +# Default value: "[::1]:5444" +#internal_listen_addr = "[::1]:5444" + +# Can also be specified via environment variable `KNOT_SSH_LISTEN_ADDR`. +# Default value: "[::]:2222" +#ssh_listen_addr = "[::]:2222" + +# Can also be specified via environment variable `KNOT_SSH_HOST_KEY_FILE`. +# Required! This value must be specified. +#ssh_host_key_file = + +# Can also be specified via environment variable `KNOT_SSH_MAX_PACK_BYTES`. +# Default value: 8589934592 +#ssh_max_pack_bytes = 8589934592 + +# Can also be specified via environment variable `KNOT_APPVIEW_ENDPOINT`. +# Default value: "https://tangled.org" +#appview_endpoint = "https://tangled.org" + +[tls] +# Can also be specified via environment variable `KNOT_TLS_CERT_PATH`. +#cert_path = + +# Can also be specified via environment variable `KNOT_TLS_KEY_PATH`. +#key_path = + +# Can also be specified via environment variable `KNOT_TLS_HTTP3`. +# Default value: true +#http3 = true + +# Can also be specified via environment variable `KNOT_TLS_ACME_ENABLED`. +# Default value: false +#acme_enabled = false + +# Can also be specified via environment variable `KNOT_TLS_ACME_CACHE_DIR`. +#acme_cache_dir = + +# Can also be specified via environment variable `KNOT_TLS_ACME_CONTACT`. +#acme_contact = + +# Can also be specified via environment variable `KNOT_TLS_ACME_STAGING`. +# Default value: false +#acme_staging = false + +# Can also be specified via environment variable `KNOT_TLS_MTLS_ENABLED`. +# Default value: false +#mtls_enabled = false + +# Can also be specified via environment variable `KNOT_TLS_MTLS_CLIENT_CA_PATH`. +#mtls_client_ca_path = + +# Can also be specified via environment variable `KNOT_TLS_MTLS_ADMIN_SPKI_PIN`. +#mtls_admin_spki_pin = + +[acl] +# Can also be specified via environment variable `KNOT_ADMISSION`. +# Default value: "closed" +#admission = "closed" + +[repo] +# Can also be specified via environment variable `KNOT_SCAN_PATH`. +# Required! This value must be specified. +#scan_path = + +# Can also be specified via environment variable `KNOT_DEFAULT_BRANCH`. +# Default value: "main" +#default_branch = "main" + +[git] +# Committer identity stamped on merge commits the knot creates. +# +# Can also be specified via environment variable `KNOT_GIT_USER_NAME`. +# +# Default value: "Tangled" +#user_name = "Tangled" + +# Can also be specified via environment variable `KNOT_GIT_USER_EMAIL`. +# Default value: "noreply@tangled.sh" +#user_email = "noreply@tangled.sh" + +# Can also be specified via environment variable `KNOT_GIT_OBJECT_FORMAT`. +# Default value: "sha256" +#object_format = "sha256" + +[secrets] +# Can also be specified via environment variable `KNOT_SEALED_KEY_FILE`. +# Required! This value must be specified. +#sealed_key_file = + +# Can also be specified via environment variable `KNOT_MASTER_KEY_ENV`. +# Required! This value must be specified. +#master_key_env = + +[http] +# Can also be specified via environment variable `KNOT_HTTP_CONNECT_TIMEOUT_MS`. +# Default value: 5000 +#connect_timeout_ms = 5000 + +# Can also be specified via environment variable `KNOT_HTTP_READ_TIMEOUT_MS`. +# Default value: 30000 +#read_timeout_ms = 30000 + +# Can also be specified via environment variable `KNOT_HTTP_REQUEST_TIMEOUT_MS`. +# Default value: 60000 +#request_timeout_ms = 60000 + +# Can also be specified via environment variable `KNOT_HTTP_MAX_RESPONSE_BYTES`. +# Default value: 16777216 +#max_response_bytes = 16777216 + +[atproto] +# Can also be specified via environment variable `KNOT_PLC_DIRECTORY`. +# Required! This value must be specified. +#plc_directory = + +[xrpc] +# Can also be specified via environment variable `KNOT_XRPC_MAX_BODY_BYTES`. +# Default value: 65536 +#max_body_bytes = 65536 + +# Can also be specified via environment variable `KNOT_XRPC_MAX_RESPONSE_BYTES`. +# Default value: 5242880 +#max_response_bytes = 5242880 + +# Can also be specified via environment variable `KNOT_XRPC_MAX_ARCHIVE_BYTES`. +# Default value: 1073741824 +#max_archive_bytes = 1073741824 + +# Can also be specified via environment variable `KNOT_XRPC_TREE_LAST_COMMIT_BUDGET_MS`. +# Default value: 300 +#tree_last_commit_budget_ms = 300 + +# Can also be specified via environment variable `KNOT_XRPC_BLOB_LAST_COMMIT_BUDGET_MS`. +# Default value: 2000 +#blob_last_commit_budget_ms = 2000 + +# Can also be specified via environment variable `KNOT_XRPC_LANGUAGES_BUDGET_MS`. +# Default value: 1000 +#languages_budget_ms = 1000 + +# Can also be specified via environment variable `KNOT_XRPC_LANGUAGES_PUSH_BUDGET_MS`. +# Default value: 2000 +#languages_push_budget_ms = 2000 + +# Body limit for the merge and mergeCheck procedures, whose patch payloads +# routinely exceed the general XRPC body limit. +# +# Can also be specified via environment variable `KNOT_XRPC_MAX_PATCH_BYTES`. +# +# Default value: 16777216 +#max_patch_bytes = 16777216 + +# Limit on the total decompressed size of a patch the merge procedures parse, +# bounding binary-delta inflation and hunk expansion apart from the +# compressed body limit above. +# +# Can also be specified via environment variable `KNOT_XRPC_MAX_PATCH_DECOMPRESSED_BYTES`. +# +# Default value: 134217728 +#max_patch_decompressed_bytes = 134217728 + +# Can also be specified via environment variable `KNOT_XRPC_PREAUTH_BURST`. +# Default value: 20 +#preauth_burst = 20 + +# Can also be specified via environment variable `KNOT_XRPC_PREAUTH_REFILL_MS`. +# Default value: 100 +#preauth_refill_ms = 100 + +# Can also be specified via environment variable `KNOT_XRPC_PER_PEER_INFLIGHT`. +# Default value: 8 +#per_peer_inflight = 8 + +# Can also be specified via environment variable `KNOT_XRPC_GLOBAL_INFLIGHT`. +# Default value: 64 +#global_inflight = 64 + +# Can also be specified via environment variable `KNOT_XRPC_MAX_PENDING_RESERVATIONS`. +# Default value: 256 +#max_pending_reservations = 256 + +# Per-account limit on reserved repository keys awaiting creation, so one +# account cannot consume the whole pending-reservation budget. +# +# Can also be specified via environment variable `KNOT_XRPC_PER_ACTOR_RESERVATIONS`. +# +# Default value: 32 +#per_actor_reservations = 32 + +# How long a reserved repository key is held before it lapses and its +# sealed key is reclaimed, in seconds. +# +# Can also be specified via environment variable `KNOT_XRPC_RESERVATION_TTL_SECS`. +# +# Default value: 3600 +#reservation_ttl_secs = 3600 + +# Can also be specified via environment variable `KNOT_XRPC_FORK_MAX_PACK_BYTES`. +# Default value: 1073741824 +#fork_max_pack_bytes = 1073741824 + +# Can also be specified via environment variable `KNOT_XRPC_FORK_FETCH_TIMEOUT_MS`. +# Default value: 600000 +#fork_fetch_timeout_ms = 600000 + +# When the knot runs behind a trusted reverse proxy that terminates TLS, +# set this to the header the proxy appends the client address to, for +# example x-forwarded-for. The rightmost entry is used. Leave unset when +# the knot is directly exposed so the socket peer address is used. Only set +# this when a trusted proxy overwrites or appends the header, since a client +# can forge it otherwise. +# +# Can also be specified via environment variable `KNOT_XRPC_TRUSTED_PROXY_HEADER`. +#trusted_proxy_header = + +# Can also be specified via environment variable `KNOT_XRPC_EVENTS_REPLAY_BUFFER`. +# Default value: 4096 +#events_replay_buffer = 4096 + +# Can also be specified via environment variable `KNOT_XRPC_EVENTS_REPLAY_BYTES`. +# Default value: 67108864 +#events_replay_bytes = 67108864 + +# Can also be specified via environment variable `KNOT_XRPC_EVENTS_MAX_SUBSCRIBERS`. +# Default value: 256 +#events_max_subscribers = 256 + +# Can also be specified via environment variable `KNOT_XRPC_EVENTS_MAX_PER_PEER`. +# Default value: 8 +#events_max_per_peer = 8 + +[maintenance] +# Can also be specified via environment variable `KNOT_MAINTENANCE_ENABLED`. +# Default value: true +#enabled = true + +# Can also be specified via environment variable `KNOT_MAINTENANCE_COMMIT_GRAPH`. +# Default value: true +#commit_graph = true + +# Can also be specified via environment variable `KNOT_MAINTENANCE_MULTI_PACK_INDEX`. +# Default value: true +#multi_pack_index = true + +# Can also be specified via environment variable `KNOT_MAINTENANCE_BITMAP`. +# Default value: true +#bitmap = true + +# Can also be specified via environment variable `KNOT_MAINTENANCE_INTERVAL_SECS`. +# Default value: 21600 +#interval_secs = 21600 + +# Can also be specified via environment variable `KNOT_MAINTENANCE_REPACK_MAX_OBJECTS`. +# Default value: 16000000 +#repack_max_objects = 16000000 + +# Can also be specified via environment variable `KNOT_MAINTENANCE_REPACK_GEOMETRIC_FACTOR`. +# Default value: 2 +#repack_geometric_factor = 2 + +# Can also be specified via environment variable `KNOT_MAINTENANCE_PRUNE_GRACE_SECS`. +# Default value: 1209600 +#prune_grace_secs = 1209600 + +# Can also be specified via environment variable `KNOT_MAINTENANCE_REFLOG_EXPIRE_SECS`. +# Default value: 7776000 +#reflog_expire_secs = 7776000 + +# Can also be specified via environment variable `KNOT_MAINTENANCE_LARGE_PUSH_BYTES`. +# Default value: 52428800 +#large_push_bytes = 52428800 + +[pack_cache] +# Can also be specified via environment variable `KNOT_PACK_CACHE_ENABLED`. +# Default value: true +#enabled = true + +# Can also be specified via environment variable `KNOT_PACK_CACHE_TTL_SECS`. +# Default value: 60 +#ttl_secs = 60 + +# Can also be specified via environment variable `KNOT_PACK_CACHE_MAX_ENTRY_BYTES`. +# Default value: 67108864 +#max_entry_bytes = 67108864 + +# Can also be specified via environment variable `KNOT_PACK_CACHE_MAX_TOTAL_BYTES`. +# Default value: 2147483648 +#max_total_bytes = 2147483648 + +[pack] +# Can also be specified via environment variable `KNOT_PACK_MAX_OBJECTS`. +# Default value: 16000000 +#max_objects = 16000000 + +# Can also be specified via environment variable `KNOT_PACK_MAX_TOTAL_BYTES`. +# Default value: 68719476736 +#max_total_bytes = 68719476736 + +# Can also be specified via environment variable `KNOT_PACK_SELECTION_MAX_OBJECTS`. +# Default value: 16000000 +#selection_max_objects = 16000000 + +# Can also be specified via environment variable `KNOT_PACK_SELECTION_TIME_BUDGET_SECS`. +# Default value: 600 +#selection_time_budget_secs = 600 + +[lfs] +# Can also be specified via environment variable `KNOT_LFS_STORE_PATH`. +#store_path = + +# Can also be specified via environment variable `KNOT_LFS_MAX_OBJECT_BYTES`. +# Default value: 5368709120 +#max_object_bytes = 5368709120 + +# Can also be specified via environment variable `KNOT_LFS_FREE_SPACE_FLOOR_BYTES`. +# Default value: 1073741824 +#free_space_floor_bytes = 1073741824 + +# Can also be specified via environment variable `KNOT_LFS_GC_GRACE_SECS`. +# Default value: 1209600 +#gc_grace_secs = 1209600 + +# Can also be specified via environment variable `KNOT_LFS_GC_INTERVAL_SECS`. +# Default value: 21600 +#gc_interval_secs = 21600 + +# Can also be specified via environment variable `KNOT_LFS_MAX_SSH_TRANSFERS`. +# Default value: 16 +#max_ssh_transfers = 16 + +# Can also be specified via environment variable `KNOT_LFS_MAX_HTTP_DOWNLOADS`. +# Default value: 64 +#max_http_downloads = 64 + +[resources] +# Can also be specified via environment variable `KNOT_MAX_THREADS`. +# Default value: 0 +#max_threads = 0 + +# Can also be specified via environment variable `KNOT_MAX_MEMORY_BYTES`. +# Default value: 0 +#max_memory_bytes = 0 + +[homepage] +# Can also be specified via environment variable `KNOT_HOMEPAGE_ENABLED`. +# Default value: true +#enabled = true + +# Can also be specified via environment variable `KNOT_HOMEPAGE_PATH`. +#path = + +[ci] +# Can also be specified via environment variable `KNOT_CI_LOGS_ADDR`. +#logs_addr = + +[messages] +[messages.push] +# Default value: ["{knot} received {refs}."] +#ack = ["{knot} received {refs}."] + +# Default value: ["", "-> Open stinky pull request for this branch:", " {url}", ""] +#pull_request = ["", "-> Open stinky pull request for this branch:", " {url}", ""] + +# Default value: ["pipeline compiled with no diagnostics"] +#pipeline_clean = ["pipeline compiled with no diagnostics"] + +# Default value: ["no pipelines to compile"] +#pipeline_none = ["no pipelines to compile"] + +# Default value: ["-> Browse CI logs in your terminal:", " ssh -t -p {port} {host} {repo} {sha}"] +#ci_logs = ["-> Browse CI logs in your terminal:", " ssh -t -p {port} {host} {repo} {sha}"] + +[messages.fetch] +# Default value: ["Thanks for using {knot}!"] +#motd = ["Thanks for using {knot}!"] + +# Default value: ["Enumerating objects: {count}, done."] +#enumerating = ["Enumerating objects: {count}, done."] + +# Default value: ["Total {count}, done."] +#total = ["Total {count}, done."] + +# Default value: "knot: {error}" +#fatal = "knot: {error}" + +[messages.reject] +# Default value: "refs/cobs/* and refs/hidden/* are reserved and cannot be pushed" +#reserved_refs = "refs/cobs/* and refs/hidden/* are reserved and cannot be pushed" + +# Default value: "existing refs/cobs/* object cannot be modified or deleted over the wire" +#cob_create_only = "existing refs/cobs/* object cannot be modified or deleted over the wire" + +# Default value: "refs/cobs/* stores append-only collaborative objects and cannot be deleted" +#cob_delete = "refs/cobs/* stores append-only collaborative objects and cannot be deleted" + +# Default value: "refs/hidden/* is reserved for server-side fork staging and cannot be pushed" +#hidden_reserved = "refs/hidden/* is reserved for server-side fork staging and cannot be pushed" + +# Default value: "collaborative-object verification failed: {error}" +#cob_verification = "collaborative-object verification failed: {error}" + +# Default value: "reference already exists" +#ref_exists = "reference already exists" + +# Default value: "stale info: old value doesn't match" +#stale_old_value = "stale info: old value doesn't match" + +# Default value: "missing necessary objects" +#missing_objects = "missing necessary objects" + +# Default value: "missing necessary objects for {ref}" +#missing_objects_for = "missing necessary objects for {ref}" + +# Default value: "atomic transaction failed" +#atomic_failed = "atomic transaction failed" + +# Default value: "atomic push aborted" +#atomic_aborted = "atomic push aborted" + +# Default value: "authorization unavailable" +#authorization_unavailable = "authorization unavailable" + +# Default value: "unpacker error" +#unpacker_error = "unpacker error" + +# Default value: "ref snapshot unavailable" +#ref_snapshot_unavailable = "ref snapshot unavailable" + +# Default value: "object migration failed" +#object_migration_failed = "object migration failed" + +[messages.ssh] +# Default value: ["Hi {user}! You're authenticated to {knot} knot.", "This knot serves git over ssh, so there's no shell here. :P", "Clone repo with: git clone {knot}:"] +#greeting = ["Hi {user}! You're authenticated to {knot} knot.", "This knot serves git over ssh, so there's no shell here. :P", "Clone repo with: git clone {knot}:"] + +# Default value: "knot: unsupported command" +#unsupported_command = "knot: unsupported command" + +# Default value: "knot: too many concurrent operations from your address, try again shortly" +#too_many_operations = "knot: too many concurrent operations from your address, try again shortly" + +# Default value: "knot: repository not found" +#repo_not_found = "knot: repository not found" + +# Default value: "knot: repository index is warming, retry shortly" +#index_warming = "knot: repository index is warming, retry shortly" + +# Default value: "knot: LFS isn't enabled on this knot" +#lfs_disabled = "knot: LFS isn't enabled on this knot" + +# Default value: "knot: your ssh key isn't registered to a user authorized to push here. If you offer several keys, make sure the registered one is offered first." +#key_not_registered = "knot: your ssh key isn't registered to a user authorized to push here. If you offer several keys, make sure the registered one is offered first." + +# Default value: "knot: you aren't authorized to push to this repository." +#push_denied = "knot: you aren't authorized to push to this repository." + +# Default value: "knot: server is shutting down" +#shutting_down = "knot: server is shutting down" + +# Default value: "knot: malformed upload-archive request" +#archive_malformed = "knot: malformed upload-archive request" + +# Default value: "knot: upload-archive request timed out" +#archive_timeout = "knot: upload-archive request timed out" + +# Default value: "knot: upload-archive failed" +#archive_failed = "knot: upload-archive failed" + +# Default value: "knot: cannot advertise refs" +#advertise_failed = "knot: cannot advertise refs" + +# Default value: "knot: push exceeds configured size limit" +#push_too_large = "knot: push exceeds configured size limit" + +# Default value: "knot: receive exceeded its time budget" +#receive_deadline = "knot: receive exceeded its time budget" + +# Default value: "knot: malformed pack stream" +#malformed_pack = "knot: malformed pack stream" + +# Default value: "knot: receive read error" +#receive_read_error = "knot: receive read error" + +# Default value: "knot: receive stream ended early" +#receive_ended_early = "knot: receive stream ended early" + +# Default value: "knot: receive-pack failed" +#receive_failed = "knot: receive-pack failed" + +[messages.http] +# Default value: "you aren't authorized to push to this repository" +#push_denied = "you aren't authorized to push to this repository" + +# Default value: "repository not found" +#repo_not_found = "repository not found" + +# Default value: "push exceeds the configured size limit" +#push_too_large = "push exceeds the configured size limit" + +# Default value: "malformed pack stream: {error}" +#malformed_pack = "malformed pack stream: {error}" + +# Default value: "receive stream ended early" +#receive_ended_early = "receive stream ended early" + +[messages.lfs] +# Default value: "invalid LFS oid {value}" +#invalid_oid = "invalid LFS oid {value}" + +# Default value: "oid mismatch, declared {declared}, computed {computed}" +#hash_mismatch = "oid mismatch, declared {declared}, computed {computed}" + +# Default value: "size mismatch, declared {declared}, received {received}" +#size_mismatch = "size mismatch, declared {declared}, received {received}" + +# Default value: "object size {declared} exceeds limit {limit}" +#size_limit_exceeded = "object size {declared} exceeds limit {limit}" + +# Default value: "free space {free} below floor {floor}" +#free_space_denied = "free space {free} below floor {floor}" + +# Default value: "object {oid} not found" +#not_found = "object {oid} not found" + +# Default value: "protocol framing fault: {detail}" +#framing = "protocol framing fault: {detail}" + +# Default value: "too many {what} in one message, limit {limit}" +#too_many = "too many {what} in one message, limit {limit}" + +# Default value: "unknown command {command}" +#unknown_command = "unknown command {command}" + +# Default value: "unsupported version {version}" +#unsupported_version = "unsupported version {version}" + +# Default value: "unsupported hash algorithm {algorithm}" +#unsupported_hash = "unsupported hash algorithm {algorithm}" + +# Default value: "put-object isn't allowed on a download channel" +#put_on_download = "put-object isn't allowed on a download channel" + +# Default value: "verify-object isn't allowed on a download channel" +#verify_on_download = "verify-object isn't allowed on a download channel" + +# Default value: "get-object isn't allowed on an upload channel" +#get_on_upload = "get-object isn't allowed on an upload channel" + +# Default value: "put-object is missing its object body" +#put_no_body = "put-object is missing its object body" diff --git a/knot2/justfile b/knot2/justfile new file mode 100644 --- /dev/null +++ b/knot2/justfile @@ -0,0 +1,148 @@ +set shell := ["bash", "-eu", "-o", "pipefail", "-c"] + +default: + @just --list + +gen-config: + cargo run -p knot-server -- config-template > example.toml + +fmt: + cargo fmt + +fmt-check: + cargo fmt --check + +clippy: + cargo clippy -p 'knot-*' --all-targets -- -D warnings + +test: + cargo test -p 'knot-*' + +fuzz crate='knot-pack' target='pack' time='60': + cd crates/{{crate}}/fuzz && RUSTUP_TOOLCHAIN=nightly cargo fuzz run {{target}} -- -max_total_time={{time}} + +fuzz-ci: (fuzz "knot-pack" "pkt" "30") (fuzz "knot-pack" "pack" "30") (fuzz "knot-pack" "receive_commands" "30") (fuzz "knot-pack" "upload_args" "30") (fuzz "knot-git" "patch" "30") (fuzz "knot-cobs" "cob_change" "30") (fuzz "knot-cobs" "cob_ref" "30") (fuzz "knot-atproto" "pubkey" "30") (fuzz "knot-atproto" "did_document" "30") (fuzz "knot-edge" "spki" "30") (fuzz "knot-edge" "spki_pin" "30") (fuzz "knot-lfs" "transfer" "30") (fuzz "knot-lfs" "batch" "30") (fuzz "knot-lfs" "pointer" "30") + +bench filter='': + cargo bench -p knot-bench --bench pack -- {{filter}} + cargo bench -p knot-bench --bench cob -- {{filter}} + cargo bench -p knot-bench --bench projection -- {{filter}} + +bench-scaling: + cargo bench -p knot-bench --bench coldstart + +bench-gate: + cargo test -p knot-bench --features instrument --test gate + +bench-gate-registry: + cargo test -p knot-bench --features instrument --test registry_gate + +differential: + cargo test -p knot-pack --test differential + +t55xx *tests: + internal_docs/t55xx/run.sh {{tests}} + +ci: fmt-check clippy test gates bench-gate fuzz-ci + +gates: gate-no-subprocess gate-no-sql gate-no-native-git gate-no-string-ids gate-no-unguarded-receive gate-fuzz-targets-enumerated + +gate-no-subprocess: + #!/usr/bin/env bash + set -euo pipefail + hits=$(grep -rn "process::Command" crates/*/src --include="*.rs" | grep -v "/_lex/" || true) + if [ -n "$hits" ]; then + echo "no-subprocess gate failed: server source spawns processes" >&2 + echo "$hits" >&2 + exit 1 + fi + echo "ok: no process spawning in server source" + +gate-no-sql: + #!/usr/bin/env bash + set -euo pipefail + hits=$(grep -inE '^name = "(rusqlite|libsqlite3-sys|sqlx|sqlx-core|sled|fjall|redb)"' ../Cargo.lock || true) + if [ -n "$hits" ]; then + echo "no-sql gate failed: an embedded database is in the dependency tree" >&2 + echo "$hits" >&2 + exit 1 + fi + echo "ok: no embedded database in the dependency tree" + +gate-no-native-git: + #!/usr/bin/env bash + set -euo pipefail + hits=$(grep -inE '^name = "(git2|libgit2-sys|openssl-sys|zlib-ng|zlib-ng-sys)"' ../Cargo.lock || true) + if [ -n "$hits" ]; then + echo "no-native-git gate failed: a native git or TLS shim is in the dependency tree" >&2 + echo "$hits" >&2 + exit 1 + fi + echo "ok: no native git or TLS shim in the dependency tree" + +gate-no-string-ids: + #!/usr/bin/env bash + set -euo pipefail + hits=$(grep -nE 'pub fn .*(-> *String|: *String\b)' crates/knot-types/src/ids.rs | grep -v 'fn to_hex' || true) + if [ -n "$hits" ]; then + echo "no-string-ids gate failed: a String-typed id crosses the knot-types boundary" >&2 + echo "$hits" >&2 + exit 1 + fi + echo "ok: no String-typed id crosses the knot-types boundary" + +gate-no-unguarded-receive: + #!/usr/bin/env bash + set -euo pipefail + hits=$(grep -rn 'receive_pack(\|receive_pack_with_limits(' crates/*/src --include="*.rs" | grep -v 'pub fn ' || true) + if [ -n "$hits" ]; then + echo "no-unguarded-receive gate failed: server source calls the unguarded receive path, use receive_pack_guarded" >&2 + echo "$hits" >&2 + exit 1 + fi + echo "ok: the unguarded receive path is reached only from tests" + +gate-fuzz-targets-enumerated: + #!/usr/bin/env bash + set -euo pipefail + disk=$(mktemp) + recipe=$(mktemp) + triplet=$(mktemp) + trap 'rm -f "$disk" "$recipe" "$triplet"' EXIT + find crates -path '*/fuzz/fuzz_targets/*.rs' -not -path '*/target/*' | sed -E 's#crates/([^/]+)/fuzz/fuzz_targets/(.+)\.rs#\1 \2#' | sort -u > "$disk" + just --show fuzz-ci | grep -oE '\(fuzz "[^"]+" "[^"]+" "[^"]+"' | sed -E 's#\(fuzz "([^"]+)" "([^"]+)" "([^"]+)"#\1 \2 \3#' | sort -u > "$triplet" + cut -d' ' -f1,2 "$triplet" > "$recipe" + while read -r crate target secs; do + if ! [[ "$secs" =~ ^[1-9][0-9]*$ ]]; then + echo "fuzz-targets-enumerated gate failed: target '$crate $target' runs for '$secs', not a positive number of seconds" >&2 + exit 1 + fi + done < "$triplet" + missing=$(comm -23 "$disk" "$recipe" || true) + extra=$(comm -13 "$disk" "$recipe" || true) + if [ -n "$missing" ] || [ -n "$extra" ]; then + echo "fuzz-targets-enumerated gate failed: the fuzz-ci recipe and the targets on disk disagree" >&2 + if [ -n "$missing" ]; then + echo "on disk but absent from fuzz-ci:" >&2 + echo "$missing" >&2 + fi + if [ -n "$extra" ]; then + echo "in fuzz-ci but no matching target on disk:" >&2 + echo "$extra" >&2 + fi + exit 1 + fi + while read -r crate target; do + manifest="crates/$crate/fuzz/Cargo.toml" + if ! grep -qF "name = \"$target\"" "$manifest" || ! grep -qF "path = \"fuzz_targets/$target.rs\"" "$manifest"; then + echo "fuzz-targets-enumerated gate failed: $manifest has no [[bin]] declaring target '$target'" >&2 + exit 1 + fi + entry=$(grep -oE 'knot_[a-z0-9_]+::fuzz::[a-z0-9_]+' "crates/$crate/fuzz/fuzz_targets/$target.rs" | head -1 | sed -E 's#.*::fuzz::##' || true) + smoke="crates/$crate/tests/fuzz_smoke.rs" + if [ -z "$entry" ] || ! grep -qE "fuzz::${entry}\(" "$smoke"; then + echo "fuzz-targets-enumerated gate failed: target '$target' entry point $(echo "$crate" | tr - _)::fuzz::$entry has no smoke-test coverage in $smoke" >&2 + exit 1 + fi + done < "$disk" + echo "ok: every fuzz target is enumerated in fuzz-ci, declared in its fuzz manifest, and smoke-tested" diff --git a/lexicons/repo/create.json b/lexicons/repo/create.json --- a/lexicons/repo/create.json +++ b/lexicons/repo/create.json @@ -16,6 +16,7 @@ "properties": { "rkey": { "type": "string", + "format": "record-key", "description": "Rkey of the repository record" }, "name": { @@ -46,6 +47,10 @@ "repoDid": { "type": "string", "format": "did" + }, + "key": { + "type": "string", + "description": "Multibase-encoded public signing key the knot holds for this repository" } } } diff --git a/lexicons/repo/delete.json b/lexicons/repo/delete.json --- a/lexicons/repo/delete.json +++ b/lexicons/repo/delete.json @@ -22,7 +22,12 @@ }, "rkey": { "type": "string", + "format": "record-key", "description": "Rkey of the repository record" + }, + "force": { + "type": "boolean", + "description": "Admin-only. Delete even though the repository record still exists on the owner's PDS." } } } diff --git a/bobbin/crates/xrpc/Cargo.toml b/bobbin/crates/xrpc/Cargo.toml --- a/bobbin/crates/xrpc/Cargo.toml +++ b/bobbin/crates/xrpc/Cargo.toml @@ -23,7 +23,7 @@ serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } -tower-http = { workspace = true } +tower-http = { workspace = true, features = ["trace"] } tracing = { workspace = true } url = { workspace = true } diff --git a/knot2/crates/knot-acl/Cargo.toml b/knot2/crates/knot-acl/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-acl/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "knot-acl" +version = "2.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +knot-index = { workspace = true } + +[dev-dependencies] +knot-cob = { workspace = true } +knot-cobs = { workspace = true } +knot-git = { workspace = true } +knot-runtime = { workspace = true } +tempfile = { workspace = true } diff --git a/knot2/crates/knot-atproto/Cargo.toml b/knot2/crates/knot-atproto/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-atproto/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "knot-atproto" +version = "2.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +knot-runtime = { workspace = true } +knot-cache = { workspace = true } +futures = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +serde_ipld_dagcbor = { workspace = true } +sha2 = { workspace = true } +base32 = { workspace = true } +bs58 = { workspace = true } +thiserror = { workspace = true } +url = { workspace = true } +base64 = { workspace = true } +bytes = { workspace = true } +http = { workspace = true } + +[dev-dependencies] +knot-lexicons = { workspace = true } +tokio = { workspace = true } +k256 = { workspace = true } +proptest = { workspace = true } diff --git a/knot2/crates/knot-bench/Cargo.toml b/knot2/crates/knot-bench/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-bench/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "knot-bench" +version = "2.0.0" +edition = { workspace = true } +rust-version = { workspace = true } +license = { workspace = true } +publish = false + +[features] +instrument = ["knot-git/instrument", "knot-cob/instrument"] + +[dependencies] +knot-types = { workspace = true } +knot-git = { workspace = true } +knot-pack = { workspace = true } +knot-cob = { workspace = true } +knot-cobs = { workspace = true } +knot-index = { workspace = true } +knot-runtime = { workspace = true } +gix = { workspace = true } +tempfile = { workspace = true } + +[dev-dependencies] +divan = { workspace = true } + +[[bench]] +name = "pack" +harness = false + +[[bench]] +name = "cob" +harness = false + +[[bench]] +name = "coldstart" +harness = false + +[[bench]] +name = "advert" +harness = false + +[[bench]] +name = "projection" +harness = false diff --git a/knot2/crates/knot-cache/Cargo.toml b/knot2/crates/knot-cache/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-cache/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "knot-cache" +version = "2.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-runtime = { workspace = true } +knot-types = { workspace = true } +moka = { workspace = true } diff --git a/knot2/crates/knot-cob/Cargo.toml b/knot2/crates/knot-cob/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-cob/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "knot-cob" +version = "2.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[features] +instrument = [] + +[dependencies] +knot-types = { workspace = true } +knot-runtime = { workspace = true } +knot-git = { workspace = true } +knot-resource = { workspace = true } +tracing = { workspace = true } +gix = { workspace = true } +gix-hash = { workspace = true } +k256 = { workspace = true } +serde = { workspace = true } +serde_ipld_dagcbor = { workspace = true } +thiserror = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } +proptest = { workspace = true } diff --git a/knot2/crates/knot-cobs/Cargo.toml b/knot2/crates/knot-cobs/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-cobs/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "knot-cobs" +version = "2.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +knot-cob = { workspace = true } +knot-runtime = { workspace = true } +serde = { workspace = true } +thiserror = { workspace = true } + +[dev-dependencies] +knot-git = { workspace = true } +knot-runtime = { workspace = true } +tempfile = { workspace = true } +gix = { workspace = true } +serde_ipld_dagcbor = { workspace = true } +proptest = { workspace = true } diff --git a/knot2/crates/knot-config/Cargo.toml b/knot2/crates/knot-config/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-config/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "knot-config" +version = "2.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +knot-runtime = { workspace = true } +knot-messages = { workspace = true } +confique = { workspace = true } +thiserror = { workspace = true } +base64 = { workspace = true } +tempfile = { workspace = true } +url = { workspace = true } diff --git a/knot2/crates/knot-edge/Cargo.toml b/knot2/crates/knot-edge/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-edge/Cargo.toml @@ -0,0 +1,41 @@ +[package] +name = "knot-edge" +version = "2.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +axum = { workspace = true } +hyper = { workspace = true } +hyper-util = { workspace = true } +tower = { workspace = true, features = ["limit", "load-shed"] } +tower-http = { workspace = true, features = ["compression-zstd", "compression-br", "compression-gzip", "timeout", "map-request-body"] } +tower_governor = { workspace = true } +governor = { workspace = true } +http-body = { workspace = true } +tokio = { workspace = true } +tokio-util = { workspace = true } +tokio-rustls = { workspace = true } +rustls = { workspace = true } +rustls-pemfile = { workspace = true } +rustls-acme = { workspace = true } +x509-parser = { workspace = true } +sha2 = { workspace = true } +base64 = { workspace = true } +quinn = { workspace = true } +h3 = { workspace = true } +h3-quinn = { workspace = true } +arc-swap = { workspace = true } +bytes = { workspace = true } +http = { workspace = true } +futures = { workspace = true } +async-trait = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } +rcgen = { workspace = true } +tempfile = { workspace = true } diff --git a/knot2/crates/knot-events/Cargo.toml b/knot2/crates/knot-events/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-events/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "knot-events" +version = "2.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +knot-runtime = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true } diff --git a/knot2/crates/knot-fixtures/Cargo.toml b/knot2/crates/knot-fixtures/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-fixtures/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "knot-fixtures" +version = "2.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false + +[dev-dependencies] +tempfile = { workspace = true } diff --git a/knot2/crates/knot-git/Cargo.toml b/knot2/crates/knot-git/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-git/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "knot-git" +version = "2.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[features] +instrument = [] + +[dependencies] +knot-types = { workspace = true } +knot-resource = { workspace = true } +gix = { workspace = true } +gix-archive = { workspace = true } +gix-bitmap = "0.3.2" +gix-pack = { workspace = true } +gix-hash = { workspace = true } +flate2 = { workspace = true } +knot-cache = { workspace = true } +scc = { workspace = true } +thiserror = { workspace = true } +base64 = { workspace = true } +walkdir = { workspace = true } + +[dev-dependencies] +knot-fixtures = { workspace = true } +tempfile = { workspace = true } +proptest = { workspace = true } diff --git a/knot2/crates/knot-index/Cargo.toml b/knot2/crates/knot-index/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-index/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "knot-index" +version = "2.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +knot-git = { workspace = true } +knot-cob = { workspace = true } +knot-cobs = { workspace = true } +knot-cache = { workspace = true } +scc = { workspace = true } +lasso = { workspace = true } +thiserror = { workspace = true } + +[dev-dependencies] +knot-runtime = { workspace = true } +tempfile = { workspace = true } +proptest = { workspace = true } +serde = { workspace = true } diff --git a/knot2/crates/knot-langs/Cargo.toml b/knot2/crates/knot-langs/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-langs/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "knot-langs" +version = "2.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +knot-git = { workspace = true } +gengo-language = { workspace = true } +regex = { workspace = true } diff --git a/knot2/crates/knot-lexicons/Cargo.toml b/knot2/crates/knot-lexicons/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-lexicons/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "knot-lexicons" +version = "2.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +jacquard-common = { workspace = true } +jacquard-derive = { workspace = true } +jacquard-lexicon = { workspace = true, default-features = false } +bytes = { workspace = true } +cid = { workspace = true } +miette = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +url = { workspace = true } + +[build-dependencies] +anyhow = { workspace = true } +jacquard-lexicon = { workspace = true, features = ["codegen"] } +walkdir = { workspace = true } + +[features] +default = ["sh_tangled"] +com_atproto = [] +sh_tangled = ["com_atproto"] +streaming = [] diff --git a/knot2/crates/knot-lexicons/build.rs b/knot2/crates/knot-lexicons/build.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-lexicons/build.rs @@ -0,0 +1,86 @@ +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use jacquard_lexicon::codegen::{CodeGenerator, CodegenMode}; +use jacquard_lexicon::corpus::LexiconCorpus; +use walkdir::WalkDir; + +const LEXICONS_SUBDIR: &str = "lexicons"; +const STAGED_SUBDIR: &str = "lexicons-staged"; +const GENERATED_SUBDIR: &str = "src/_lex"; +const TEMP_SEGMENT: &str = "temp"; + +fn main() -> Result<()> { + let manifest_dir = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR")?); + let knot_root = manifest_dir + .parent() + .and_then(Path::parent) + .context("resolve knot root from manifest dir")? + .to_path_buf(); + let workspace_root = knot_root + .parent() + .context("resolve workspace root from knot root")? + .to_path_buf(); + + let lexicons_dir = std::env::var("KNOT_LEXICONS_DIR") + .map(PathBuf::from) + .unwrap_or_else(|_| workspace_root.join(LEXICONS_SUBDIR)); + let vendored_dir = knot_root.join(LEXICONS_SUBDIR); + + println!("cargo:rerun-if-env-changed=KNOT_LEXICONS_DIR"); + println!("cargo:rerun-if-changed={}", lexicons_dir.display()); + println!("cargo:rerun-if-changed={}", vendored_dir.display()); + println!("cargo:rerun-if-changed=build.rs"); + + let out_dir = PathBuf::from(std::env::var("OUT_DIR")?); + let staged = out_dir.join(STAGED_SUBDIR); + if staged.exists() { + std::fs::remove_dir_all(&staged).context("clean staged lexicons")?; + } + stage_lexicons(&lexicons_dir, &staged)?; + stage_lexicons(&vendored_dir, &staged)?; + + let corpus = LexiconCorpus::load_from_dir(&staged) + .map_err(|e| anyhow::anyhow!("load lexicon corpus: {e:?}"))?; + + let generated = manifest_dir.join(GENERATED_SUBDIR); + if generated.exists() { + std::fs::remove_dir_all(&generated).context("clean generated dir")?; + } + std::fs::create_dir_all(&generated).context("create generated dir")?; + + let codegen = CodeGenerator::with_mode(&corpus, "crate", CodegenMode::Pretty); + codegen + .write_to_disk(&generated) + .map_err(|e| anyhow::anyhow!("write generated code: {e:?}"))?; + + Ok(()) +} + +fn stage_lexicons(src: &Path, dst: &Path) -> Result<()> { + WalkDir::new(src) + .into_iter() + .filter_map(std::result::Result::ok) + .filter(|entry| entry.file_type().is_file()) + .filter(|entry| { + entry + .path() + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("json")) + }) + .filter(|entry| { + !entry + .path() + .components() + .any(|component| component.as_os_str() == TEMP_SEGMENT) + }) + .try_for_each(|entry| -> Result<()> { + let rel = entry.path().strip_prefix(src).context("strip src prefix")?; + let target = dst.join(rel); + if let Some(parent) = target.parent() { + std::fs::create_dir_all(parent).context("create staged parent")?; + } + std::fs::copy(entry.path(), &target).context("copy lexicon file")?; + Ok(()) + }) +} diff --git a/knot2/crates/knot-lfs/Cargo.toml b/knot2/crates/knot-lfs/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-lfs/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "knot-lfs" +version = "2.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +knot-messages = { workspace = true } +knot-git = { workspace = true } +knot-runtime = { workspace = true } +knot-resource = { workspace = true } +gix-packetline = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } +thiserror = { workspace = true } +tempfile = { workspace = true } +tracing = { workspace = true } +url = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } diff --git a/knot2/crates/knot-maintenance/Cargo.toml b/knot2/crates/knot-maintenance/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-maintenance/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "knot-maintenance" +version = "2.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +knot-resource = { workspace = true } +knot-config = { workspace = true } +knot-git = { workspace = true } +knot-lfs = { workspace = true } +knot-pack = { workspace = true } +knot-runtime = { workspace = true } +tracing = { workspace = true } +gix = { workspace = true } +gix-pack = { workspace = true } +gix-hash = { workspace = true } +walkdir = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } + +[dev-dependencies] +knot-fixtures = { workspace = true } +tempfile = { workspace = true } +sha2 = { workspace = true } diff --git a/knot2/crates/knot-messages/Cargo.toml b/knot2/crates/knot-messages/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-messages/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "knot-messages" +version = "2.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +confique = { workspace = true } +thiserror = { workspace = true } diff --git a/knot2/crates/knot-migrate/Cargo.toml b/knot2/crates/knot-migrate/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-migrate/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "knot-migrate" +version = "2.0.0" +edition = { workspace = true } +rust-version = { workspace = true } +license = { workspace = true } +publish = false + +[[bin]] +name = "knot-migrate" +path = "src/main.rs" + +[dependencies] +knot-types = { workspace = true } +knot-runtime = { workspace = true } +knot-git = { workspace = true } +knot-cob = { workspace = true } +knot-cobs = { workspace = true } +knot-config = { workspace = true } +knot-secrets = { workspace = true } +rusqlite = { version = "0.38", features = ["bundled"] } +rustix = { workspace = true, features = ["fs"] } +ssh-key = { version = "0.7.0-rc.10", default-features = false, features = ["std", "ed25519", "ecdsa"] } +chrono = { workspace = true } +walkdir = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +url = { workspace = true } +base64 = { workspace = true } +thiserror = { workspace = true } +zeroize = { workspace = true } + +[dev-dependencies] +knot-index = { workspace = true } +tempfile = { workspace = true } diff --git a/knot2/crates/knot-pack/Cargo.toml b/knot2/crates/knot-pack/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/Cargo.toml @@ -0,0 +1,53 @@ +[package] +name = "knot-pack" +version = "2.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +knot-git = { workspace = true } +knot-index = { workspace = true } +knot-runtime = { workspace = true } +knot-resource = { workspace = true } +knot-cache = { workspace = true } +knot-edge = { workspace = true } +knot-cob = { workspace = true } +knot-cobs = { workspace = true } +knot-messages = { workspace = true } +url = { workspace = true } +gix = { workspace = true } +gix-pack = { workspace = true } +gix-hash = { workspace = true } +scc = { workspace = true } +gix-packetline = { workspace = true } +flate2 = { workspace = true } +axum = { workspace = true } +tokio = { workspace = true } +tokio-stream = { workspace = true } +thiserror = { workspace = true } +walkdir = { workspace = true } +tempfile = { workspace = true } + +[dev-dependencies] +knot-fixtures = { workspace = true } +tempfile = { workspace = true } +flate2 = { workspace = true } +gix-hash = { workspace = true } +tracing-subscriber = { workspace = true } +tikv-jemallocator = { workspace = true } +knot-resource = { workspace = true } +proptest = { workspace = true } +tower = { version = "0.5", features = ["util"] } +http-body-util = "0.1" +knot-bench = { workspace = true } +knot-maintenance = { workspace = true } +quinn = { workspace = true } +h3 = { workspace = true } +h3-quinn = { workspace = true } +rustls = { workspace = true } +rcgen = { workspace = true } +tokio-util = { workspace = true } +http = { workspace = true } +bytes = { workspace = true } diff --git a/knot2/crates/knot-postreceive/Cargo.toml b/knot2/crates/knot-postreceive/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-postreceive/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "knot-postreceive" +version = "2.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +knot-git = { workspace = true } +knot-langs = { workspace = true } +knot-events = { workspace = true } +knot-workflow = { workspace = true } +knot-messages = { workspace = true } +tracing = { workspace = true } +url = { workspace = true } + +[dev-dependencies] +knot-runtime = { workspace = true } +knot-fixtures = { workspace = true } +tempfile = { workspace = true } +serde_json = { workspace = true } diff --git a/knot2/crates/knot-receive/Cargo.toml b/knot2/crates/knot-receive/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-receive/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "knot-receive" +version = "2.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +knot-git = { workspace = true } +knot-pack = { workspace = true } +knot-resource = { workspace = true } +knot-cob = { workspace = true } +knot-index = { workspace = true } +knot-atproto = { workspace = true } +knot-events = { workspace = true } +knot-maintenance = { workspace = true } +knot-postreceive = { workspace = true } +knot-messages = { workspace = true } +knot-runtime = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } diff --git a/knot2/crates/knot-resource/Cargo.toml b/knot2/crates/knot-resource/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-resource/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "knot-resource" +version = "2.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +rustix = { workspace = true, features = ["fs"] } +tokio = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["rt", "macros"] } +tempfile = { workspace = true } diff --git a/knot2/crates/knot-runtime/Cargo.toml b/knot2/crates/knot-runtime/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-runtime/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "knot-runtime" +version = "2.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +serde = { workspace = true } +reqwest = { workspace = true } +http = { workspace = true } +url = { workspace = true } +bytes = { workspace = true } +getrandom = { workspace = true } +k256 = { workspace = true } +thiserror = { workspace = true } +futures = { workspace = true } +tokio = { workspace = true } +hickory-resolver = { workspace = true } + +[dev-dependencies] +futures = { workspace = true } +tokio = { workspace = true } diff --git a/knot2/crates/knot-secrets/Cargo.toml b/knot2/crates/knot-secrets/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-secrets/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "knot-secrets" +version = "2.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-resource = { workspace = true } +knot-types = { workspace = true } +knot-runtime = { workspace = true } +k256 = { workspace = true } +aes-gcm = { workspace = true } +hkdf = { workspace = true } +sha2 = { workspace = true } +base64 = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +zeroize = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } diff --git a/knot2/crates/knot-server/Cargo.toml b/knot2/crates/knot-server/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-server/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "knot-server" +version = "2.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-config = { workspace = true } +knot-types = { workspace = true } +knot-git = { workspace = true } +knot-lfs = { workspace = true } +knot-pack = { workspace = true } +knot-resource = { workspace = true } +knot-cache = { workspace = true } +knot-index = { workspace = true } +knot-atproto = { workspace = true } +knot-runtime = { workspace = true } +knot-ssh = { workspace = true } +knot-edge = { workspace = true } +knot-secrets = { workspace = true } +knot-xrpc = { workspace = true } +tikv-jemallocator = { workspace = true } +tikv-jemalloc-ctl = { workspace = true } +knot-events = { workspace = true } +knot-maintenance = { workspace = true } +knot-postreceive = { workspace = true } +knot-messages = { workspace = true } +axum = { workspace = true } +tower-http = { workspace = true, features = ["fs"] } +tokio = { workspace = true } +tokio-util = { workspace = true } +anyhow = { workspace = true } +base64 = { workspace = true } +zeroize = { workspace = true } +rustix = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } + +[dev-dependencies] +confique = { workspace = true } +http = { workspace = true } +walkdir = { workspace = true } +tempfile = { workspace = true } diff --git a/knot2/crates/knot-sim/Cargo.toml b/knot2/crates/knot-sim/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-sim/Cargo.toml @@ -0,0 +1,52 @@ +[package] +name = "knot-sim" +version = "2.0.0" +edition = { workspace = true } +rust-version = { workspace = true } +license = { workspace = true } +publish = false + +[[bin]] +name = "knot-sim" +path = "src/main.rs" + +[dependencies] +knot-types = { workspace = true } +knot-config = { workspace = true } +knot-runtime = { workspace = true } +knot-git = { workspace = true } +knot-index = { workspace = true } +knot-pack = { workspace = true } +knot-resource = { workspace = true } +knot-atproto = { workspace = true } +knot-secrets = { workspace = true } +knot-xrpc = { workspace = true } +knot-events = { workspace = true } +knot-cob = { workspace = true } +knot-cobs = { workspace = true } +knot-maintenance = { workspace = true } +knot-messages = { workspace = true } +axum = { workspace = true } +tower = { workspace = true } +http = { workspace = true } +bytes = { workspace = true } +tokio = { workspace = true } +url = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +base64 = { workspace = true } +futures = { workspace = true } +tempfile = { workspace = true } + +[dev-dependencies] +knot-fixtures = { workspace = true } +knot-ssh = { workspace = true } +knot-edge = { workspace = true } +knot-lfs = { workspace = true } +sha2 = { workspace = true } +quinn = { workspace = true } +h3 = { workspace = true } +h3-quinn = { workspace = true } +rustls = { workspace = true } +rcgen = { workspace = true } +tokio-util = { workspace = true } diff --git a/knot2/crates/knot-ssh/Cargo.toml b/knot2/crates/knot-ssh/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-ssh/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "knot-ssh" +version = "2.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +knot-runtime = { workspace = true } +knot-resource = { workspace = true } +knot-git = { workspace = true } +knot-lfs = { workspace = true } +knot-pack = { workspace = true } +knot-index = { workspace = true } +knot-acl = { workspace = true } +knot-atproto = { workspace = true } +knot-cob = { workspace = true } +knot-cobs = { workspace = true } +knot-events = { workspace = true } +knot-maintenance = { workspace = true } +knot-postreceive = { workspace = true } +knot-receive = { workspace = true } +knot-messages = { workspace = true } +tracing = { workspace = true } +russh = "0.61" +tokio = { workspace = true } +tokio-util = { workspace = true, features = ["io-util"] } +futures = { workspace = true } +thiserror = { workspace = true } + +[dev-dependencies] +knot-fixtures = { workspace = true } +tempfile = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } +http = { workspace = true } +bytes = { workspace = true } +url = { workspace = true } +russh = "0.61" +tikv-jemallocator = { workspace = true } +tikv-jemalloc-ctl = { workspace = true } diff --git a/knot2/crates/knot-types/Cargo.toml b/knot2/crates/knot-types/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-types/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "knot-types" +version = "2.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +jacquard-common = { workspace = true } +gix-hash = { workspace = true } +http = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +url = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } diff --git a/knot2/crates/knot-workflow/Cargo.toml b/knot2/crates/knot-workflow/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-workflow/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "knot-workflow" +version = "2.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +serde = { workspace = true } +serde_norway = { workspace = true } +globset = { workspace = true } + +[dev-dependencies] +serde_json = { workspace = true } diff --git a/knot2/crates/knot-xrpc/Cargo.toml b/knot2/crates/knot-xrpc/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-xrpc/Cargo.toml @@ -0,0 +1,54 @@ +[package] +name = "knot-xrpc" +version = "2.0.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +knot-types = { workspace = true } +knot-git = { workspace = true } +knot-lfs = { workspace = true } +knot-cache = { workspace = true } +knot-pack = { workspace = true } +knot-resource = { workspace = true } +knot-index = { workspace = true } +knot-acl = { workspace = true } +knot-atproto = { workspace = true } +knot-cob = { workspace = true } +knot-cobs = { workspace = true } +knot-secrets = { workspace = true } +knot-runtime = { workspace = true } +knot-events = { workspace = true } +knot-langs = { workspace = true } +knot-postreceive = { workspace = true } +knot-messages = { workspace = true } +knot-receive = { workspace = true } +knot-maintenance = { workspace = true } +tracing = { workspace = true } +axum = { workspace = true, features = ["ws"] } +tower = { workspace = true } +tower-http = { workspace = true, features = ["fs"] } +http-body = { workspace = true } +tokio-util = { workspace = true, features = ["io-util"] } +tokio = { workspace = true } +futures = { workspace = true } +tokio-stream = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +http = { workspace = true } +httpdate = { workspace = true } +base64 = { workspace = true } +chrono = { workspace = true } +tempfile = { workspace = true } +sha2 = { workspace = true } +url = { workspace = true } + +[dev-dependencies] +knot-fixtures = { workspace = true } +knot-config = { workspace = true } +bytes = { workspace = true } +k256 = { workspace = true } +sha2 = { workspace = true } +tokio-tungstenite = "0.29" diff --git a/knot2/lexicons/knot/ban.json b/knot2/lexicons/knot/ban.json new file mode 100644 --- /dev/null +++ b/knot2/lexicons/knot/ban.json @@ -0,0 +1,26 @@ +{ + "lexicon": 1, + "id": "sh.tangled.knot.ban", + "defs": { + "main": { + "type": "procedure", + "description": "Block an account from creating repositories or pushing on this knot", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "subject" + ], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "DID of the account to block" + } + } + } + } + } + } +} diff --git a/knot2/lexicons/knot/unban.json b/knot2/lexicons/knot/unban.json new file mode 100644 --- /dev/null +++ b/knot2/lexicons/knot/unban.json @@ -0,0 +1,26 @@ +{ + "lexicon": 1, + "id": "sh.tangled.knot.unban", + "defs": { + "main": { + "type": "procedure", + "description": "Lift a block previously placed on an account on this knot", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "subject" + ], + "properties": { + "subject": { + "type": "string", + "format": "did", + "description": "DID of the account to unblock" + } + } + } + } + } + } +} diff --git a/knot2/lexicons/repo/push.json b/knot2/lexicons/repo/push.json new file mode 100644 --- /dev/null +++ b/knot2/lexicons/repo/push.json @@ -0,0 +1,10 @@ +{ + "lexicon": 1, + "id": "sh.tangled.repo.push", + "defs": { + "main": { + "type": "procedure", + "description": "Authorizes a git push to a knot over HTTP. A client mints a service-auth token with this method with the knot as its audience, then gives it on git-receive-pack & LFS object requests!" + } + } +} diff --git a/knot2/lexicons/repo/rename.json b/knot2/lexicons/repo/rename.json new file mode 100644 --- /dev/null +++ b/knot2/lexicons/repo/rename.json @@ -0,0 +1,38 @@ +{ + "lexicon": 1, + "id": "sh.tangled.repo.rename", + "defs": { + "main": { + "type": "procedure", + "description": "Rename a repository on this knot. The given rkey becomes the canonical record key. Prior rkeys keep resolving as aliases until the repository is deleted.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "repo", + "rkey", + "name" + ], + "properties": { + "repo": { + "type": "string", + "format": "did", + "description": "DID of the repository to rename" + }, + "rkey": { + "type": "string", + "format": "record-key", + "description": "Rkey of the sh.tangled.repo record written for the new name" + }, + "name": { + "type": "string", + "maxLength": 100, + "description": "New display name of the repository" + } + } + } + } + } + } +} diff --git a/knot2/lexicons/repo/reserveKey.json b/knot2/lexicons/repo/reserveKey.json new file mode 100644 --- /dev/null +++ b/knot2/lexicons/repo/reserveKey.json @@ -0,0 +1,46 @@ +{ + "lexicon": 1, + "id": "sh.tangled.repo.reserveKey", + "defs": { + "main": { + "type": "procedure", + "description": "Reserve a signing key for a bring-your-own did:web repository identity. The knot seals the key and returns its multibase encoding so the owner can publish it in their did:web document before calling sh.tangled.repo.create.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "repoDid" + ], + "properties": { + "repoDid": { + "type": "string", + "format": "did", + "description": "The did:web identity the repository will use." + } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": [ + "repoDid", + "key" + ], + "properties": { + "repoDid": { + "type": "string", + "format": "did" + }, + "key": { + "type": "string", + "description": "Multibase-encoded public key to publish in the did:web document." + } + } + } + } + } + } +} diff --git a/knot2/third_party/gix-pack/.cargo_vcs_info.json b/knot2/third_party/gix-pack/.cargo_vcs_info.json new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "10c58bb56597d9335611da121aac21f9b09b6e5b" + }, + "path_in_vcs": "gix-pack" +} \ No newline at end of file diff --git a/knot2/third_party/gix-pack/Cargo.toml b/knot2/third_party/gix-pack/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/Cargo.toml @@ -0,0 +1,235 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2024" +rust-version = "1.85" +name = "gix-pack" +version = "0.71.0" +authors = ["Sebastian Thiel "] +build = false +include = [ + "/src/**/*", + "/LICENSE-*", +] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Implements git packs and related data structures" +readme = false +license = "MIT OR Apache-2.0" +repository = "https://github.com/GitoxideLabs/gitoxide" +resolver = "2" + +[package.metadata.docs.rs] +all-features = true +features = [ + "sha1", + "sha256", + "document-features", + "pack-cache-lru-dynamic", + "object-cache-dynamic", + "serde", +] + +[features] +default = [ + "generate", + "streaming-input", +] +generate = [ + "dep:gix-traverse", + "dep:gix-diff", + "dep:parking_lot", + "dep:gix-hashtable", +] +object-cache-dynamic = [ + "dep:clru", + "dep:gix-hashtable", +] +pack-cache-lru-dynamic = ["dep:clru"] +pack-cache-lru-static = ["dep:uluru"] +parallel = ["gix-features/parallel"] +serde = [ + "dep:serde", + "gix-object/serde", +] +sha1 = ["gix-hash/sha1"] +sha256 = ["gix-hash/sha256"] +streaming-input = [ + "dep:parking_lot", + "dep:gix-tempfile", +] +wasm = ["gix-diff?/wasm"] + +[lib] +name = "gix_pack" +path = "src/lib.rs" +doctest = false + +[dependencies.clru] +version = "0.6.1" +optional = true + +[dependencies.document-features] +version = "0.2.0" +optional = true + +[dependencies.gix-chunk] +version = "^0.7.2" + +[dependencies.gix-diff] +version = "^0.64.0" +optional = true +default-features = false + +[dependencies.gix-error] +version = "^0.2.4" + +[dependencies.gix-features] +version = "^0.48.1" +features = [ + "crc32", + "progress", + "zlib", +] + +[dependencies.gix-hash] +version = "^0.25.1" + +[dependencies.gix-hashtable] +version = "^0.15.1" +optional = true + +[dependencies.gix-object] +version = "^0.61.0" + +[dependencies.gix-path] +version = "^0.12.1" + +[dependencies.gix-traverse] +version = "^0.58.0" +optional = true + +[dependencies.parking_lot] +version = "0.12.4" +optional = true +default-features = false + +[dependencies.serde] +version = "1.0.114" +features = ["derive"] +optional = true +default-features = false + +[dependencies.smallvec] +version = "1.15.1" + +[dependencies.thiserror] +version = "2.0.18" + +[dependencies.uluru] +version = "3.0.0" +optional = true + +[dev-dependencies.bstr] +version = "1.12.0" +features = ["std"] +default-features = false + +[dev-dependencies.maplit] +version = "1.0.2" + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies.gix-tempfile] +version = "^23.0.0" +optional = true +default-features = false + +[lints.clippy] +bool_to_int_with_if = "allow" +borrow_as_ptr = "allow" +cast_lossless = "allow" +cast_possible_truncation = "allow" +cast_possible_wrap = "allow" +cast_precision_loss = "allow" +cast_sign_loss = "allow" +checked_conversions = "allow" +copy_iterator = "allow" +default_trait_access = "allow" +doc_markdown = "allow" +empty_docs = "allow" +enum_glob_use = "allow" +explicit_deref_methods = "allow" +explicit_into_iter_loop = "allow" +explicit_iter_loop = "allow" +filter_map_next = "allow" +fn_params_excessive_bools = "allow" +from_iter_instead_of_collect = "allow" +if_not_else = "allow" +ignored_unit_patterns = "allow" +implicit_clone = "allow" +inconsistent_struct_constructor = "allow" +inefficient_to_string = "allow" +inline_always = "allow" +items_after_statements = "allow" +iter_not_returning_iterator = "allow" +iter_without_into_iter = "allow" +large_enum_variant = "allow" +large_stack_arrays = "allow" +manual_assert = "allow" +manual_is_variant_and = "allow" +manual_let_else = "allow" +manual_string_new = "allow" +many_single_char_names = "allow" +match_bool = "allow" +match_same_arms = "allow" +match_wild_err_arm = "allow" +match_wildcard_for_single_variants = "allow" +missing_errors_doc = "allow" +missing_panics_doc = "allow" +module_name_repetitions = "allow" +must_use_candidate = "allow" +mut_mut = "allow" +naive_bytecount = "allow" +needless_continue = "allow" +needless_for_each = "allow" +needless_pass_by_value = "allow" +needless_raw_string_hashes = "allow" +no_effect_underscore_binding = "allow" +option_option = "allow" +range_plus_one = "allow" +redundant_else = "allow" +result_large_err = "allow" +return_self_not_must_use = "allow" +should_panic_without_expect = "allow" +similar_names = "allow" +single_match_else = "allow" +stable_sort_primitive = "allow" +struct_excessive_bools = "allow" +struct_field_names = "allow" +too_long_first_doc_paragraph = "allow" +too_many_lines = "allow" +transmute_ptr_to_ptr = "allow" +trivially_copy_pass_by_ref = "allow" +unnecessary_join = "allow" +unnecessary_wraps = "allow" +unreadable_literal = "allow" +unused_self = "allow" +used_underscore_binding = "allow" +wildcard_imports = "allow" + +[lints.clippy.pedantic] +level = "warn" +priority = -1 + +[lints.rust] diff --git a/knot2/third_party/gix-pack/LICENSE-APACHE b/knot2/third_party/gix-pack/LICENSE-APACHE new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/LICENSE-APACHE @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/knot2/third_party/gix-pack/LICENSE-MIT b/knot2/third_party/gix-pack/LICENSE-MIT new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/LICENSE-MIT @@ -0,0 +1,17 @@ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/bobbin/crates/edge-index/src/state_index.rs b/bobbin/crates/edge-index/src/state_index.rs --- a/bobbin/crates/edge-index/src/state_index.rs +++ b/bobbin/crates/edge-index/src/state_index.rs @@ -353,9 +353,11 @@ fn unknown_variant_is_reported() { use bobbin_types::sh_tangled::repo::issue::state::State as IssueStateRec; use jacquard_common::deps::smol_str::SmolStr; + use jacquard_common::types::string::Datetime; let issue_idx = idx(); let pull_idx = StateIndex::::new(RuntimeHasher::default()); let rec = Record::IssueState(IssueStateRec { + created_at: Datetime::raw_str("2026-06-11T00:00:00Z"), issue: at("at://did:plc:limpet/sh.tangled.repo.issue/i1"), state: StateState::Other(SmolStr::new_static("sh.tangled.repo.issue.state.reopened")), extra_data: None, diff --git a/knot2/crates/knot-acl/src/lib.rs b/knot2/crates/knot-acl/src/lib.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-acl/src/lib.rs @@ -0,0 +1,701 @@ +use std::collections::BTreeSet; + +use knot_index::{Index, Resolved}; +use knot_types::{AccountDid, AdmissionPolicy, OwnerDid, RepoDid}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[must_use] +pub enum Decision { + Allow, + Deny, +} + +impl Decision { + pub fn is_allowed(self) -> bool { + matches!(self, Decision::Allow) + } + + fn allow_if(granted: bool) -> Self { + if granted { + Decision::Allow + } else { + Decision::Deny + } + } +} + +pub trait Acl { + fn is_admin(&self, who: &AccountDid) -> bool; + fn admission(&self) -> AdmissionPolicy; + fn is_member(&self, who: &AccountDid) -> Resolved; + fn is_blocked(&self, who: &AccountDid) -> Resolved; + fn is_collaborator(&self, repo: &RepoDid, who: &AccountDid) -> Resolved; + fn repo_owner(&self, repo: &RepoDid) -> Resolved>; +} + +fn confirmed(resolved: Resolved) -> bool { + matches!(resolved, Resolved::Ready(true)) +} + +fn owns_repo(acl: &impl Acl, who: &AccountDid, repo: &RepoDid) -> bool { + confirmed( + acl.repo_owner(repo) + .map(|owner| owner.is_some_and(|owner| owner.is(who))), + ) +} + +fn not_blocked(acl: &impl Acl, who: &AccountDid) -> bool { + acl.is_admin(who) || matches!(acl.is_blocked(who), Resolved::Ready(false)) +} + +pub fn can_admin_knot(acl: &impl Acl, who: &AccountDid) -> Decision { + Decision::allow_if(acl.is_admin(who)) +} + +pub fn can_create_repo(acl: &impl Acl, who: &AccountDid) -> Decision { + Decision::allow_if( + acl.is_admin(who) + || (not_blocked(acl, who) + && match acl.admission() { + AdmissionPolicy::Open => true, + AdmissionPolicy::Closed => confirmed(acl.is_member(who)), + }), + ) +} + +pub fn can_push(acl: &impl Acl, who: &AccountDid, repo: &RepoDid) -> Decision { + Decision::allow_if( + not_blocked(acl, who) + && (owns_repo(acl, who, repo) || confirmed(acl.is_collaborator(repo, who))), + ) +} + +pub fn can_manage_collaborators(acl: &impl Acl, who: &AccountDid, repo: &RepoDid) -> Decision { + Decision::allow_if(not_blocked(acl, who) && owns_repo(acl, who, repo)) +} + +pub fn can_delete_repo(acl: &impl Acl, who: &AccountDid, repo: &RepoDid) -> Decision { + Decision::allow_if(acl.is_admin(who) || owns_repo(acl, who, repo)) +} + +pub struct KnotAcl<'a> { + admins: &'a BTreeSet, + policy: AdmissionPolicy, + index: &'a Index, +} + +impl<'a> KnotAcl<'a> { + pub fn new( + admins: &'a BTreeSet, + policy: AdmissionPolicy, + index: &'a Index, + ) -> Self { + Self { + admins, + policy, + index, + } + } +} + +impl Acl for KnotAcl<'_> { + fn is_admin(&self, who: &AccountDid) -> bool { + self.admins.contains(who) + } + + fn admission(&self) -> AdmissionPolicy { + self.policy + } + + fn is_member(&self, who: &AccountDid) -> Resolved { + self.index.is_member(who) + } + + fn is_blocked(&self, who: &AccountDid) -> Resolved { + self.index.is_blocked(who) + } + + fn is_collaborator(&self, repo: &RepoDid, who: &AccountDid) -> Resolved { + self.index.is_collaborator(repo, who) + } + + fn repo_owner(&self, repo: &RepoDid) -> Resolved> { + self.index.owner_of(repo) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn acc(suffix: &str) -> AccountDid { + AccountDid::new(format!("did:plc:{suffix}")).unwrap() + } + + fn owner(suffix: &str) -> OwnerDid { + OwnerDid::new(format!("did:plc:{suffix}")).unwrap() + } + + fn repo(suffix: &str) -> RepoDid { + RepoDid::new(format!("did:plc:{suffix}")).unwrap() + } + + struct Fake { + admins: BTreeSet, + admission: AdmissionPolicy, + member: Resolved, + blocked: Resolved, + collaborator: Resolved, + owner: Resolved>, + } + + impl Fake { + fn new() -> Self { + Self { + admins: BTreeSet::new(), + admission: AdmissionPolicy::Closed, + member: Resolved::Warming, + blocked: Resolved::Ready(false), + collaborator: Resolved::Warming, + owner: Resolved::Warming, + } + } + + fn admin(mut self, who: &str) -> Self { + self.admins.insert(acc(who)); + self + } + + fn open(mut self) -> Self { + self.admission = AdmissionPolicy::Open; + self + } + + fn member(mut self, resolved: Resolved) -> Self { + self.member = resolved; + self + } + + fn blocked(mut self, resolved: Resolved) -> Self { + self.blocked = resolved; + self + } + + fn collaborator(mut self, resolved: Resolved) -> Self { + self.collaborator = resolved; + self + } + + fn owner(mut self, resolved: Resolved>) -> Self { + self.owner = resolved; + self + } + } + + impl Acl for Fake { + fn is_admin(&self, who: &AccountDid) -> bool { + self.admins.contains(who) + } + + fn admission(&self) -> AdmissionPolicy { + self.admission + } + + fn is_member(&self, _who: &AccountDid) -> Resolved { + self.member.clone() + } + + fn is_blocked(&self, _who: &AccountDid) -> Resolved { + self.blocked.clone() + } + + fn is_collaborator(&self, _repo: &RepoDid, _who: &AccountDid) -> Resolved { + self.collaborator.clone() + } + + fn repo_owner(&self, _repo: &RepoDid) -> Resolved> { + self.owner.clone() + } + } + + #[test] + fn an_admin_administers_and_creates_but_does_not_push_arbitrary_repos() { + let acl = Fake::new() + .admin("nel") + .owner(Resolved::Ready(Some(owner("olaren")))) + .collaborator(Resolved::Ready(false)); + assert_eq!(can_admin_knot(&acl, &acc("nel")), Decision::Allow); + assert_eq!(can_create_repo(&acl, &acc("nel")), Decision::Allow); + assert_eq!( + can_push(&acl, &acc("nel"), &repo("squid")), + Decision::Deny, + "knot admin has no push on repo it neither owns nor collaborates on" + ); + } + + #[test] + fn decisions() { + type Case = (&'static str, Fake, fn(&Fake) -> Decision, Decision); + let cases: Vec = vec![ + ( + "a_member_creates_repos", + Fake::new().member(Resolved::Ready(true)), + |acl| can_create_repo(acl, &acc("olaren")), + Decision::Allow, + ), + ( + "a_member_cannot_administer_the_knot", + Fake::new().member(Resolved::Ready(true)), + |acl| can_admin_knot(acl, &acc("olaren")), + Decision::Deny, + ), + ( + "an_open_knot_admits_a_non_member", + Fake::new().open().member(Resolved::Ready(false)), + |acl| can_create_repo(acl, &acc("teq")), + Decision::Allow, + ), + ( + "an_open_knot_does_not_widen_push", + Fake::new() + .open() + .owner(Resolved::Ready(Some(owner("nel")))) + .collaborator(Resolved::Ready(false)), + |acl| can_push(acl, &acc("teq"), &repo("squid")), + Decision::Deny, + ), + ( + "a_blocked_account_cannot_create", + Fake::new() + .open() + .member(Resolved::Ready(true)) + .blocked(Resolved::Ready(true)), + |acl| can_create_repo(acl, &acc("squid")), + Decision::Deny, + ), + ( + "an_admin_is_immune_to_the_blocklist", + Fake::new() + .open() + .admin("nel") + .blocked(Resolved::Ready(true)), + |acl| can_create_repo(acl, &acc("nel")), + Decision::Allow, + ), + ( + "the_repo_owner_pushes", + Fake::new() + .owner(Resolved::Ready(Some(owner("nel")))) + .collaborator(Resolved::Ready(false)), + |acl| can_push(acl, &acc("nel"), &repo("squid")), + Decision::Allow, + ), + ( + "a_collaborator_pushes_without_owning", + Fake::new() + .owner(Resolved::Ready(Some(owner("nel")))) + .collaborator(Resolved::Ready(true)), + |acl| can_push(acl, &acc("olaren"), &repo("squid")), + Decision::Allow, + ), + ( + "push_allows_on_a_confirmed_collaborator_while_the_registry_warms", + Fake::new() + .owner(Resolved::Warming) + .collaborator(Resolved::Ready(true)), + |acl| can_push(acl, &acc("olaren"), &repo("squid")), + Decision::Allow, + ), + ( + "push_allows_a_confirmed_owner_while_collaborators_warm", + Fake::new() + .owner(Resolved::Ready(Some(owner("nel")))) + .collaborator(Resolved::Warming), + |acl| can_push(acl, &acc("nel"), &repo("squid")), + Decision::Allow, + ), + ( + "push_denies_when_ownership_is_warming_and_not_a_collaborator", + Fake::new() + .owner(Resolved::Warming) + .collaborator(Resolved::Ready(false)), + |acl| can_push(acl, &acc("nel"), &repo("squid")), + Decision::Deny, + ), + ( + "push_denies_an_unregistered_repo", + Fake::new() + .owner(Resolved::Ready(None)) + .collaborator(Resolved::Ready(false)), + |acl| can_push(acl, &acc("nel"), &repo("squid")), + Decision::Deny, + ), + ( + "push_matches_a_did_web_owner_across_authority_case", + Fake::new() + .owner(Resolved::Ready(Some( + OwnerDid::new("did:web:OYSTER.cafe").unwrap(), + ))) + .collaborator(Resolved::Ready(false)), + |acl| { + can_push( + acl, + &AccountDid::new("did:web:oyster.cafe").unwrap(), + &repo("squid"), + ) + }, + Decision::Allow, + ), + ( + "push_denies_a_did_plc_owner_whose_case_differs", + Fake::new() + .owner(Resolved::Ready(Some(OwnerDid::new("did:plc:ABC").unwrap()))) + .collaborator(Resolved::Ready(false)), + |acl| { + can_push( + acl, + &AccountDid::new("did:plc:abc").unwrap(), + &repo("squid"), + ) + }, + Decision::Deny, + ), + ( + "manage_collaborators_fails_closed_while_ownership_is_warming", + Fake::new().owner(Resolved::Warming), + |acl| can_manage_collaborators(acl, &acc("olaren"), &repo("squid")), + Decision::Deny, + ), + ( + "an_admin_is_authorized_before_the_projection_warms", + Fake::new().admin("nel"), + |acl| can_admin_knot(acl, &acc("nel")), + Decision::Allow, + ), + ( + "an_admin_creates_before_the_projection_warms", + Fake::new().admin("nel"), + |acl| can_create_repo(acl, &acc("nel")), + Decision::Allow, + ), + ( + "an_admin_deletes_a_repo_before_the_registry_warms", + Fake::new().admin("nel").owner(Resolved::Warming), + |acl| can_delete_repo(acl, &acc("nel"), &repo("squid")), + Decision::Allow, + ), + ]; + cases.iter().for_each(|(label, acl, eval, expected)| { + assert_eq!(eval(acl), *expected, "{label}"); + }); + } + + #[test] + fn a_blocked_owner_cannot_push_or_invite() { + let acl = Fake::new() + .owner(Resolved::Ready(Some(owner("squid")))) + .collaborator(Resolved::Ready(false)) + .blocked(Resolved::Ready(true)); + assert_eq!( + can_push(&acl, &acc("squid"), &repo("anemone")), + Decision::Deny, + "ban overrides ownership on write path" + ); + assert_eq!( + can_manage_collaborators(&acl, &acc("squid"), &repo("anemone")), + Decision::Deny + ); + } + + #[test] + fn a_warming_blocklist_fails_create_and_push_closed() { + let acl = Fake::new() + .open() + .blocked(Resolved::Warming) + .owner(Resolved::Ready(Some(owner("squid")))); + assert_eq!( + can_create_repo(&acl, &acc("squid")), + Decision::Deny, + "unresolved blocklist must not admit, ban could be hiding in it" + ); + assert_eq!( + can_push(&acl, &acc("squid"), &repo("anemone")), + Decision::Deny + ); + } + + #[test] + fn a_stranger_is_denied_everything() { + let acl = Fake::new() + .member(Resolved::Ready(false)) + .collaborator(Resolved::Ready(false)) + .owner(Resolved::Ready(Some(owner("nel")))); + assert_eq!(can_admin_knot(&acl, &acc("teq")), Decision::Deny); + assert_eq!(can_create_repo(&acl, &acc("teq")), Decision::Deny); + assert_eq!(can_push(&acl, &acc("teq"), &repo("squid")), Decision::Deny); + } + + #[test] + fn a_fully_warming_index_denies_every_index_backed_decision() { + let acl = Fake::new(); + assert_eq!(can_create_repo(&acl, &acc("olaren")), Decision::Deny); + assert_eq!(can_push(&acl, &acc("nel"), &repo("squid")), Decision::Deny); + } + + #[test] + fn is_allowed_reports_the_verdict() { + assert!(Decision::Allow.is_allowed()); + assert!(!Decision::Deny.is_allowed()); + } + + #[test] + fn only_the_repo_owner_manages_collaborators() { + let acl = Fake::new() + .admin("nel") + .owner(Resolved::Ready(Some(owner("olaren")))) + .collaborator(Resolved::Ready(true)); + assert_eq!( + can_manage_collaborators(&acl, &acc("olaren"), &repo("squid")), + Decision::Allow, + "repo owner manages its own collaborators" + ); + assert_eq!( + can_manage_collaborators(&acl, &acc("lyna"), &repo("squid")), + Decision::Deny, + "collaborator cannot manage collaborator set" + ); + assert_eq!( + can_manage_collaborators(&acl, &acc("nel"), &repo("squid")), + Decision::Deny, + "knot admin has no collaborator-invite right on repo it does not own" + ); + } + + #[test] + fn repo_deletion_is_the_owner_or_a_knot_admin() { + let acl = Fake::new() + .admin("nel") + .owner(Resolved::Ready(Some(owner("olaren")))) + .collaborator(Resolved::Ready(true)); + assert_eq!( + can_delete_repo(&acl, &acc("olaren"), &repo("squid")), + Decision::Allow, + "repo owner deletes its own repo" + ); + assert_eq!( + can_delete_repo(&acl, &acc("nel"), &repo("squid")), + Decision::Allow, + "knot admin deletes any repo" + ); + assert_eq!( + can_delete_repo(&acl, &acc("lyna"), &repo("squid")), + Decision::Deny, + "collaborator cannot delete the repo" + ); + } + + mod integration { + use super::*; + use knot_cob::{ChangePayload, CobHome, CobStore}; + use knot_cobs::{CollaboratorsChange, Grant, MembersChange, Registration, RegistryChange}; + use knot_git::{Layout, Repo}; + use knot_runtime::{K256Signer, SeededEntropy}; + use knot_types::{KnotId, RepoName, RepoRkey, UnixSeconds}; + use std::path::PathBuf; + + fn knot_home() -> CobHome { + CobHome::from(&KnotId::new("did:web:knot.nel.pet").unwrap()) + } + + fn grant(subject: &str, at: i64) -> Grant { + Grant { + subject: acc(subject), + added_by: acc("nel"), + created_at: UnixSeconds::new(at), + } + } + + fn registration( + owner_id: &str, + key: &str, + repo_did: &knot_types::RepoDid, + at: i64, + ) -> Registration { + Registration { + owner: owner(owner_id), + rkey: RepoRkey::new(key).unwrap(), + name: RepoName::new(key).unwrap(), + repo: repo_did.clone(), + created_at: UnixSeconds::new(at), + } + } + + fn world() -> (tempfile::TempDir, PathBuf, Layout, K256Signer) { + let dir = tempfile::tempdir().unwrap(); + let meta_path = dir.path().join("meta"); + Repo::create(&meta_path).unwrap(); + let layout = Layout::new(dir.path().join("repos")); + let signer = K256Signer::generate(&SeededEntropy::new(1)); + (dir, meta_path, layout, signer) + } + + fn seed( + store: &CobStore, + home: &CobHome, + change: &P, + signer: &K256Signer, + at: UnixSeconds, + ) { + store.create(home, change, signer, at).unwrap(); + } + + #[test] + fn the_enforcer_decides_over_a_real_rebuilt_index() { + let (_dir, meta_path, layout, signer) = world(); + let at = UnixSeconds::new; + + let meta = Repo::open(&meta_path).unwrap(); + let store = CobStore::new(&meta); + let squid = repo("squid"); + seed( + &store, + &knot_home(), + &MembersChange::Add(grant("olaren", 1)), + &signer, + at(1), + ); + seed( + &store, + &knot_home(), + &RegistryChange::Register(registration("nel", "anemone", &squid, 1)), + &signer, + at(1), + ); + let git = layout.create(&squid).unwrap(); + seed( + &CobStore::new(&git), + &CobHome::from(&squid), + &CollaboratorsChange::Add(grant("lyna", 1)), + &signer, + at(1), + ); + + let index = Index::new(&meta_path, layout.clone()); + index.rebuild().unwrap(); + index.ensure_collaborators(&squid).unwrap(); + let admins = BTreeSet::from([acc("nel")]); + let acl = KnotAcl::new(&admins, AdmissionPolicy::Closed, &index); + + assert_eq!(can_admin_knot(&acl, &acc("nel")), Decision::Allow); + assert_eq!(can_create_repo(&acl, &acc("nel")), Decision::Allow); + assert_eq!( + can_push(&acl, &acc("nel"), &squid), + Decision::Allow, + "nel owns squid in the registry" + ); + + assert_eq!(can_admin_knot(&acl, &acc("olaren")), Decision::Deny); + assert_eq!(can_create_repo(&acl, &acc("olaren")), Decision::Allow); + assert_eq!( + can_push(&acl, &acc("olaren"), &squid), + Decision::Deny, + "member who is neither owner nor collaborator cannot push" + ); + + assert_eq!( + can_push(&acl, &acc("lyna"), &squid), + Decision::Allow, + "lyna collaborates on squid" + ); + assert_eq!(can_create_repo(&acl, &acc("lyna")), Decision::Deny); + + assert_eq!(can_push(&acl, &acc("teq"), &squid), Decision::Deny); + assert_eq!(can_create_repo(&acl, &acc("teq")), Decision::Deny); + + let cold = Index::new(&meta_path, layout); + let cold_acl = KnotAcl::new(&admins, AdmissionPolicy::Closed, &cold); + assert_eq!( + can_admin_knot(&cold_acl, &acc("nel")), + Decision::Allow, + "admin is config, answered before any rebuild" + ); + assert_eq!( + can_push(&cold_acl, &acc("nel"), &squid), + Decision::Deny, + "before rebuild owner lookup is warming, so push fails closed" + ); + assert_eq!(can_create_repo(&cold_acl, &acc("olaren")), Decision::Deny); + } + + #[test] + fn a_repo_re_registered_under_a_second_owner_grants_push_only_to_the_later_owner() { + let (_dir, meta_path, layout, signer) = world(); + let at = UnixSeconds::new; + + let squid = repo("squid"); + layout.create(&squid).unwrap(); + + let meta = Repo::open(&meta_path).unwrap(); + let store = CobStore::new(&meta); + let created = store + .create( + &knot_home(), + &RegistryChange::Register(registration("nel", "anemone", &squid, 1)), + &signer, + at(1), + ) + .unwrap(); + store + .update( + &knot_home(), + created.object, + &RegistryChange::Register(registration("olaren", "fork", &squid, 2)), + &signer, + at(2), + ) + .unwrap(); + + let index = Index::new(&meta_path, layout); + index.rebuild().unwrap(); + let admins = BTreeSet::new(); + let acl = KnotAcl::new(&admins, AdmissionPolicy::Closed, &index); + + assert_eq!( + can_push(&acl, &acc("nel"), &squid), + Decision::Deny, + "re-register moves repo wholesale, so displaced owner loses push" + ); + assert_eq!( + can_push(&acl, &acc("olaren"), &squid), + Decision::Allow, + "linear causal order gives later registrant deterministic ownership" + ); + } + + #[test] + fn a_collaborator_on_an_unregistered_repo_cannot_push_after_a_real_rebuild() { + let (_dir, meta_path, layout, signer) = world(); + let at = UnixSeconds::new; + + let squid = repo("squid"); + let git = layout.create(&squid).unwrap(); + seed( + &CobStore::new(&git), + &CobHome::from(&squid), + &CollaboratorsChange::Add(grant("lyna", 1)), + &signer, + at(1), + ); + + let index = Index::new(&meta_path, layout); + index.rebuild().unwrap(); + let admins = BTreeSet::new(); + let acl = KnotAcl::new(&admins, AdmissionPolicy::Closed, &index); + assert_eq!( + can_push(&acl, &acc("lyna"), &squid), + Decision::Deny, + "rebuild folds collaborators only for registered repos, so collaborator COB on unregistered repo never warms and grants no push" + ); + } + } +} diff --git a/knot2/crates/knot-atproto/fuzz/.gitignore b/knot2/crates/knot-atproto/fuzz/.gitignore new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-atproto/fuzz/.gitignore @@ -0,0 +1,4 @@ +target +corpus +artifacts +coverage diff --git a/knot2/crates/knot-atproto/fuzz/Cargo.lock b/knot2/crates/knot-atproto/fuzz/Cargo.lock new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-atproto/fuzz/Cargo.lock @@ -0,0 +1,4225 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + +[[package]] +name = "async-compression" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "atomic-polyfill" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" +dependencies = [ + "critical-section", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "aws-lc-rs" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + +[[package]] +name = "base-x" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" + +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + +[[package]] +name = "base256emoji" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e9430d9a245a77c92176e649af6e275f20839a48389859d1661e9a128d077c" +dependencies = [ + "const-str", + "match-lookup", +] + +[[package]] +name = "base32" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "022dfe9eb35f19ebbcb51e0b40a5ab759f46ad60cadf7297e0bd085afb50e076" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bon" +version = "3.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a602c73c7b0148ec6d12af6fd5cc7a46e2eacc8878271a999abac56eed12f561" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + +[[package]] +name = "borrow-or-share" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" + +[[package]] +name = "borsh" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3f6da4992df95bbcd9af42a6c7dcb994498fc9048230405f3b36ff7cd3f145" +dependencies = [ + "bytes", + "cfg_aliases", +] + +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +dependencies = [ + "serde", +] + +[[package]] +name = "cbor4ii" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544cf8c89359205f4f990d0e6f3828db42df85b5dac95d09157a250eb0749c4" +dependencies = [ + "serde", +] + +[[package]] +name = "cc" +version = "1.2.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "cid" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a304f95f84d169a6f31c4d0a30d784643aaa0bbc9c1e449a2c23e963ec4971" +dependencies = [ + "multibase", + "multihash", + "serde", + "serde_bytes", + "unsigned-varint 0.8.0", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.18", +] + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "brotli", + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "const-str" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f421161cb492475f1661ddc9815a745a1c894592070661180fdec3d4872e9c3" + +[[package]] +name = "cordyceps" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "688d7fbb8092b8de775ef2536f36c8c31f2bc4006ece2e8d8ad2d17d00ce0a2a" +dependencies = [ + "loom", + "tracing", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "data-encoding-macro" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3259c913752a86488b501ed8680446a5ed2d5aeac6e596cb23ba3800768ea32c" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccc2776f0c61eca1ca32528f85548abd1a4be8fb53d1b21c013e4f18da1e7090" +dependencies = [ + "data-encoding", + "syn", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "derive_more" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "diatomic-waker" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab03c107fafeb3ee9f5925686dbb7a73bc76e3932abb0d2b365cb64b169cf04c" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.6", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest 0.10.7", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "rand_core 0.6.4", + "serde", + "sha2 0.10.9", + "subtle", + "zeroize", +] + +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "pem-rfc7468", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "faster-hex" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7223ae2d2f179b803433d9c830478527e92b8117eab39460edae7f1614d9fb73" +dependencies = [ + "heapless 0.8.0", + "serde", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fluent-uri" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-buffered" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4421cb78ee172b6b06080093479d3c50f058e7c81b7d577bbb8d118d551d4cd5" +dependencies = [ + "cordyceps", + "diatomic-waker", + "futures-core", + "pin-project-lite", + "spin 0.10.0", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generator" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + +[[package]] +name = "generic-array" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +dependencies = [ + "typenum", + "version_check", + "zeroize", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + +[[package]] +name = "gix-features" +version = "0.48.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1849ae154d38bc403185be14fa871e38e3c93ee606875d94e207fdb9fba52dbc" +dependencies = [ + "gix-trace", + "libc", + "prodash", +] + +[[package]] +name = "gix-hash" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0926d3819c837750b4e03c7754901e73f68b8c9b690753a6372a1bed4eedce" +dependencies = [ + "faster-hex", + "gix-features", + "sha1-checked", + "sha2 0.11.0", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-trace" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44dc45eae785c0eb14173e0f152e6e224dcf4d45b6a6999a3aed22af541ad678" + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "h2" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heapless" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" +dependencies = [ + "atomic-polyfill", + "hash32 0.2.1", + "rustc_version", + "serde", + "spin 0.9.8", + "stable_deref_trait", +] + +[[package]] +name = "heapless" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32 0.3.1", + "stable_deref_trait", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hybrid-array" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "inventory" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +dependencies = [ + "rustversion", +] + +[[package]] +name = "ipld-core" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090f624976d72f0b0bb71b86d58dc16c15e069193067cb3a3a09d655246cbbda" +dependencies = [ + "cid", + "serde", + "serde_bytes", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "iroh-car" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f8cd4cb9aa083fba8b52e921764252d0b4dcb1cd6d120b809dbfe1106e81a" +dependencies = [ + "anyhow", + "cid", + "futures", + "serde", + "serde_ipld_dagcbor", + "thiserror 1.0.69", + "tokio", + "unsigned-varint 0.7.2", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jacquard-api" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5c803a3c097e3ef8aea63747b4fe3fc9e339cd18272dd0366b1d10dd90d5c3f" +dependencies = [ + "jacquard-common", + "jacquard-derive", + "jacquard-lexicon", + "miette", + "serde", + "thiserror 2.0.18", +] + +[[package]] +name = "jacquard-common" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fec8b6661e6fcfb4d8b9eb1132503be71bc3e3c6f6ff15fa5500ec62655f6da7" +dependencies = [ + "base64", + "bon", + "bytes", + "chrono", + "ciborium", + "ciborium-io", + "cid", + "ed25519-dalek", + "fluent-uri", + "futures", + "getrandom 0.2.17", + "getrandom 0.3.4", + "hashbrown 0.15.5", + "http", + "ipld-core", + "k256", + "maitake-sync", + "miette", + "multibase", + "multihash", + "n0-future", + "oxilangtag", + "p256", + "phf", + "postcard", + "rand 0.9.4", + "regex", + "regex-automata", + "regex-lite", + "reqwest 0.12.28", + "rustversion", + "serde", + "serde_bytes", + "serde_html_form", + "serde_ipld_dagcbor", + "serde_json", + "signature", + "smol_str", + "spin 0.10.0", + "thiserror 2.0.18", + "tokio", + "tokio-tungstenite-wasm", + "tokio-util", + "trait-variant", + "unicode-segmentation", +] + +[[package]] +name = "jacquard-derive" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e41b82a9a72c0d2b907a5e0734e0692615e717cda903502c6c034ae6f3ddddf7" +dependencies = [ + "heck", + "jacquard-lexicon", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jacquard-lexicon" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "160049c269e3d7ec56f130736d25d61953fc6216da660c8b5e102f6b052dd3a1" +dependencies = [ + "cid", + "dashmap", + "heck", + "inventory", + "jacquard-common", + "miette", + "multihash", + "prettyplease", + "proc-macro2", + "quote", + "serde", + "serde_ipld_dagcbor", + "serde_json", + "serde_path_to_error", + "serde_repr", + "serde_with", + "sha2 0.10.9", + "syn", + "thiserror 2.0.18", + "unicode-segmentation", +] + +[[package]] +name = "jacquard-repo" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98986367bb78dadaa0f2f07196bab357786c0e3670d8311b350585b91f84d6eb" +dependencies = [ + "bytes", + "cid", + "ed25519-dalek", + "iroh-car", + "jacquard-api", + "jacquard-common", + "jacquard-derive", + "k256", + "miette", + "multihash", + "n0-future", + "p256", + "serde", + "serde_bytes", + "serde_ipld_dagcbor", + "sha2 0.10.9", + "smol_str", + "thiserror 2.0.18", + "tokio", + "trait-variant", +] + +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "k256" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" +dependencies = [ + "cfg-if", + "ecdsa", + "elliptic-curve", + "once_cell", + "sha2 0.10.9", + "signature", +] + +[[package]] +name = "knot-atproto" +version = "0.1.0" +dependencies = [ + "base32", + "base64", + "bs58", + "bytes", + "futures", + "http", + "knot-runtime", + "knot-types", + "moka", + "scc", + "serde", + "serde_ipld_dagcbor", + "serde_json", + "sha2 0.11.0", + "thiserror 2.0.18", + "url", +] + +[[package]] +name = "knot-atproto-fuzz" +version = "0.0.0" +dependencies = [ + "knot-atproto", + "libfuzzer-sys", +] + +[[package]] +name = "knot-runtime" +version = "0.1.0" +dependencies = [ + "bytes", + "futures", + "getrandom 0.4.3", + "http", + "k256", + "reqwest 0.13.1", + "thiserror 2.0.18", + "tokio", + "url", +] + +[[package]] +name = "knot-types" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "chrono", + "cid", + "gix-hash", + "http", + "jacquard-common", + "jacquard-derive", + "jacquard-lexicon", + "jacquard-repo", + "miette", + "serde", + "serde_json", + "thiserror 2.0.18", + "walkdir", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" + +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "maitake-sync" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6816ab14147f80234c675b80ed6dc4f440d8a1cefc158e766067aedb84c0bcd5" +dependencies = [ + "cordyceps", + "loom", + "mycelium-bitfield", + "pin-project", + "portable-atomic", +] + +[[package]] +name = "match-lookup" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "757aee279b8bdbb9f9e676796fd459e4207a1f986e87886700abf589f5abf771" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "miette" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" +dependencies = [ + "cfg-if", + "miette-derive", + "unicode-width", +] + +[[package]] +name = "miette-derive" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "moka" +version = "0.12.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "event-listener", + "futures-util", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "multibase" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8694bb4835f452b0e3bb06dbebb1d6fc5385b6ca1caf2e55fd165c042390ec77" +dependencies = [ + "base-x", + "base256emoji", + "data-encoding", + "data-encoding-macro", +] + +[[package]] +name = "multihash" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "577c63b00ad74d57e8c9aa870b5fccebf2fd64a308a5aee9f1bb88e4aea19447" +dependencies = [ + "serde", + "unsigned-varint 0.8.0", +] + +[[package]] +name = "mycelium-bitfield" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24e0cc5e2c585acbd15c5ce911dff71e1f4d5313f43345873311c4f5efd741cc" + +[[package]] +name = "n0-future" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb0e5d99e681ab3c938842b96fcb41bf8a7bb4bfdb11ccbd653a7e83e06c794" +dependencies = [ + "cfg_aliases", + "derive_more", + "futures-buffered", + "futures-lite", + "futures-util", + "js-sys", + "pin-project", + "send_wrapper", + "tokio", + "tokio-util", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "oxilangtag" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d3b4eb570abd4a1dcb062c31fd37b832264d9dc7292c3e69acfe926c87b063f" +dependencies = [ + "serde", +] + +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2 0.10.9", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand 0.8.6", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "heapless 0.7.17", + "serde", +] + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prodash" +version = "31.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "962200e2d7d551451297d9fdce85138374019ada198e30ea9ede38034e27604c" +dependencies = [ + "parking_lot", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.4", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "regex" +version = "1.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "mime", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "reqwest" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04e9018c9d814e5f30cc16a0f03271aeab3571e609612d9fe78c1aa8d11c2f62" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "aws-lc-rs", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "saa" +version = "5.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f5acb362a0e75c2a963532fa7fabf13dff81626dc494df16488d30befcbea0" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scc" +version = "3.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5581cd5dd2cb79cbe9d8137071d67f62f0db926e83a07b4d14561c5b7d423776" +dependencies = [ + "saa", + "sdd", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sdd" +version = "4.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5f0e40a01b94e35d1dacbcfbe5bfd3d31e37d9590b2e6d86a82b0e87bd4f551" +dependencies = [ + "saa", +] + +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "send_wrapper" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_html_form" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acf96b1d9364968fce46ebb548f1c0e1d7eceae27bdff73865d42e6c7369d94" +dependencies = [ + "form_urlencoded", + "indexmap", + "itoa", + "serde_core", +] + +[[package]] +name = "serde_ipld_dagcbor" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46182f4f08349a02b45c998ba3215d3f9de826246ba02bb9dddfe9a2a2100778" +dependencies = [ + "cbor4ii", + "ipld-core", + "scopeguard", + "serde", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" +dependencies = [ + "base64", + "bs58", + "chrono", + "hex", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha1-checked" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89f599ac0c323ebb1c6082821a54962b839832b03984598375bff3975b804423" +dependencies = [ + "digest 0.10.7", + "sha1", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "smol_str" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa7368fcf4852a4c2dd92df0cace6a71f2091ca0a23391ce7f3a31833f1523" +dependencies = [ + "borsh", + "serde_core", +] + +[[package]] +name = "socket2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711a53c2d47bbd818258c498c8dbfe186a2526c631495cfe7e078567f86b8469" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" +dependencies = [ + "futures-util", + "log", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tungstenite", +] + +[[package]] +name = "tokio-tungstenite-wasm" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e21a5c399399c3db9f08d8297ac12b500e86bca82e930253fdc62eaf9c0de6ae" +dependencies = [ + "futures-channel", + "futures-util", + "http", + "httparse", + "js-sys", + "rustls", + "thiserror 1.0.69", + "tokio", + "tokio-tungstenite", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "async-compression", + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tokio", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "trait-variant" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.8.6", + "rustls", + "rustls-pki-types", + "sha1", + "thiserror 1.0.69", + "utf-8", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "unsigned-varint" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" + +[[package]] +name = "unsigned-varint" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", + "serde_derive", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "webpki-roots" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/knot2/crates/knot-atproto/fuzz/Cargo.toml b/knot2/crates/knot-atproto/fuzz/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-atproto/fuzz/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "knot-atproto-fuzz" +version = "0.0.0" +publish = false +edition = "2024" + +[package.metadata] +cargo-fuzz = true + +[workspace] + +[dependencies] +libfuzzer-sys = "0.4" + +[dependencies.knot-atproto] +path = ".." + +[[bin]] +name = "pubkey" +path = "fuzz_targets/pubkey.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "did_document" +path = "fuzz_targets/did_document.rs" +test = false +doc = false +bench = false diff --git a/knot2/crates/knot-atproto/src/auth.rs b/knot2/crates/knot-atproto/src/auth.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-atproto/src/auth.rs @@ -0,0 +1,72 @@ +use http::HeaderValue; +use knot_runtime::{Entropy, HttpRequest, Signer}; +use knot_types::{KnotId, Nsid, ServiceDid, UnixSeconds}; + +use crate::AtprotoError; +use crate::jwt; +use crate::jwt::JwtNonce; + +const POINTER_NONCE_BYTES: usize = 16; + +pub struct PointerAuth<'a> { + pub issuer: &'a KnotId, + pub audience: &'a ServiceDid, + pub lxm: &'a Nsid, + pub now_unix: UnixSeconds, +} + +pub trait PointerAuthorizer { + fn authorize( + &self, + request: &mut HttpRequest, + ctx: &PointerAuth<'_>, + ) -> Result<(), AtprotoError>; +} + +pub struct ServiceAuth<'a> { + signer: &'a dyn Signer, + entropy: &'a dyn Entropy, +} + +impl<'a> ServiceAuth<'a> { + pub fn new(signer: &'a dyn Signer, entropy: &'a dyn Entropy) -> Self { + Self { signer, entropy } + } +} + +impl PointerAuthorizer for ServiceAuth<'_> { + fn authorize( + &self, + request: &mut HttpRequest, + ctx: &PointerAuth<'_>, + ) -> Result<(), AtprotoError> { + let mut bytes = [0u8; POINTER_NONCE_BYTES]; + self.entropy.fill(&mut bytes); + let nonce = JwtNonce::new(knot_types::lowercase_hex(&bytes))?; + let token = jwt::mint( + self.signer, + ctx.issuer, + ctx.audience, + ctx.lxm, + nonce, + ctx.now_unix, + ); + let header = HeaderValue::from_str(&format!("Bearer {token}")) + .expect("base64url jwt is valid header value"); + request.headers.insert(http::header::AUTHORIZATION, header); + Ok(()) + } +} + +pub struct OauthAuthorizer; + +impl PointerAuthorizer for OauthAuthorizer { + fn authorize( + &self, + _request: &mut HttpRequest, + _ctx: &PointerAuth<'_>, + ) -> Result<(), AtprotoError> { + todo!("OAuth+DPoP authorizer is waiting on an OAuth client") + // one day... + } +} diff --git a/knot2/crates/knot-atproto/src/identity.rs b/knot2/crates/knot-atproto/src/identity.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-atproto/src/identity.rs @@ -0,0 +1,337 @@ +use std::collections::BTreeMap; + +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use knot_runtime::{Entropy, PublicKeyBytes, Signer}; +use knot_types::{ActorId, KnotId, KnotServiceUrl, OwnerDid, RepoDid, RepoRkey}; +use serde::Serialize; +use sha2::{Digest, Sha256}; + +const PLC_OP_TYPE: &str = "plc_operation"; +const ATPROTO_METHOD: &str = "atproto"; +const KNOT_HOME_SERVICE: &str = "tangled_knot"; +const KNOT_HOME_TYPE: &str = "TangledKnot"; +const DID_PLC_PREFIX: &str = "did:plc:"; +const DID_SUFFIX_LEN: usize = 24; +const MINT_NONCE_LEN: usize = 16; + +#[derive(Debug, thiserror::Error)] +pub enum IdentityError { + #[error("plc operation couldn't be encoded: {0}")] + Encode(String), + #[error("derived did:plc isn't valid DID: {0}")] + Did(#[from] knot_types::ParseError), +} + +#[derive(Serialize)] +struct PlcService { + r#type: &'static str, + endpoint: String, +} + +#[derive(Serialize)] +struct PlcOperation { + #[serde(rename = "type")] + op_type: &'static str, + #[serde(rename = "rotationKeys")] + rotation_keys: Vec, + #[serde(rename = "verificationMethods")] + verification_methods: BTreeMap<&'static str, String>, + #[serde(rename = "alsoKnownAs")] + also_known_as: Vec, + services: BTreeMap<&'static str, PlcService>, + prev: Option, + #[serde(skip_serializing_if = "Option::is_none")] + sig: Option, +} + +pub struct PreparedRepoDid { + pub did: RepoDid, + operation_json: Vec, +} + +impl PreparedRepoDid { + pub fn operation_json(&self) -> &[u8] { + &self.operation_json + } +} + +fn did_key(public: &PublicKeyBytes) -> String { + format!("did:key:{}", multikey_secp256k1(public)) +} + +fn multikey_secp256k1(public: &PublicKeyBytes) -> ActorId { + ActorId::from_secp256k1(public.as_bytes()) +} + +fn encode_cbor(operation: &PlcOperation) -> Result, IdentityError> { + serde_ipld_dagcbor::to_vec(operation).map_err(|error| IdentityError::Encode(error.to_string())) +} + +pub struct MintNonce([u8; MINT_NONCE_LEN]); + +impl MintNonce { + pub fn mint(entropy: &dyn Entropy, owner: &OwnerDid, rkey: &RepoRkey) -> Self { + let mut bytes = [0u8; MINT_NONCE_LEN]; + entropy.derive(mint_label(owner, rkey)).fill(&mut bytes); + Self(bytes) + } + + fn as_bytes(&self) -> &[u8] { + &self.0 + } +} + +fn mint_label(owner: &OwnerDid, rkey: &RepoRkey) -> u64 { + [owner.as_str(), rkey.as_str()] + .iter() + .flat_map(|part| part.bytes().chain(std::iter::once(0u8))) + .fold(0xcbf2_9ce4_8422_2325u64, |hash, byte| { + (hash ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01b3) + }) +} + +pub fn prepare_repo_did( + signer: &dyn Signer, + knot_service_url: &KnotServiceUrl, + mint_nonce: &MintNonce, +) -> Result { + let tag = base32::encode( + base32::Alphabet::Rfc4648 { padding: false }, + mint_nonce.as_bytes(), + ) + .to_lowercase(); + let base = knot_service_url.as_str(); + let mut operation = PlcOperation { + op_type: PLC_OP_TYPE, + rotation_keys: vec![did_key(&signer.public_key())], + verification_methods: BTreeMap::new(), + also_known_as: Vec::new(), + services: BTreeMap::from([( + KNOT_HOME_SERVICE, + PlcService { + r#type: KNOT_HOME_TYPE, + endpoint: format!("{base}/repo/{tag}"), + }, + )]), + prev: None, + sig: None, + }; + + let unsigned = encode_cbor(&operation)?; + operation.sig = Some(URL_SAFE_NO_PAD.encode(signer.sign(&unsigned).as_bytes())); + + let signed = encode_cbor(&operation)?; + let did = derive_did_plc(&signed)?; + let operation_json = + serde_json::to_vec(&operation).map_err(|error| IdentityError::Encode(error.to_string()))?; + Ok(PreparedRepoDid { + did, + operation_json, + }) +} + +fn derive_did_plc(signed_cbor: &[u8]) -> Result { + let digest = Sha256::digest(signed_cbor); + let encoded = + base32::encode(base32::Alphabet::Rfc4648 { padding: false }, &digest).to_lowercase(); + let suffix: String = encoded.chars().take(DID_SUFFIX_LEN).collect(); + Ok(RepoDid::new(format!("{DID_PLC_PREFIX}{suffix}"))?) +} + +pub fn knot_did_document( + knot: &KnotId, + signing_key: &PublicKeyBytes, + service_url: &KnotServiceUrl, +) -> serde_json::Value { + did_web_document(knot, signing_key, service_url) +} + +fn did_web_document( + id: &KnotId, + signing_key: &PublicKeyBytes, + service_url: &KnotServiceUrl, +) -> serde_json::Value { + serde_json::json!({ + "@context": [ + "https://www.w3.org/ns/did/v1", + "https://w3id.org/security/multikey/v1", + "https://w3id.org/security/suites/secp256k1-2019/v1" + ], + "id": id, + "verificationMethod": [{ + "id": format!("{id}#{ATPROTO_METHOD}"), + "type": "Multikey", + "controller": id, + "publicKeyMultibase": multikey_secp256k1(signing_key) + }], + "service": [{ + "id": format!("#{KNOT_HOME_SERVICE}"), + "type": KNOT_HOME_TYPE, + "serviceEndpoint": service_url + }] + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::*; + use knot_runtime::verify; + + #[test] + fn did_derivation_is_deterministic_and_nonce_sensitive() { + let key = runtime_signer(2); + let first = prepare_repo_did( + &key, + &KnotServiceUrl::new("https://nel.pet").unwrap(), + &repo_nonce(101), + ) + .unwrap(); + let again = prepare_repo_did( + &key, + &KnotServiceUrl::new("https://nel.pet").unwrap(), + &repo_nonce(101), + ) + .unwrap(); + assert_eq!(first.did, again.did); + assert_eq!(first.operation_json(), again.operation_json()); + + let slashed = prepare_repo_did( + &key, + &KnotServiceUrl::new("https://nel.pet/").unwrap(), + &repo_nonce(101), + ) + .unwrap(); + assert_eq!( + first.did, slashed.did, + "a trailing slash on the knot url changes neither the endpoint nor the did" + ); + + let other_nonce = prepare_repo_did( + &key, + &KnotServiceUrl::new("https://nel.pet").unwrap(), + &repo_nonce(102), + ) + .unwrap(); + assert_ne!( + first.did, other_nonce.did, + "shared knot key no longer distinguishes repos; mint nonce must" + ); + } + + #[test] + fn the_derived_did_plc_is_pinned_and_well_formed() { + let prepared = prepare_repo_did( + &runtime_signer(1), + &KnotServiceUrl::new("https://knot.oyster.cafe").unwrap(), + &repo_nonce(104), + ) + .unwrap(); + let did = prepared.did.as_str(); + assert_eq!( + did, "did:plc:obafda42ebtgg5thl7bzyjso", + "any change to this value means did:plc derivation no longer matches the PLC directory" + ); + let suffix = did.strip_prefix(DID_PLC_PREFIX).unwrap(); + assert_eq!(suffix.len(), DID_SUFFIX_LEN); + assert!( + suffix + .chars() + .all(|c| c.is_ascii_lowercase() || ('2'..='7').contains(&c)), + "the did:plc suffix is lowercase base32" + ); + } + + #[test] + fn the_operation_signature_verifies_against_the_repo_key_over_the_unsigned_cbor() { + let key = runtime_signer(5); + let prepared = prepare_repo_did( + &key, + &KnotServiceUrl::new("https://nel.pet").unwrap(), + &repo_nonce(106), + ) + .unwrap(); + let operation: serde_json::Value = + serde_json::from_slice(prepared.operation_json()).unwrap(); + + let signature_b64 = operation["sig"].as_str().unwrap(); + let signature = + knot_runtime::Signature::from_bytes(URL_SAFE_NO_PAD.decode(signature_b64).unwrap()); + + let mut unsigned = operation.clone(); + unsigned.as_object_mut().unwrap().remove("sig"); + let unsigned_cbor = serde_ipld_dagcbor::to_vec(&unsigned).unwrap(); + + assert!( + verify(&key.public_key(), &unsigned_cbor, &signature), + "genesis op is self-signed by repo rotation key over its unsigned dag-cbor" + ); + } + + #[test] + fn the_genesis_op_marks_the_home_knot_and_has_no_signing_key() { + let key = runtime_signer(6); + let prepared = prepare_repo_did( + &key, + &KnotServiceUrl::new("https://knot.oyster.cafe").unwrap(), + &repo_nonce(107), + ) + .unwrap(); + let operation: serde_json::Value = + serde_json::from_slice(prepared.operation_json()).unwrap(); + + assert_eq!(operation["type"], "plc_operation"); + assert_eq!(operation["prev"], serde_json::Value::Null); + assert_eq!(operation["alsoKnownAs"], serde_json::json!([])); + assert!( + operation["services"][KNOT_HOME_SERVICE]["endpoint"] + .as_str() + .unwrap() + .starts_with("https://knot.oyster.cafe/repo/") + ); + assert_eq!( + operation["services"][KNOT_HOME_SERVICE]["type"], + KNOT_HOME_TYPE + ); + assert!(operation["services"]["atproto_pds"].is_null()); + assert_eq!( + operation["verificationMethods"], + serde_json::json!({}), + "repos have no atproto signing key" + ); + let expected_key = did_key(&key.public_key()); + assert_eq!(operation["rotationKeys"][0], expected_key); + assert_eq!(operation["rotationKeys"].as_array().unwrap().len(), 1); + } + + #[test] + fn the_knot_document_declares_its_signing_key_and_tangled_knot_service() { + let key = runtime_signer(7); + let knot = KnotId::new("did:web:knot.oyster.cafe").unwrap(); + let document = knot_did_document( + &knot, + &key.public_key(), + &KnotServiceUrl::new("https://knot.oyster.cafe").unwrap(), + ); + + assert_eq!( + document["verificationMethod"][0]["id"], "did:web:knot.oyster.cafe#atproto", + "knot publishes its signing key under the atproto method" + ); + assert_eq!( + document["verificationMethod"][0]["publicKeyMultibase"], + serde_json::json!(multikey_secp256k1(&key.public_key())), + "published verification method is the knot's own signing key" + ); + assert_eq!( + document["service"][0], + serde_json::json!({ + "id": "#tangled_knot", + "type": "TangledKnot", + "serviceEndpoint": "https://knot.oyster.cafe" + }), + "knot self-declares its tangled_knot service at the knot root" + ); + } +} diff --git a/knot2/crates/knot-atproto/src/jwt.rs b/knot2/crates/knot-atproto/src/jwt.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-atproto/src/jwt.rs @@ -0,0 +1,475 @@ +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use knot_runtime::{SignatureScheme, Signer}; +use knot_types::crypto::{KeyCodec, PublicKey as CryptoKey}; +use knot_types::service_auth::{ + JwtHeader, ParsedJwt, PublicKey as VerifyKey, ServiceAuthClaims, ServiceAuthError, parse_jwt, +}; +use knot_types::{AccountDid, CowStr, Did, DidService, KnotId, Nsid, ServiceDid, UnixSeconds}; + +pub(crate) const CLOCK_SKEW_SECS: i64 = 60; +pub(crate) const SERVICE_TOKEN_LIFETIME_SECS: i64 = 60; +const MAX_TOKEN_LIFETIME_SECS: i64 = 300; +const MAX_NONCE_BYTES: usize = 256; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct JwtNonce(String); + +#[derive(Clone, PartialEq, Eq)] +pub struct ServiceJwt(String); + +impl std::fmt::Debug for ServiceJwt { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_tuple("ServiceJwt").finish_non_exhaustive() + } +} + +impl ServiceJwt { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + let three_segments = + value.split('.').count() == 3 && value.split('.').all(|segment| !segment.is_empty()); + match three_segments { + true => Ok(Self(value)), + false => Err(JwtError::NotAJwt), + } + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl std::fmt::Display for ServiceJwt { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl JwtNonce { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + let len = value.len(); + (len <= MAX_NONCE_BYTES) + .then_some(Self(value)) + .ok_or(JwtError::OversizedNonce { len }) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +#[derive(Debug, thiserror::Error)] +pub enum JwtError { + #[error("malformed token: {0}")] + Parse(#[from] ServiceAuthError), + #[error("token type {typ:?} isn't JWT")] + UnexpectedType { typ: String }, + #[error("token isn't three dot-separated JWT segments")] + NotAJwt, + #[error("issuer {value:?} isn't valid account DID")] + MalformedIssuer { value: String }, + #[error("token has no jti, refusing write without replay protection")] + MissingNonce, + #[error("{len}-byte jti exceeds {MAX_NONCE_BYTES}-byte nonce limit")] + OversizedNonce { len: usize }, + #[error("issuer key codec isn't a signing algorithm")] + UnsupportedKeyCodec, + #[error("issuer key isn't valid verifying key: {0}")] + MalformedKey(String), + #[error("signature doesn't verify against issuer key")] + InvalidSignature, + #[error("audience mismatch: token addressed {actual}, expected {expected}")] + AudienceMismatch { expected: String, actual: String }, + #[error("token expired at {exp}, now {now}")] + Expired { exp: UnixSeconds, now: UnixSeconds }, + #[error("token issued in future: iat {iat}, now {now}")] + IssuedInFuture { iat: UnixSeconds, now: UnixSeconds }, + #[error("token lifetime is too long: iat {iat}, exp {exp}, limit {max}s")] + LifetimeTooLong { + exp: UnixSeconds, + iat: UnixSeconds, + max: i64, + }, + #[error("token expires at {exp}, before its issue at {iat}")] + ExpiresBeforeIssued { exp: UnixSeconds, iat: UnixSeconds }, + #[error("method binding mismatch: token bound to {actual:?}, expected {expected}")] + MethodMismatch { + expected: String, + actual: Option, + }, +} + +pub(crate) fn mint( + signer: &dyn Signer, + issuer: &KnotId, + audience: &ServiceDid, + method: &Nsid, + nonce: JwtNonce, + now_unix: UnixSeconds, +) -> ServiceJwt { + let alg = match signer.scheme() { + SignatureScheme::Secp256k1 => "ES256K", + SignatureScheme::P256 => "ES256", + }; + let header = JwtHeader { + alg: CowStr::new_static(alg), + typ: CowStr::new_static("JWT"), + }; + let claims = ServiceAuthClaims { + iss: Did::new_owned(issuer.as_str()).expect("knot DID parses as a DID"), + aud: DidService::new_owned(audience.as_str()).expect("service DID parses as a DID"), + exp: now_unix + .saturating_add_secs(SERVICE_TOKEN_LIFETIME_SECS) + .get(), + iat: now_unix.get(), + jti: Some(nonce.as_str().into()), + lxm: Some(method.clone()), + }; + let header_b64 = + URL_SAFE_NO_PAD.encode(serde_json::to_vec(&header).expect("jwt header serializes")); + let payload_b64 = + URL_SAFE_NO_PAD.encode(serde_json::to_vec(&claims).expect("service-auth claims serialize")); + let signing_input = format!("{header_b64}.{payload_b64}"); + let signature = signer.sign(signing_input.as_bytes()); + ServiceJwt(format!( + "{signing_input}.{}", + URL_SAFE_NO_PAD.encode(signature.as_bytes()) + )) +} + +pub fn parse(token: &ServiceJwt) -> Result { + let parsed = parse_jwt(token.as_str())?; + let typ = parsed.header().typ.as_str(); + if !typ.eq_ignore_ascii_case("JWT") { + return Err(JwtError::UnexpectedType { + typ: typ.to_string(), + }); + } + Ok(parsed) +} + +pub fn issuer(parsed: &ParsedJwt) -> Result { + let iss = parsed.claims().iss.as_str(); + AccountDid::new(iss).map_err(|_| JwtError::MalformedIssuer { + value: iss.to_string(), + }) +} + +fn verifying_key(key: &CryptoKey<'_>) -> Result { + match key.codec { + KeyCodec::Secp256k1 => VerifyKey::from_k256_bytes(&key.bytes) + .map_err(|e| JwtError::MalformedKey(e.to_string())), + KeyCodec::P256 => VerifyKey::from_p256_bytes(&key.bytes) + .map_err(|e| JwtError::MalformedKey(e.to_string())), + KeyCodec::Ed25519 | KeyCodec::Unknown(_) => Err(JwtError::UnsupportedKeyCodec), + } +} + +pub trait TokenAudience { + fn as_str(&self) -> &str; + fn canonicalizes(&self, claimed: &str) -> bool; +} + +impl TokenAudience for KnotId { + fn as_str(&self) -> &str { + KnotId::as_str(self) + } + + fn canonicalizes(&self, claimed: &str) -> bool { + KnotId::new(claimed).is_ok_and(|aud| aud.as_str() == KnotId::as_str(self)) + } +} + +impl TokenAudience for ServiceDid { + fn as_str(&self) -> &str { + ServiceDid::as_str(self) + } + + fn canonicalizes(&self, claimed: &str) -> bool { + ServiceDid::new(claimed).is_ok_and(|aud| aud.as_str() == ServiceDid::as_str(self)) + } +} + +pub fn check_claims( + parsed: &ParsedJwt, + audience: &impl TokenAudience, + method: &Nsid, + now_unix: UnixSeconds, +) -> Result<(), JwtError> { + let claims = parsed.claims(); + let exp = UnixSeconds::new(claims.exp); + let iat = UnixSeconds::new(claims.iat); + + if !audience.canonicalizes(claims.aud.as_str()) { + return Err(JwtError::AudienceMismatch { + expected: audience.as_str().to_string(), + actual: claims.aud.as_str().to_string(), + }); + } + + if exp.saturating_add_secs(CLOCK_SKEW_SECS) < now_unix { + return Err(JwtError::Expired { exp, now: now_unix }); + } + + if iat.saturating_sub_secs(CLOCK_SKEW_SECS) > now_unix { + return Err(JwtError::IssuedInFuture { iat, now: now_unix }); + } + + if exp < iat { + return Err(JwtError::ExpiresBeforeIssued { exp, iat }); + } + + if exp.get().saturating_sub(iat.get()) > MAX_TOKEN_LIFETIME_SECS { + return Err(JwtError::LifetimeTooLong { + exp, + iat, + max: MAX_TOKEN_LIFETIME_SECS, + }); + } + + let bound = claims.lxm.as_ref().map(|lxm| lxm.as_str()); + if bound != Some(method.as_str()) { + return Err(JwtError::MethodMismatch { + expected: method.as_str().to_string(), + actual: bound.map(str::to_string), + }); + } + + Ok(()) +} + +pub(crate) fn nonce(parsed: &ParsedJwt) -> Result { + let jti: &str = parsed + .claims() + .jti + .as_ref() + .map(|jti| jti.as_ref()) + .ok_or(JwtError::MissingNonce)?; + JwtNonce::new(jti) +} + +pub fn verify_signature(parsed: &ParsedJwt, issuer_key: &CryptoKey<'_>) -> Result<(), JwtError> { + let key = verifying_key(issuer_key)?; + knot_types::service_auth::verify_signature(parsed, &key).map_err(|error| match error { + ServiceAuthError::InvalidSignature => JwtError::InvalidSignature, + other => JwtError::Parse(other), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::mint as mint_claims; + use crate::test_support::*; + use std::borrow::Cow; + + fn claims(iss: &str, aud: &str, exp: UnixSeconds, lxm: &str) -> serde_json::Value { + serde_json::json!({ + "iss": iss, + "aud": aud, + "exp": exp.get(), + "iat": exp.saturating_sub_secs(60).get(), + "lxm": lxm, + }) + } + + #[test] + fn a_well_formed_token_authenticates_its_issuer() { + let signing = signer(1); + let public = k256_public(&signing); + let token = mint_claims( + &signing, + &claims(SQUID, KNOT, UnixSeconds::new(1_000), METHOD), + ); + let parsed = parse(&token).unwrap(); + check_claims( + &parsed, + &knot_did(KNOT), + &member_method(), + UnixSeconds::new(900), + ) + .unwrap(); + verify_signature(&parsed, &public).unwrap(); + assert_eq!(issuer(&parsed).unwrap(), AccountDid::new(SQUID).unwrap()); + } + + struct ClaimsCase { + name: &'static str, + claims: fn() -> serde_json::Value, + now: i64, + expect: fn(&Result<(), JwtError>) -> bool, + } + + const CLAIMS_CASES: &[ClaimsCase] = &[ + ClaimsCase { + name: "expired past the skew window", + claims: || serde_json::json!({ "iss": SQUID, "aud": KNOT, "exp": 1_000, "iat": 940, "lxm": METHOD }), + now: 1_100, + expect: |r| { + matches!(r, Err(JwtError::Expired { exp, now }) + if *exp == UnixSeconds::new(1_000) && *now == UnixSeconds::new(1_100)) + }, + }, + ClaimsCase { + name: "within the skew window past exp", + claims: || serde_json::json!({ "iss": SQUID, "aud": KNOT, "exp": 1_000, "iat": 940, "lxm": METHOD }), + now: 1_030, + expect: |r| r.is_ok(), + }, + ClaimsCase { + name: "addressed to another knot", + claims: || serde_json::json!({ "iss": SQUID, "aud": "did:web:oyster.cafe", "exp": 1_000, "iat": 940, "lxm": METHOD }), + now: 900, + expect: |r| matches!(r, Err(JwtError::AudienceMismatch { .. })), + }, + ClaimsCase { + name: "bound to another method", + claims: || serde_json::json!({ "iss": SQUID, "aud": KNOT, "exp": 1_000, "iat": 940, "lxm": "sh.tangled.repo.delete" }), + now: 900, + expect: |r| matches!(r, Err(JwtError::MethodMismatch { .. })), + }, + ClaimsCase { + name: "has no method binding", + claims: || serde_json::json!({ "iss": SQUID, "aud": KNOT, "exp": 1_000, "iat": 940 }), + now: 900, + expect: |r| matches!(r, Err(JwtError::MethodMismatch { actual: None, .. })), + }, + ClaimsCase { + name: "issued in the future", + claims: || serde_json::json!({ "iss": SQUID, "aud": KNOT, "exp": 2_001, "iat": 2_000, "lxm": METHOD }), + now: 900, + expect: |r| { + matches!(r, Err(JwtError::IssuedInFuture { iat, now }) + if *iat == UnixSeconds::new(2_000) && *now == UnixSeconds::new(900)) + }, + }, + ClaimsCase { + name: "lifetime exceeds the limit", + claims: || serde_json::json!({ "iss": SQUID, "aud": KNOT, "exp": 1_400, "iat": 1_000, "lxm": METHOD }), + now: 1_000, + expect: |r| { + matches!(r, Err(JwtError::LifetimeTooLong { exp, iat, max }) + if *exp == UnixSeconds::new(1_400) && *iat == UnixSeconds::new(1_000) && *max == 300) + }, + }, + ClaimsCase { + name: "expires before it was issued", + claims: || serde_json::json!({ "iss": SQUID, "aud": KNOT, "exp": 1_040, "iat": 1_050, "lxm": METHOD }), + now: 1_000, + expect: |r| { + matches!(r, Err(JwtError::ExpiresBeforeIssued { exp, iat }) + if *exp == UnixSeconds::new(1_040) && *iat == UnixSeconds::new(1_050)) + }, + }, + ClaimsCase { + name: "audience in a different case still matches", + claims: || serde_json::json!({ "iss": SQUID, "aud": "did:web:NEL.PET", "exp": 1_000, "iat": 940, "lxm": METHOD }), + now: 900, + expect: |r| r.is_ok(), + }, + ]; + + #[test] + fn check_claims_enforces_the_audience_window_and_method_binding() { + let signing = signer(1); + CLAIMS_CASES.iter().for_each(|case| { + let token = mint_claims(&signing, &(case.claims)()); + let parsed = parse(&token).unwrap(); + let result = check_claims( + &parsed, + &knot_did(KNOT), + &member_method(), + UnixSeconds::new(case.now), + ); + assert!( + (case.expect)(&result), + "case {:?} got {result:?}", + case.name + ); + }); + } + + #[test] + fn nonce_extraction_enforces_presence_and_limit() { + let signing = signer(1); + let with = |jti: Option| { + let mut body = claims(SQUID, KNOT, UnixSeconds::new(1_000), METHOD); + if let Some(jti) = jti { + body["jti"] = serde_json::json!(jti); + } + parse(&mint_claims(&signing, &body)).unwrap() + }; + + assert!(matches!(nonce(&with(None)), Err(JwtError::MissingNonce))); + assert_eq!( + nonce(&with(Some("nonce-1".to_string()))).unwrap().as_str(), + "nonce-1" + ); + assert!(matches!( + nonce(&with(Some("n".repeat(MAX_NONCE_BYTES + 1)))), + Err(JwtError::OversizedNonce { len }) if len == MAX_NONCE_BYTES + 1 + )); + assert!(nonce(&with(Some("n".repeat(MAX_NONCE_BYTES)))).is_ok()); + } + + #[test] + fn a_minted_token_round_trips_through_the_verify_half_and_honors_its_lifetime() { + let key = runtime_signer(8); + let knot_issuer = knot_did(KNOT); + let audience = ServiceDid::new("did:web:pds.oyster.cafe").unwrap(); + let bound = Nsid::new_owned("com.atproto.repo.putRecord").unwrap(); + let token = super::mint( + &key, + &knot_issuer, + &audience, + &bound, + JwtNonce::new("nonce-minted").unwrap(), + UnixSeconds::new(1_000), + ); + let parsed = parse(&token).unwrap(); + + assert_eq!(parsed.claims().iat, 1_000); + assert_eq!(parsed.claims().exp, 1_000 + SERVICE_TOKEN_LIFETIME_SECS); + check_claims(&parsed, &audience, &bound, UnixSeconds::new(1_005)).unwrap(); + assert_eq!(nonce(&parsed).unwrap().as_str(), "nonce-minted"); + assert_eq!(issuer(&parsed).unwrap(), AccountDid::new(KNOT).unwrap()); + + let public = CryptoKey { + codec: KeyCodec::Secp256k1, + bytes: Cow::Owned(knot_runtime::Signer::public_key(&key).as_bytes().to_vec()), + }; + verify_signature(&parsed, &public).unwrap(); + + let stranger = k256_public(&signer(4)); + assert!(matches!( + verify_signature(&parsed, &stranger).unwrap_err(), + JwtError::InvalidSignature + )); + } + + #[test] + fn a_jwt_nonce_preserves_its_string_and_compares_by_value() { + let nonce = JwtNonce::new("nonce-value").unwrap(); + assert_eq!(nonce.as_str(), "nonce-value"); + assert_eq!(nonce, JwtNonce::new("nonce-value".to_string()).unwrap()); + assert_ne!(nonce, JwtNonce::new("other").unwrap()); + } + + #[test] + fn an_ed25519_issuer_key_is_unsupported() { + let signing = signer(1); + let token = mint_claims( + &signing, + &claims(SQUID, KNOT, UnixSeconds::new(1_000), METHOD), + ); + let parsed = parse(&token).unwrap(); + let ed = CryptoKey { + codec: KeyCodec::Ed25519, + bytes: Cow::Owned(vec![0u8; 32]), + }; + let error = verify_signature(&parsed, &ed).unwrap_err(); + assert!(matches!(error, JwtError::UnsupportedKeyCodec)); + } +} diff --git a/knot2/crates/knot-atproto/src/lib.rs b/knot2/crates/knot-atproto/src/lib.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-atproto/src/lib.rs @@ -0,0 +1,2102 @@ +mod auth; +mod identity; +mod jwt; +mod pointer; +mod pubkeys; +mod resolve; +#[cfg(test)] +mod test_support; + +pub use auth::{OauthAuthorizer, PointerAuth, PointerAuthorizer, ServiceAuth}; +pub use identity::{ + IdentityError, MintNonce, PreparedRepoDid, knot_did_document, prepare_repo_did, +}; +pub use jwt::{JwtError, JwtNonce, ServiceJwt}; +pub use pointer::PointerReceipt; +pub use pubkeys::{KeyParseError, parse_authorized_key}; +pub use resolve::{Identity, PdsEndpoint, PlcDirectory, ResolveError}; + +#[doc(hidden)] +pub mod fuzz { + pub fn pubkey(data: &[u8]) { + let _ = crate::pubkeys::offered_page(data, 100); + let _ = crate::parse_authorized_key(&String::from_utf8_lossy(data)); + } + + pub fn did_document(data: &[u8]) { + let did = knot_types::AccountDid::new("did:plc:nel").expect("constant test did is valid"); + let _ = crate::resolve::identity_from_document(&did, data); + let repo_did = knot_types::RepoDid::new("did:plc:nel").expect("constant test did is valid"); + let _ = crate::resolve::document_publishes_key( + &repo_did, + data, + &knot_runtime::PublicKeyBytes::from_bytes(data.to_vec()), + ); + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RecordPresence { + Present, + Absent, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReplayGuard { + SingleUse, + ReusableUntilExpiry, +} + +use std::sync::Arc; +use std::time::Duration; + +use futures::stream::{self, TryStreamExt}; +use http::StatusCode; +use knot_cache::{ + Admitted, AsyncCache, EntryCount, Expiring, GroupQuota, MokaFuture, Quotas, Rejected, + TotalQuota, +}; +use knot_runtime::{ + Clock, DnsTxtResolver, HttpRequest, HttpTransport, NetworkError, PublicKeyBytes, SystemDns, + UnixMicros, +}; +use knot_types::{ + AccountDid, Collection, Handle, HttpStatus, KnotId, Nsid, OfferedKey, RepoDid, RepoRkey, Rkey, + UnixSeconds, +}; +use pubkeys::Cursor; +use serde::{Deserialize, Serialize}; +use url::Url; + +const DEFAULT_TTL: Duration = Duration::from_secs(300); +const NEGATIVE_TTL: Duration = Duration::from_secs(30); +const STALE_TTL: Duration = Duration::from_secs(30); +const PUBKEY_PAGE_LIMIT: u16 = 100; +fn repo_collection() -> Nsid { + Nsid::new_static("sh.tangled.repo").expect("literal nsid parses") +} +const PUBKEY_MAX_PAGES: usize = 8; +const MAX_IDENTITY_CACHE: usize = 4096; +const MAX_SEEN_JTI: usize = 8192; + +const MAX_JTI_PER_ISSUER: usize = MAX_SEEN_JTI / 16; + +#[derive(Debug, thiserror::Error)] +pub enum AtprotoError { + #[error(transparent)] + Resolve(#[from] ResolveError), + #[error(transparent)] + Jwt(#[from] JwtError), + #[error("network failure: {0}")] + Network(#[from] NetworkError), + #[error("listRecords for {did} returned HTTP {status}")] + ListRecords { did: AccountDid, status: HttpStatus }, + #[error("listRecords response wasn't valid JSON: {0}")] + MalformedRecords(String), + #[error("PDS endpoint {pds:?} isn't a usable base URL")] + BadPdsEndpoint { pds: String }, + #[error("service token {jti:?} has already been presented")] + Replay { jti: JwtNonce }, + #[error("replay-protection store is full and cannot accept another nonce")] + ReplayStoreSaturated, + #[error("issuer {issuer} has too many live replay nonces")] + ReplayShareExhausted { issuer: AccountDid }, + #[error(transparent)] + Identity(#[from] IdentityError), + #[error("plc submission for {did} returned HTTP {status}")] + PlcSubmit { did: RepoDid, status: HttpStatus }, + #[error("putRecord for {subject} returned HTTP {status}")] + PutRecord { + subject: AccountDid, + status: HttpStatus, + }, + #[error("getRecord for {owner} returned HTTP {status}")] + GetRecord { + owner: AccountDid, + status: HttpStatus, + }, + #[error("pointer record couldn't be encoded: {0}")] + PointerEncode(String), + #[error("putRecord response isn't a valid receipt: {0}")] + MalformedReceipt(String), +} + +impl AtprotoError { + pub fn is_transient(&self) -> bool { + match self { + AtprotoError::Network(_) + | AtprotoError::ReplayStoreSaturated + | AtprotoError::ReplayShareExhausted { .. } => true, + AtprotoError::Resolve(error) => error.is_transient(), + AtprotoError::ListRecords { status, .. } + | AtprotoError::PlcSubmit { status, .. } + | AtprotoError::PutRecord { status, .. } + | AtprotoError::GetRecord { status, .. } => status.is_transient(), + _ => false, + } + } +} + +#[derive(Clone)] +enum Resolution { + Found(Identity), + Failed(ResolveError), + Transient(ResolveError), +} + +#[derive(Clone)] +enum HandleResolution { + Bound(AccountDid), + Unbound(ResolveError), + Transient(ResolveError), +} + +#[derive(Clone)] +struct Cached { + resolution: R, + expires_at: UnixMicros, +} + +pub struct Atproto { + http: H, + clock: C, + knot_did: KnotId, + plc_directory: PlcDirectory, + dns: Arc, + identities: MokaFuture>, + handles: MokaFuture>, + seen_jti: Expiring<(AccountDid, JwtNonce), AccountDid, ()>, +} + +impl Atproto { + pub fn new(http: H, clock: C, knot_did: KnotId, plc_directory: PlcDirectory) -> Self { + Self { + http, + clock, + knot_did, + plc_directory, + dns: Arc::new(SystemDns::new()), + identities: MokaFuture::by_count(EntryCount::new(MAX_IDENTITY_CACHE as u64)), + handles: MokaFuture::by_count(EntryCount::new(MAX_IDENTITY_CACHE as u64)), + seen_jti: Expiring::new(Quotas { + per_group: GroupQuota::new(MAX_JTI_PER_ISSUER), + total: TotalQuota::new(MAX_SEEN_JTI), + }), + } + } + + pub fn with_dns(mut self, dns: Arc) -> Self { + self.dns = dns; + self + } + + pub fn now(&self) -> UnixMicros { + self.clock.now_unix_micros() + } + + pub async fn resolve_identity(&self, did: &AccountDid) -> Result { + self.resolve_identity_inner(did).await.map_err(Into::into) + } + + async fn resolve_identity_inner(&self, did: &AccountDid) -> Result { + let now = self.clock.now_unix_micros(); + let filled = self + .identities + .get_or_fill_if( + did.clone(), + |cached: &Cached| cached.expires_at <= now, + self.fill_identity(did, now), + ) + .await; + let fresh = filled.fresh; + match filled.value.resolution { + Resolution::Found(identity) => Ok(identity), + Resolution::Transient(error) => Err(error), + Resolution::Failed(error) if fresh => Err(error), + Resolution::Failed(_) => Err(ResolveError::RecentlyFailed { did: did.clone() }), + } + } + + pub async fn resolve_handle_to_did(&self, handle: &Handle) -> Result { + let now = self.clock.now_unix_micros(); + let filled = self + .handles + .get_or_fill_if( + handle.clone(), + |cached: &Cached| cached.expires_at <= now, + self.fill_handle(handle, now), + ) + .await; + let fresh = filled.fresh; + match filled.value.resolution { + HandleResolution::Bound(did) => Ok(did), + HandleResolution::Transient(error) => Err(error.into()), + HandleResolution::Unbound(error) if fresh => Err(error.into()), + HandleResolution::Unbound(_) => Err(ResolveError::HandleRecentlyFailed { + handle: handle.clone(), + } + .into()), + } + } + + async fn stale_handle_did(&self, handle: &Handle) -> Option { + // `fill_handle` calls this from inside its own + // `or_insert_with_if` init for this key. + // Moka will keep the prior entry readable until init returns, + // so this `get` will yield the last resolved DID to re-serve, + // when there's a temporary outage. + match self.handles.get(handle).await?.resolution { + HandleResolution::Bound(did) => Some(did), + _ => None, + } + } + + async fn fill_handle(&self, handle: &Handle, now: UnixMicros) -> Cached { + match self.verify_handle(handle).await { + Ok(did) => Cached { + resolution: HandleResolution::Bound(did), + expires_at: expires(now, DEFAULT_TTL), + }, + Err(error) if error.is_transient() => match self.stale_handle_did(handle).await { + Some(did) => Cached { + resolution: HandleResolution::Bound(did), + expires_at: expires(now, STALE_TTL), + }, + None => Cached { + resolution: HandleResolution::Transient(error), + expires_at: now, + }, + }, + Err(error) => Cached { + resolution: HandleResolution::Unbound(error), + expires_at: expires(now, NEGATIVE_TTL), + }, + } + } + + async fn verify_handle(&self, handle: &Handle) -> Result { + let candidate = match self.dns_txt_did(handle).await { + Ok(Some(did)) => did, + Ok(None) => self.wellknown_did(handle).await?, + Err(dns_error) if dns_error.is_transient() => match self.wellknown_did(handle).await { + Ok(did) => did, + Err(_) => return Err(dns_error), + }, + Err(dns_error) => return Err(dns_error), + }; + let identity = self.resolve_identity_inner(&candidate).await?; + if identity.claims_handle(handle) { + Ok(candidate) + } else { + Err(ResolveError::HandleMismatch { + handle: handle.clone(), + resolved: candidate, + claimed: identity.primary_handle().cloned(), + }) + } + } + + async fn dns_txt_did(&self, handle: &Handle) -> Result, ResolveError> { + let records = self + .dns + .lookup_txt(format!("_atproto.{}", handle.as_str())) + .await?; + let dids = records + .iter() + .filter_map(|record| record.trim().strip_prefix("did=").map(str::trim)) + .map(|value| { + AccountDid::new(value).map_err(|_| ResolveError::HandleForwardMalformed { + handle: handle.clone(), + value: value.to_string(), + }) + }) + .collect::, ResolveError>>()?; + let distinct = dids + .iter() + .map(AccountDid::as_str) + .collect::>() + .len(); + match distinct { + 0 => Ok(None), + 1 => Ok(dids.into_iter().next()), + _ => Err(ResolveError::HandleAmbiguous { + handle: handle.clone(), + }), + } + } + + async fn wellknown_did(&self, handle: &Handle) -> Result { + let url = Url::parse(&format!( + "https://{}/.well-known/atproto-did", + handle.as_str() + )) + .map_err(|_| ResolveError::HandleUnresolvable { + handle: handle.clone(), + })?; + resolve::guard_fetch_url(&url)?; + let response = self + .http + .execute(HttpRequest::get(url)) + .await + .map_err(ResolveError::from)?; + if !response.status.is_success() { + let status = HttpStatus::from(response.status); + if status.is_transient() { + return Err(ResolveError::Status { status }); + } + return Err(ResolveError::HandleUnresolvable { + handle: handle.clone(), + }); + } + let value = std::str::from_utf8(&response.body) + .map_err(|_| ResolveError::HandleUnresolvable { + handle: handle.clone(), + })? + .trim(); + AccountDid::new(value).map_err(|_| ResolveError::HandleForwardMalformed { + handle: handle.clone(), + value: value.to_string(), + }) + } + + async fn fill_identity(&self, did: &AccountDid, now: UnixMicros) -> Cached { + match self.fetch_identity(did).await { + Ok(identity) => Cached { + resolution: Resolution::Found(identity), + expires_at: expires(now, DEFAULT_TTL), + }, + Err(error) if warrants_negative_cache(&error) => Cached { + resolution: Resolution::Failed(error), + expires_at: expires(now, NEGATIVE_TTL), + }, + Err(error) => Cached { + resolution: Resolution::Transient(error), + expires_at: now, + }, + } + } + + async fn fetch_identity(&self, did: &AccountDid) -> Result { + let url = resolve::document_url(did, &self.plc_directory)?; + resolve::guard_fetch_url(&url)?; + let response = self + .http + .execute(HttpRequest::get(url)) + .await + .map_err(ResolveError::from)?; + if !response.status.is_success() { + return Err(ResolveError::Status { + status: HttpStatus::from(response.status), + }); + } + resolve::identity_from_document(did, &response.body) + } + + pub async fn resolve_pubkeys(&self, did: &AccountDid) -> Result, AtprotoError> { + let identity = self.resolve_identity(did).await?; + let http = &self.http; + let pds = &identity.pds; + let pages = stream::try_unfold(Page::First(PUBKEY_MAX_PAGES), move |state| async move { + let (cursor, budget) = match state { + Page::Done | Page::First(0) | Page::Next(_, 0) => { + return Ok::<_, AtprotoError>(None); + } + Page::First(budget) => (None, budget), + Page::Next(cursor, budget) => (Some(cursor), budget), + }; + let url = list_records_url(pds, did, cursor.as_ref())?; + resolve::guard_fetch_url(&url)?; + let response = http.execute(HttpRequest::get(url)).await?; + if !response.status.is_success() { + return Err(AtprotoError::ListRecords { + did: did.clone(), + status: HttpStatus::from(response.status), + }); + } + let page = pubkeys::offered_page(&response.body, PUBKEY_PAGE_LIMIT as usize) + .map_err(|error| AtprotoError::MalformedRecords(error.to_string()))?; + let next = match page.cursor { + Some(cursor) => Page::Next(cursor, budget - 1), + None => Page::Done, + }; + Ok(Some((page.keys, next))) + }); + pages + .try_fold(Vec::new(), |mut acc, keys| async move { + acc.extend(keys); + Ok(acc) + }) + .await + } + + pub async fn repo_record_present( + &self, + owner: &AccountDid, + rkey: &RepoRkey, + ) -> Result { + let identity = self.resolve_identity(owner).await?; + let url = get_record_url(&identity.pds, owner, &repo_collection(), rkey)?; + resolve::guard_fetch_url(&url)?; + let response = self.http.execute(HttpRequest::get(url)).await?; + match response.status { + status if status.is_success() => Ok(RecordPresence::Present), + StatusCode::NOT_FOUND => Ok(RecordPresence::Absent), + StatusCode::BAD_REQUEST if record_not_found(response.body.as_ref()) => { + Ok(RecordPresence::Absent) + } + status => Err(AtprotoError::GetRecord { + owner: owner.clone(), + status: HttpStatus::from(status), + }), + } + } + + pub async fn verify_service_jwt( + &self, + token: &ServiceJwt, + method: &Nsid, + ) -> Result { + self.verify_service_jwt_guarded(token, method, ReplayGuard::SingleUse) + .await + } + + pub async fn verify_service_jwt_guarded( + &self, + token: &ServiceJwt, + method: &Nsid, + replay: ReplayGuard, + ) -> Result { + let parsed = jwt::parse(token)?; + let issuer = jwt::issuer(&parsed)?; + let now_micros = self.clock.now_unix_micros(); + let now = UnixSeconds::new((now_micros.get() / 1_000_000) as i64); + + jwt::check_claims(&parsed, &self.knot_did, method, now)?; + let jti = jwt::nonce(&parsed)?; + + let identity = self.resolve_identity(&issuer).await?; + jwt::verify_signature(&parsed, &identity.signing_key)?; + + match replay { + ReplayGuard::SingleUse => { + let exp = UnixSeconds::new(parsed.claims().exp); + self.record_jti(&issuer, jti, exp, now_micros)?; + } + ReplayGuard::ReusableUntilExpiry => drop(jti), + } + Ok(issuer) + } + + pub async fn submit_plc_operation( + &self, + prepared: &PreparedRepoDid, + ) -> Result<(), AtprotoError> { + let account = AccountDid::from(prepared.did.clone()); + let url = resolve::document_url(&account, &self.plc_directory)?; + resolve::guard_fetch_url(&url)?; + let request = json_post( + url, + bytes::Bytes::copy_from_slice(prepared.operation_json()), + ); + let response = self.http.execute(request).await?; + if response.status.is_success() { + Ok(()) + } else { + Err(AtprotoError::PlcSubmit { + did: prepared.did.clone(), + status: HttpStatus::from(response.status), + }) + } + } + + pub async fn publish_pointer( + &self, + authorizer: &dyn PointerAuthorizer, + subject: &AccountDid, + rkey: &Rkey, + record: &R, + ) -> Result { + let identity = self.resolve_identity(subject).await?; + let method = pointer::put_record_method(); + let url = xrpc_url(&identity.pds, &method)?; + resolve::guard_fetch_url(&url)?; + let audience = pointer::pds_service_did(&identity.pds)?; + let now = UnixSeconds::new((self.clock.now_unix_micros().get() / 1_000_000) as i64); + let body = pointer::put_record_body(subject, rkey, record)?; + let mut request = json_post(url, bytes::Bytes::from(body)); + authorizer.authorize( + &mut request, + &PointerAuth { + issuer: &self.knot_did, + audience: &audience, + lxm: &method, + now_unix: now, + }, + )?; + let response = self.http.execute(request).await?; + if !response.status.is_success() { + return Err(AtprotoError::PutRecord { + subject: subject.clone(), + status: HttpStatus::from(response.status), + }); + } + pointer::receipt_from_response(&response.body) + } + + pub async fn verify_did_web_publishes_key( + &self, + did: &RepoDid, + expected: &PublicKeyBytes, + ) -> Result<(), AtprotoError> { + let url = resolve::web_document_url_for(did)?; + resolve::guard_fetch_url(&url)?; + let response = self + .http + .execute(HttpRequest::get(url)) + .await + .map_err(ResolveError::from)?; + if !response.status.is_success() { + return Err(ResolveError::Status { + status: HttpStatus::from(response.status), + } + .into()); + } + resolve::document_publishes_key(did, &response.body, expected).map_err(Into::into) + } + + fn record_jti( + &self, + issuer: &AccountDid, + jti: JwtNonce, + exp: UnixSeconds, + now: UnixMicros, + ) -> Result<(), AtprotoError> { + let horizon = exp.saturating_add_secs(jwt::CLOCK_SKEW_SECS).get().max(0) as u64; + let expires_at = UnixMicros::new(horizon.saturating_mul(1_000_000)); + match self.seen_jti.admit( + (issuer.clone(), jti.clone()), + issuer.clone(), + (), + expires_at, + now, + ) { + Ok(Admitted::Inserted) => Ok(()), + Ok(Admitted::Occupied(())) => Err(AtprotoError::Replay { jti }), + Err(Rejected::Total) => Err(AtprotoError::ReplayStoreSaturated), + Err(Rejected::Group) => Err(AtprotoError::ReplayShareExhausted { + issuer: issuer.clone(), + }), + } + } +} + +fn expires(now: UnixMicros, ttl: Duration) -> UnixMicros { + let micros = u64::try_from(ttl.as_micros()).unwrap_or(u64::MAX); + UnixMicros::new(now.get().saturating_add(micros)) +} + +fn warrants_negative_cache(error: &ResolveError) -> bool { + match error { + ResolveError::Status { status } => { + (400..500).contains(&status.get()) && status.get() != 429 + } + ResolveError::Malformed(_) + | ResolveError::IdMismatch { .. } + | ResolveError::BadSigningKey(_) + | ResolveError::BadPds { .. } => true, + _ => false, + } +} + +enum Page { + First(usize), + Next(Cursor, usize), + Done, +} + +fn json_post(url: Url, body: bytes::Bytes) -> HttpRequest { + let mut request = HttpRequest::post(url, body); + request.headers.insert( + http::header::CONTENT_TYPE, + http::HeaderValue::from_static("application/json"), + ); + request +} + +fn xrpc_url(pds: &PdsEndpoint, method: &Nsid) -> Result { + let mut url = pds.url().clone(); + url.path_segments_mut() + .map_err(|_| AtprotoError::BadPdsEndpoint { + pds: pds.url().as_str().to_string(), + })? + .pop_if_empty() + .extend(["xrpc", method.as_str()]); + Ok(url) +} + +fn list_records_url( + pds: &PdsEndpoint, + did: &AccountDid, + cursor: Option<&Cursor>, +) -> Result { + let method: Nsid = + Nsid::new_static("com.atproto.repo.listRecords").expect("literal nsid parses"); + let collection: Nsid = Nsid::new_static("sh.tangled.publicKey").expect("literal nsid parses"); + let mut url = xrpc_url(pds, &method)?; + url.query_pairs_mut() + .append_pair("repo", did.as_str()) + .append_pair("collection", collection.as_str()) + .append_pair("limit", &PUBKEY_PAGE_LIMIT.to_string()); + if let Some(cursor) = cursor { + url.query_pairs_mut().append_pair("cursor", cursor.as_str()); + } + Ok(url) +} + +#[derive(Deserialize)] +struct XrpcErrorBody { + error: String, +} + +fn record_not_found(body: &[u8]) -> bool { + serde_json::from_slice::(body) + .is_ok_and(|parsed| parsed.error == "RecordNotFound") +} + +fn get_record_url( + pds: &PdsEndpoint, + owner: &AccountDid, + collection: &Nsid, + rkey: &RepoRkey, +) -> Result { + let method: Nsid = Nsid::new_static("com.atproto.repo.getRecord").expect("literal nsid parses"); + let mut url = xrpc_url(pds, &method)?; + url.query_pairs_mut() + .append_pair("repo", owner.as_str()) + .append_pair("collection", collection.as_str()) + .append_pair("rkey", rkey.as_str()); + Ok(url) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::*; + use bytes::Bytes; + use futures::StreamExt; + use http::StatusCode; + use knot_runtime::{DnsTxtResolver, FakeDns, FakeHttp, ManualClock, NetworkError}; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + const POINTER_RKEY: &str = "3jzfcijpj2z2a"; + const POINTER_CID: &str = "bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a"; + + fn squid_doc(signing: &k256::ecdsa::SigningKey) -> Bytes { + did_doc(DocSpec { + id: SQUID, + signing, + handle: "nel.pet", + pds: "https://pds.oyster.cafe", + method: MethodKind::Multikey, + }) + } + + fn resolver(dns: impl DnsTxtResolver, http: T) -> Atproto { + Atproto::new(http, clock(), knot_did(KNOT), plc()).with_dns(Arc::new(dns)) + } + + #[tokio::test] + async fn an_identity_is_resolved_from_a_did_document() { + let signing = signer(9); + let http = FakeHttp::new(move |request| { + assert_eq!(request.url.as_str(), "https://plc.directory/did:plc:squid"); + Ok(ok(squid_doc(&signing))) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let identity = atproto.resolve_identity(&did(SQUID)).await.unwrap(); + assert_eq!(identity.pds.url().as_str(), "https://pds.oyster.cafe/"); + assert_eq!(identity.primary_handle().unwrap().as_str(), "nel.pet"); + } + + #[tokio::test] + async fn a_second_resolution_is_served_from_cache() { + let signing = signer(9); + let hits = Arc::new(AtomicUsize::new(0)); + let counter = hits.clone(); + let http = FakeHttp::new(move |_| { + counter.fetch_add(1, Ordering::SeqCst); + Ok(ok(squid_doc(&signing))) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + atproto.resolve_identity(&did(SQUID)).await.unwrap(); + atproto.resolve_identity(&did(SQUID)).await.unwrap(); + assert_eq!(hits.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn an_expired_cache_entry_is_refetched() { + let signing = signer(9); + let hits = Arc::new(AtomicUsize::new(0)); + let counter = hits.clone(); + let http = FakeHttp::new(move |_| { + counter.fetch_add(1, Ordering::SeqCst); + Ok(ok(squid_doc(&signing))) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + atproto.resolve_identity(&did(SQUID)).await.unwrap(); + atproto + .clock + .advance(DEFAULT_TTL + Duration::from_micros(1)); + atproto.resolve_identity(&did(SQUID)).await.unwrap(); + assert_eq!(hits.load(Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn a_handle_resolves_bidirectionally_via_dns_and_is_cached() { + let signing = signer(9); + let dns_hits = Arc::new(AtomicUsize::new(0)); + let counter = dns_hits.clone(); + let dns = FakeDns::new(move |name: &str| { + assert_eq!(name, "_atproto.nel.pet"); + counter.fetch_add(1, Ordering::SeqCst); + Ok(vec!["did=did:plc:squid".to_string()]) + }); + let http = FakeHttp::new(move |request| { + assert_eq!(request.url.as_str(), "https://plc.directory/did:plc:squid"); + let mut doc: serde_json::Value = serde_json::from_slice(&squid_doc(&signing)).unwrap(); + doc["alsoKnownAs"] = serde_json::json!(["at://olaren.dev", "at://nel.pet"]); + Ok(ok(Bytes::from(serde_json::to_vec(&doc).unwrap()))) + }); + let atproto = resolver(dns, http); + let owner = handle("nel.pet"); + assert_eq!( + atproto + .resolve_handle_to_did(&owner) + .await + .unwrap() + .as_str(), + "did:plc:squid", + "a handle listed anywhere in alsoKnownAs must resolve, even when it isn't first" + ); + assert_eq!( + atproto + .resolve_handle_to_did(&owner) + .await + .unwrap() + .as_str(), + "did:plc:squid" + ); + assert_eq!( + dns_hits.load(Ordering::SeqCst), + 1, + "a resolved handle is served from cache" + ); + } + + #[tokio::test] + async fn a_handle_falls_back_to_well_known_when_dns_is_empty_or_transient() { + let well_known = || { + let signing = signer(9); + FakeHttp::new(move |request| match request.url.as_str() { + "https://nel.pet/.well-known/atproto-did" => { + Ok(ok(Bytes::from_static(b"did:plc:squid\n"))) + } + "https://plc.directory/did:plc:squid" => Ok(ok(squid_doc(&signing))), + other => panic!("unexpected url {other}"), + }) + }; + let empty = resolver(FakeDns::new(|_| Ok(Vec::new())), well_known()); + let transient = resolver( + FakeDns::new(|_| Err(NetworkError::Request("dns unreachable".to_string()))), + well_known(), + ); + for atproto in [empty, transient] { + assert_eq!( + atproto + .resolve_handle_to_did(&handle("nel.pet")) + .await + .unwrap() + .as_str(), + "did:plc:squid" + ); + } + } + + #[tokio::test] + async fn a_handle_is_rejected_when_ambiguous_or_disowned() { + let ambiguous = resolver( + FakeDns::new(|_| { + Ok(vec![ + "did=did:plc:squid".to_string(), + "did=did:plc:limpet".to_string(), + ]) + }), + FakeHttp::new(|_| panic!("resolution must stop before any fetch")), + ); + assert!(matches!( + ambiguous + .resolve_handle_to_did(&handle("nel.pet")) + .await + .unwrap_err(), + AtprotoError::Resolve(ResolveError::HandleAmbiguous { .. }) + )); + + let signing = signer(9); + let disowned = resolver( + FakeDns::new(|_| Ok(vec!["did=did:plc:squid".to_string()])), + FakeHttp::new(move |_| Ok(ok(squid_doc(&signing)))), + ); + assert!(matches!( + disowned + .resolve_handle_to_did(&handle("olaren.dev")) + .await + .unwrap_err(), + AtprotoError::Resolve(ResolveError::HandleMismatch { .. }) + )); + } + + #[tokio::test] + async fn handle_failures_cache_by_class() { + let owner = handle("nel.pet"); + + let hits = Arc::new(AtomicUsize::new(0)); + let counter = hits.clone(); + let unresolvable = resolver( + FakeDns::new(move |_| { + counter.fetch_add(1, Ordering::SeqCst); + Ok(Vec::new()) + }), + FakeHttp::new(|_| Ok(status(StatusCode::NOT_FOUND, Bytes::new()))), + ); + assert!(matches!( + unresolvable + .resolve_handle_to_did(&owner) + .await + .unwrap_err(), + AtprotoError::Resolve(ResolveError::HandleUnresolvable { .. }) + )); + assert!(matches!( + unresolvable + .resolve_handle_to_did(&owner) + .await + .unwrap_err(), + AtprotoError::Resolve(ResolveError::HandleRecentlyFailed { .. }) + )); + assert_eq!( + hits.load(Ordering::SeqCst), + 1, + "an unresolvable handle is resolved once then served from the negative cache" + ); + + let hits = Arc::new(AtomicUsize::new(0)); + let counter = hits.clone(); + let transient = resolver( + FakeDns::new(move |_| { + counter.fetch_add(1, Ordering::SeqCst); + Ok(Vec::new()) + }), + FakeHttp::new(|_| Ok(status(StatusCode::SERVICE_UNAVAILABLE, Bytes::new()))), + ); + transient.resolve_handle_to_did(&owner).await.unwrap_err(); + transient.resolve_handle_to_did(&owner).await.unwrap_err(); + assert_eq!( + hits.load(Ordering::SeqCst), + 2, + "a transient well-known status mustn't be negatively cached" + ); + } + + #[tokio::test] + async fn a_transient_outage_serves_the_last_good_did() { + let signing = signer(9); + let outage = Arc::new(AtomicUsize::new(0)); + let switch = outage.clone(); + let dns = FakeDns::new(move |_| match switch.load(Ordering::SeqCst) { + 0 => Ok(vec!["did=did:plc:squid".to_string()]), + _ => Err(NetworkError::Request("dns unreachable".to_string())), + }); + let atproto = resolver(dns, FakeHttp::new(move |_| Ok(ok(squid_doc(&signing))))); + let owner = handle("nel.pet"); + assert_eq!( + atproto + .resolve_handle_to_did(&owner) + .await + .unwrap() + .as_str(), + "did:plc:squid" + ); + + atproto.clock.advance(DEFAULT_TTL + Duration::from_secs(1)); + outage.store(1, Ordering::SeqCst); + assert_eq!( + atproto + .resolve_handle_to_did(&owner) + .await + .unwrap() + .as_str(), + "did:plc:squid", + "a transient outage must serve the last resolved DID" + ); + + atproto.clock.advance(STALE_TTL + Duration::from_secs(1)); + outage.store(0, Ordering::SeqCst); + assert_eq!( + atproto + .resolve_handle_to_did(&owner) + .await + .unwrap() + .as_str(), + "did:plc:squid" + ); + } + + #[tokio::test] + async fn a_404_identity_is_negatively_cached() { + let hits = Arc::new(AtomicUsize::new(0)); + let counter = hits.clone(); + let http = FakeHttp::new(move |_| { + counter.fetch_add(1, Ordering::SeqCst); + Ok(status(StatusCode::NOT_FOUND, Bytes::new())) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let first = atproto.resolve_identity(&did(SQUID)).await.unwrap_err(); + assert!( + matches!(first, AtprotoError::Resolve(ResolveError::Status { status }) if status.get() == 404), + "got {first:?}" + ); + let second = atproto.resolve_identity(&did(SQUID)).await.unwrap_err(); + assert!( + matches!( + second, + AtprotoError::Resolve(ResolveError::RecentlyFailed { .. }) + ), + "got {second:?}" + ); + assert_eq!( + hits.load(Ordering::SeqCst), + 1, + "404 is served from the negative cache" + ); + } + + #[tokio::test] + async fn a_transient_5xx_identity_is_not_cached() { + let hits = Arc::new(AtomicUsize::new(0)); + let counter = hits.clone(); + let http = FakeHttp::new(move |_| { + counter.fetch_add(1, Ordering::SeqCst); + Ok(status(StatusCode::SERVICE_UNAVAILABLE, Bytes::new())) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let _ = atproto.resolve_identity(&did(SQUID)).await.unwrap_err(); + let _ = atproto.resolve_identity(&did(SQUID)).await.unwrap_err(); + assert_eq!( + hits.load(Ordering::SeqCst), + 2, + "a transient 503 is re-fetched instead of negatively cached" + ); + } + + #[tokio::test] + async fn a_document_describing_another_did_is_rejected() { + let doc_key = signer(7); + let http = FakeHttp::new(move |_| { + Ok(ok(did_doc(DocSpec { + id: LIMPET, + signing: &doc_key, + handle: "nel.pet", + pds: "https://pds.oyster.cafe", + method: MethodKind::Multikey, + }))) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let error = atproto.resolve_identity(&did(SQUID)).await.unwrap_err(); + assert!( + matches!( + error, + AtprotoError::Resolve(ResolveError::IdMismatch { .. }) + ), + "got {error:?}" + ); + } + + #[tokio::test] + async fn a_claimed_handle_is_returned_unverified() { + let signing = signer(9); + let http = FakeHttp::new(move |_| { + Ok(ok(did_doc(DocSpec { + id: SQUID, + signing: &signing, + handle: "olaren.dev", + pds: "https://pds.oyster.cafe", + method: MethodKind::Multikey, + }))) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let identity = atproto.resolve_identity(&did(SQUID)).await.unwrap(); + assert_eq!( + identity.primary_handle().unwrap().as_str(), + "olaren.dev", + "alsoKnownAs handle is taken at face value with no bidirectional verification" + ); + } + + #[tokio::test] + async fn an_internal_ip_with_a_port_is_refused() { + let (sink, urls) = recorder(); + let http = FakeHttp::new(move |request| { + sink.lock().unwrap().push(request.url.clone()); + Ok(ok(Bytes::new())) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let error = atproto + .resolve_identity(&did("did:web:169.254.169.254%3A6379")) + .await + .unwrap_err(); + assert!( + matches!( + error, + AtprotoError::Resolve(ResolveError::BlockedHost { .. }) + ), + "got {error:?}" + ); + assert!(urls.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn repo_record_presence_maps_pds_status_to_a_verdict() { + let signing = signer(9); + let http = FakeHttp::new(move |request| { + if request.url.path().ends_with("did.json") + || request.url.host_str() == Some("plc.directory") + { + return Ok(ok(squid_doc(&signing))); + } + assert!(request.url.path().ends_with("com.atproto.repo.getRecord")); + assert!(request.url.query().unwrap().contains("sh.tangled.repo")); + let rkey = request + .url + .query_pairs() + .find(|(key, _)| key == "rkey") + .map(|(_, value)| value.into_owned()) + .unwrap_or_default(); + let (st, body) = match rkey.as_str() { + "present" => (StatusCode::OK, Bytes::new()), + "missing" => ( + StatusCode::BAD_REQUEST, + Bytes::from_static(b"{\"error\":\"RecordNotFound\"}"), + ), + _ => (StatusCode::INTERNAL_SERVER_ERROR, Bytes::new()), + }; + Ok(status(st, body)) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let owner = did(SQUID); + assert_eq!( + atproto + .repo_record_present(&owner, &RepoRkey::new("present").unwrap()) + .await + .unwrap(), + RecordPresence::Present + ); + assert_eq!( + atproto + .repo_record_present(&owner, &RepoRkey::new("missing").unwrap()) + .await + .unwrap(), + RecordPresence::Absent + ); + assert!( + atproto + .repo_record_present(&owner, &RepoRkey::new("boom").unwrap()) + .await + .is_err(), + "5xx from the PDS surfaces as an error so the caller can fall back to best-effort" + ); + } + + #[tokio::test] + async fn pubkeys_are_fetched_from_the_resolved_pds() { + let signing = signer(9); + let line = ssh_line("ssh-ed25519", &[4u8; 32], "nel@oyster.cafe"); + let expected = parse_authorized_key(&line).unwrap(); + let body = list_body(&[line], None); + let http = FakeHttp::new(move |request| { + if request.url.path().ends_with("did.json") + || request.url.host_str() == Some("plc.directory") + { + Ok(ok(squid_doc(&signing))) + } else { + assert_eq!(request.url.host_str(), Some("pds.oyster.cafe")); + assert!(request.url.path().ends_with("com.atproto.repo.listRecords")); + assert!( + request + .url + .query() + .unwrap() + .contains("sh.tangled.publicKey") + ); + Ok(ok(body.clone())) + } + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let keys = atproto.resolve_pubkeys(&did(SQUID)).await.unwrap(); + assert_eq!(keys, vec![expected]); + } + + #[tokio::test] + async fn pubkey_resolution_follows_the_cursor() { + let doc_key = signer(9); + let list_hits = Arc::new(AtomicUsize::new(0)); + let counter = list_hits.clone(); + let page_one = list_body( + &[ssh_line("ssh-ed25519", &[1u8; 32], "one")], + Some("page-2"), + ); + let page_two = list_body(&[ssh_line("ssh-ed25519", &[2u8; 32], "two")], None); + let http = FakeHttp::new(move |request| { + if request.url.host_str() == Some("plc.directory") { + return Ok(ok(squid_doc(&doc_key))); + } + counter.fetch_add(1, Ordering::SeqCst); + let on_second = request.url.query().unwrap().contains("cursor=page-2"); + Ok(ok(if on_second { + page_two.clone() + } else { + page_one.clone() + })) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let keys = atproto.resolve_pubkeys(&did(SQUID)).await.unwrap(); + assert_eq!(keys.len(), 2); + assert_eq!(list_hits.load(Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn a_plain_http_pds_is_refused() { + let doc_key = signer(9); + let (sink, urls) = recorder(); + let http = FakeHttp::new(move |request| { + sink.lock().unwrap().push(request.url.clone()); + if request.url.host_str() == Some("plc.directory") { + Ok(ok(did_doc(DocSpec { + id: SQUID, + signing: &doc_key, + handle: "nel.pet", + pds: "http://127.0.0.1:6379", + method: MethodKind::Multikey, + }))) + } else { + Ok(ok(list_body(&[], None))) + } + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let error = atproto.resolve_pubkeys(&did(SQUID)).await.unwrap_err(); + assert!( + matches!( + error, + AtprotoError::Resolve(ResolveError::InsecureScheme { .. }) + ), + "got {error:?}" + ); + assert_eq!(urls.lock().unwrap().len(), 1); + } + + #[tokio::test] + async fn a_single_page_is_bounded_by_the_pubkey_page_limit() { + let signing = signer(9); + let lines: Vec = (0..3_000u32) + .map(|seed| { + let mut material = [0u8; 32]; + material[..4].copy_from_slice(&seed.to_be_bytes()); + ssh_line("ssh-ed25519", &material, "k") + }) + .collect(); + let page = list_body(&lines, None); + let http = FakeHttp::new(move |request| { + if request.url.host_str() == Some("plc.directory") { + Ok(ok(squid_doc(&signing))) + } else { + Ok(ok(page.clone())) + } + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let keys = atproto.resolve_pubkeys(&did(SQUID)).await.unwrap(); + assert_eq!( + keys.len(), + PUBKEY_PAGE_LIMIT as usize, + "single page yields at most the page limit even when the PDS floods it" + ); + } + + #[tokio::test] + async fn a_service_jwt_authenticates_against_the_resolved_issuer_key() { + let signing = signer(9); + let doc = signing.clone(); + let http = FakeHttp::new(move |_| Ok(ok(squid_doc(&doc)))); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let method = member_method(); + let claims = serde_json::json!({ + "iss": SQUID, "aud": KNOT, "exp": 1_001, "iat": 999, + "jti": "nonce-authenticates", "lxm": METHOD, + }); + let token = mint(&signing, &claims); + let authed = atproto.verify_service_jwt(&token, &method).await.unwrap(); + assert_eq!(authed, did(SQUID)); + } + + #[tokio::test] + async fn a_service_jwt_signed_by_an_impostor_is_rejected() { + let real = signer(9); + let impostor = signer(3); + let http = FakeHttp::new(move |_| Ok(ok(squid_doc(&real)))); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let method = member_method(); + let claims = serde_json::json!({ + "iss": SQUID, "aud": KNOT, "exp": 1_001, "iat": 999, + "jti": "nonce-impostor", "lxm": METHOD, + }); + let token = mint(&impostor, &claims); + let error = atproto + .verify_service_jwt(&token, &method) + .await + .unwrap_err(); + assert!(matches!( + error, + AtprotoError::Jwt(JwtError::InvalidSignature) + )); + } + + #[tokio::test] + async fn a_replayed_token_is_rejected() { + let key = signer(9); + let http = FakeHttp::new(move |_| Ok(ok(squid_doc(&key)))); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let method = member_method(); + let claims = serde_json::json!({ + "iss": SQUID, "aud": KNOT, "exp": 1_001, "iat": 999, + "jti": "single-use-nonce", "lxm": METHOD, + }); + let token = mint(&signer(9), &claims); + assert_eq!( + atproto.verify_service_jwt(&token, &method).await.unwrap(), + did(SQUID) + ); + let replay = atproto + .verify_service_jwt(&token, &method) + .await + .unwrap_err(); + assert!( + matches!(replay, AtprotoError::Replay { .. }), + "got {replay:?}" + ); + } + + #[tokio::test] + async fn two_issuers_may_share_a_nonce() { + let squid_key = signer(1); + let limpet_key = signer(2); + let sq = squid_key.clone(); + let li = limpet_key.clone(); + let http = FakeHttp::new(move |request| { + if request.url.as_str().ends_with(LIMPET) { + Ok(ok(did_doc(DocSpec { + id: LIMPET, + signing: &li, + handle: "nel.pet", + pds: "https://pds.oyster.cafe", + method: MethodKind::Multikey, + }))) + } else { + Ok(ok(squid_doc(&sq))) + } + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let method = member_method(); + let squid = serde_json::json!({ + "iss": SQUID, "aud": KNOT, "exp": 1_001, "iat": 999, + "jti": "shared-nonce", "lxm": METHOD, + }); + let limpet = serde_json::json!({ + "iss": LIMPET, "aud": KNOT, "exp": 1_001, "iat": 999, + "jti": "shared-nonce", "lxm": METHOD, + }); + assert_eq!( + atproto + .verify_service_jwt(&mint(&squid_key, &squid), &method) + .await + .unwrap(), + did(SQUID) + ); + assert_eq!( + atproto + .verify_service_jwt(&mint(&limpet_key, &limpet), &method) + .await + .unwrap(), + did(LIMPET) + ); + } + + #[test] + fn knot_id_lowercases_its_host() { + assert_eq!(knot_did("did:web:NEL.PET").as_str(), "did:web:nel.pet"); + } + + #[test] + fn list_records_url_is_well_formed() { + let url = list_records_url( + &PdsEndpoint::new(Url::parse("https://pds.oyster.cafe").unwrap()).unwrap(), + &did(SQUID), + None, + ) + .unwrap(); + assert_eq!(url.host_str(), Some("pds.oyster.cafe")); + assert_eq!(url.path(), "/xrpc/com.atproto.repo.listRecords"); + let query = url.query().unwrap(); + assert!(query.contains("repo=did%3Aplc%3Asquid")); + assert!(query.contains("collection=sh.tangled.publicKey")); + } + + #[test] + fn list_records_url_preserves_a_pds_base_path() { + let url = list_records_url( + &PdsEndpoint::new(Url::parse("https://shared.host/account-pds").unwrap()).unwrap(), + &did(SQUID), + Some(&Cursor::new("page2")), + ) + .unwrap(); + assert_eq!(url.path(), "/account-pds/xrpc/com.atproto.repo.listRecords"); + assert!(url.query().unwrap().contains("cursor=page2")); + } + + struct JwtCase { + name: &'static str, + header: &'static [u8], + mutate: fn(&mut serde_json::Value), + expect: fn(&Result) -> bool, + zero_network: bool, + } + + const JWT_HEADER: &[u8] = br#"{"alg":"ES256K","typ":"JWT"}"#; + + const JWT_CASES: &[JwtCase] = &[ + JwtCase { + name: "internal-ip issuer is blocked before any fetch", + header: JWT_HEADER, + mutate: |c| c["iss"] = serde_json::json!("did:web:169.254.169.254"), + expect: |r| { + matches!( + r, + Err(AtprotoError::Resolve(ResolveError::BlockedHost { .. })) + ) + }, + zero_network: true, + }, + JwtCase { + name: "decade-long token exceeds the lifetime limit", + header: JWT_HEADER, + mutate: |c| { + c["exp"] = serde_json::json!(1000 + 315_360_000i64); + c["iat"] = serde_json::json!(1000); + }, + expect: |r| matches!(r, Err(AtprotoError::Jwt(JwtError::LifetimeTooLong { .. }))), + zero_network: true, + }, + JwtCase { + name: "audience in a different case is accepted", + header: JWT_HEADER, + mutate: |c| c["aud"] = serde_json::json!("did:web:NEL.PET"), + expect: |r| r.is_ok(), + zero_network: false, + }, + JwtCase { + name: "legacy-typed issuer key verifies", + header: JWT_HEADER, + mutate: |c| c["iss"] = serde_json::json!(LIMPET), + expect: |r| matches!(r, Ok(did) if did.as_str() == LIMPET), + zero_network: false, + }, + JwtCase { + name: "token addressed to another knot is refused", + header: JWT_HEADER, + mutate: |c| c["aud"] = serde_json::json!("did:web:somewhere.else"), + expect: |r| matches!(r, Err(AtprotoError::Jwt(JwtError::AudienceMismatch { .. }))), + zero_network: true, + }, + JwtCase { + name: "stale token is expired before resolution", + header: JWT_HEADER, + mutate: |c| { + c["exp"] = serde_json::json!(1); + c["iat"] = serde_json::json!(0); + }, + expect: |r| matches!(r, Err(AtprotoError::Jwt(JwtError::Expired { .. }))), + zero_network: true, + }, + JwtCase { + name: "nonceless token is refused before resolution", + header: JWT_HEADER, + mutate: |c| { + c.as_object_mut().unwrap().remove("jti"); + }, + expect: |r| matches!(r, Err(AtprotoError::Jwt(JwtError::MissingNonce))), + zero_network: true, + }, + JwtCase { + name: "alg none is refused", + header: br#"{"alg":"none","typ":"JWT"}"#, + mutate: |_| {}, + expect: |r| matches!(r, Err(AtprotoError::Jwt(JwtError::Parse(_)))), + zero_network: false, + }, + JwtCase { + name: "es256 header against a k256 doc key is refused", + header: br#"{"alg":"ES256","typ":"JWT"}"#, + mutate: |_| {}, + expect: |r| matches!(r, Err(AtprotoError::Jwt(JwtError::Parse(_)))), + zero_network: false, + }, + JwtCase { + name: "token type that isn't JWT is refused", + header: br#"{"alg":"ES256K","typ":"secevent+jwt"}"#, + mutate: |_| {}, + expect: |r| matches!(r, Err(AtprotoError::Jwt(JwtError::UnexpectedType { .. }))), + zero_network: true, + }, + JwtCase { + name: "method mismatch is refused before resolution", + header: JWT_HEADER, + mutate: |c| c["lxm"] = serde_json::json!("sh.tangled.repo.delete"), + expect: |r| matches!(r, Err(AtprotoError::Jwt(JwtError::MethodMismatch { .. }))), + zero_network: true, + }, + JwtCase { + name: "oversized nonce is refused before resolution", + header: JWT_HEADER, + mutate: |c| c["jti"] = serde_json::json!("n".repeat(100_000)), + expect: |r| matches!(r, Err(AtprotoError::Jwt(JwtError::OversizedNonce { .. }))), + zero_network: true, + }, + ]; + + #[tokio::test] + async fn verify_service_jwt_rejects_every_malformed_or_adversarial_token() { + let method = member_method(); + stream::iter(JWT_CASES) + .for_each(|case| { + let method = &method; + async move { + let doc_key = signer(1); + let calls = Arc::new(AtomicUsize::new(0)); + let counter = calls.clone(); + let http = FakeHttp::new(move |request| { + counter.fetch_add(1, Ordering::SeqCst); + if request.url.as_str().ends_with(LIMPET) { + Ok(ok(did_doc(DocSpec { + id: LIMPET, + signing: &doc_key, + handle: "nel.pet", + pds: "https://pds.oyster.cafe", + method: MethodKind::LegacyK256, + }))) + } else { + Ok(ok(squid_doc(&doc_key))) + } + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let mut claims = serde_json::json!({ + "iss": SQUID, "aud": KNOT, "exp": 1_001, "iat": 999, + "jti": "nonce-3a", "lxm": METHOD, + }); + (case.mutate)(&mut claims); + let token = mint_with_header(&signer(1), case.header, &claims); + let result = atproto.verify_service_jwt(&token, method).await; + assert!( + (case.expect)(&result), + "case {:?} got {result:?}", + case.name + ); + if case.zero_network { + assert_eq!( + calls.load(Ordering::SeqCst), + 0, + "case {:?} must be refused before any network resolution", + case.name + ); + } + } + }) + .await; + } + + #[tokio::test] + async fn the_jti_replay_store_is_bounded_and_fails_closed_when_saturated() { + let signing = signer(9); + let doc = signing.clone(); + let http = FakeHttp::new(move |request| { + let host = request.url.host_str().unwrap().to_string(); + let id = format!("did:web:{host}"); + Ok(ok(did_doc(DocSpec { + id: &id, + signing: &doc, + handle: "nel.pet", + pds: "https://pds.oyster.cafe", + method: MethodKind::Multikey, + }))) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let method = member_method(); + let issuers = MAX_SEEN_JTI / MAX_JTI_PER_ISSUER; + stream::iter( + (0..issuers).flat_map(|issuer| (0..MAX_JTI_PER_ISSUER).map(move |i| (issuer, i))), + ) + .for_each(|(issuer, i)| { + let atproto = &atproto; + let method = &method; + let signing = &signing; + async move { + let claims = serde_json::json!({ + "iss": format!("did:web:i{issuer}.oyster.cafe"), "aud": KNOT, + "exp": 1_001, "iat": 999, + "jti": format!("nonce-{issuer}-{i}"), "lxm": METHOD, + }); + atproto + .verify_service_jwt(&mint(signing, &claims), method) + .await + .unwrap(); + } + }) + .await; + assert_eq!(atproto.seen_jti.len(), MAX_SEEN_JTI); + let overflow = serde_json::json!({ + "iss": "did:web:fresh.oyster.cafe", "aud": KNOT, "exp": 1_001, "iat": 999, + "jti": "nonce-overflow", "lxm": METHOD, + }); + let error = atproto + .verify_service_jwt(&mint(&signing, &overflow), &method) + .await + .unwrap_err(); + assert!( + matches!(error, AtprotoError::ReplayStoreSaturated), + "every verify past the global limit fails closed, got {error:?}" + ); + assert!( + atproto.seen_jti.len() <= MAX_SEEN_JTI, + "replay store must stay bounded, held {}", + atproto.seen_jti.len() + ); + } + + #[tokio::test] + async fn one_issuer_cannot_hog_the_replay_store() { + let signing = signer(9); + let doc = signing.clone(); + let http = FakeHttp::new(move |request| { + let host = request.url.host_str().unwrap().to_string(); + if host == "plc.directory" { + Ok(ok(squid_doc(&doc))) + } else { + let id = format!("did:web:{host}"); + Ok(ok(did_doc(DocSpec { + id: &id, + signing: &doc, + handle: "nel.pet", + pds: "https://pds.oyster.cafe", + method: MethodKind::Multikey, + }))) + } + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let method = member_method(); + stream::iter(0..MAX_JTI_PER_ISSUER) + .for_each(|i| { + let atproto = &atproto; + let method = &method; + let signing = &signing; + async move { + let claims = serde_json::json!({ + "iss": SQUID, "aud": KNOT, "exp": 1_001, "iat": 999, + "jti": format!("nonce-{i}"), "lxm": METHOD, + }); + atproto + .verify_service_jwt(&mint(signing, &claims), method) + .await + .unwrap(); + } + }) + .await; + + let hogged = serde_json::json!({ + "iss": SQUID, "aud": KNOT, "exp": 1_001, "iat": 999, + "jti": "nonce-over-budget", "lxm": METHOD, + }); + let error = atproto + .verify_service_jwt(&mint(&signing, &hogged), &method) + .await + .unwrap_err(); + assert!( + matches!(error, AtprotoError::ReplayShareExhausted { .. }), + "issuer past its share fails closed, got {error:?}" + ); + + let bystander = serde_json::json!({ + "iss": "did:web:bystander.oyster.cafe", "aud": KNOT, "exp": 1_001, "iat": 999, + "jti": "nonce-bystander", "lxm": METHOD, + }); + atproto + .verify_service_jwt(&mint(&signing, &bystander), &method) + .await + .expect("unrelated issuer is unaffected by the hog"); + + atproto.clock.advance(Duration::from_secs(62)); + let after_expiry = serde_json::json!({ + "iss": SQUID, "aud": KNOT, "exp": 1_100, "iat": 1_050, + "jti": "nonce-after-expiry", "lxm": METHOD, + }); + atproto + .verify_service_jwt(&mint(&signing, &after_expiry), &method) + .await + .expect("hog recovers once its nonces expire from the store"); + } + + #[tokio::test] + async fn the_identity_cache_stays_bounded_and_retains_hot_entries() { + let signing = signer(9); + let hot_hits = Arc::new(AtomicUsize::new(0)); + let sentinel_hits = Arc::new(AtomicUsize::new(0)); + let hot = hot_hits.clone(); + let sentinel = sentinel_hits.clone(); + let http = FakeHttp::new(move |request| { + let host = request.url.host_str().unwrap().to_string(); + if host == "hot.oyster.cafe" { + hot.fetch_add(1, Ordering::SeqCst); + } + // yeah so what + if host == "sentinel.oyster.cafe" { + sentinel.fetch_add(1, Ordering::SeqCst); + } + let id = format!("did:web:{host}"); + Ok(ok(did_doc(DocSpec { + id: &id, + signing: &signing, + handle: "nel.pet", + pds: "https://pds.oyster.cafe", + method: MethodKind::Multikey, + }))) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let hot_did = did("did:web:hot.oyster.cafe"); + stream::iter(0..64) + .for_each(|_| { + let atproto = &atproto; + let hot_did = hot_did.clone(); + async move { + atproto.resolve_identity(&hot_did).await.unwrap(); + } + }) + .await; + stream::iter(0..MAX_IDENTITY_CACHE * 2) + .for_each(|i| { + let atproto = &atproto; + let hot_did = hot_did.clone(); + async move { + let filler = AccountDid::new(format!("did:web:c{i}.oyster.cafe")).unwrap(); + atproto.resolve_identity(&filler).await.unwrap(); + if i.is_multiple_of(8) { + atproto.resolve_identity(&hot_did).await.unwrap(); + } + } + }) + .await; + atproto.identities.run_pending_tasks().await; + assert!( + atproto.identities.entry_count().get() <= MAX_IDENTITY_CACHE as u64, + "cache stays bounded under a cold-DID flood" + ); + let before = hot_hits.load(Ordering::SeqCst); + atproto.resolve_identity(&hot_did).await.unwrap(); + assert_eq!( + hot_hits.load(Ordering::SeqCst), + before, + "frequently resolved identity is retained through the flood" + ); + let novel = did("did:web:sentinel.oyster.cafe"); + atproto.resolve_identity(&novel).await.unwrap(); + atproto.resolve_identity(&novel).await.unwrap(); + assert_eq!( + sentinel_hits.load(Ordering::SeqCst), + 1, + "saturated cache still admits a new entry and reuses it without a re-fetch" + ); + atproto.identities.run_pending_tasks().await; + assert!( + atproto.identities.entry_count().get() <= MAX_IDENTITY_CACHE as u64, + "cache stays bounded after eviction" + ); + } + + struct GatedHttp { + status: StatusCode, + body: Bytes, + calls: Arc, + gate: Arc, + } + + impl HttpTransport for GatedHttp { + fn execute(&self, _request: HttpRequest) -> knot_runtime::HttpFuture { + self.calls.fetch_add(1, Ordering::SeqCst); + let gate = self.gate.clone(); + let response = status(self.status, self.body.clone()); + Box::pin(async move { + gate.notified().await; + Ok(response) + }) + } + } + + async fn gated_wave( + st: StatusCode, + body: Bytes, + ) -> (Vec>, usize) { + let calls = Arc::new(AtomicUsize::new(0)); + let gate = Arc::new(tokio::sync::Notify::new()); + let http = GatedHttp { + status: st, + body, + calls: calls.clone(), + gate: gate.clone(), + }; + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let subject = did(SQUID); + let resolves = + futures::future::join_all((0..2000).map(|_| atproto.resolve_identity(&subject))); + let releaser = async { + stream::iter(0..16) + .for_each(|_| async { + tokio::task::yield_now().await; + }) + .await; + gate.notify_one(); + }; + let (results, ()) = futures::join!(resolves, releaser); + (results, calls.load(Ordering::SeqCst)) + } + + #[tokio::test] + async fn concurrent_cold_resolves_coalesce_into_one_fetch() { + let (results, calls) = gated_wave(StatusCode::OK, squid_doc(&signer(9))).await; + assert_eq!(results.len(), 2000); + assert!(results.iter().all(|outcome| outcome.is_ok())); + assert_eq!( + calls, 1, + "2000 concurrent resolves of one cold DID issue exactly one outbound fetch" + ); + } + + #[tokio::test] + async fn a_concurrent_429_wave_gets_the_real_error_and_never_poisons_the_cache() { + let (results, calls) = gated_wave(StatusCode::TOO_MANY_REQUESTS, Bytes::new()).await; + assert_eq!(calls, 1, "failing wave coalesces into one outbound fetch"); + assert!( + results.iter().all(|outcome| matches!( + outcome, + Err(AtprotoError::Resolve(ResolveError::Status { status })) if status.get() == 429 + )), + "every caller in the wave receives a real 429, never a poisoned RecentlyFailed" + ); + } + + #[tokio::test] + async fn a_coalesced_404_wave_gives_one_caller_the_real_error_and_masks_the_rest() { + let (results, calls) = gated_wave(StatusCode::NOT_FOUND, Bytes::new()).await; + assert_eq!(calls, 1, "404 wave coalesces into one outbound fetch"); + let real = results + .iter() + .filter(|outcome| { + matches!( + outcome, + Err(AtprotoError::Resolve(ResolveError::Status { status })) if status.get() == 404 + ) + }) + .count(); + let masked = results + .iter() + .filter(|outcome| { + matches!( + outcome, + Err(AtprotoError::Resolve(ResolveError::RecentlyFailed { .. })) + ) + }) + .count(); + assert_eq!(real, 1, "exactly one caller observes the real 404"); + assert_eq!( + masked, 1999, + "the rest of the coalesced wave is masked as RecentlyFailed" + ); + } + + #[tokio::test] + async fn a_prepared_plc_operation_is_posted_and_a_rejection_is_typed() { + let prepared = prepare_repo_did( + &runtime_signer(11), + &knot_types::KnotServiceUrl::new("https://knot.nel.pet").unwrap(), + &repo_nonce(111), + ) + .unwrap(); + let expected_path = format!("/{}", prepared.did.as_str()); + let http = FakeHttp::new(move |request| { + assert_eq!(request.method, http::Method::POST); + assert_eq!(request.url.path(), expected_path); + assert!(request.body.as_ref().is_some_and(|body| !body.is_empty())); + Ok(ok(Bytes::new())) + }); + Atproto::new(http, clock(), knot_did(KNOT), plc()) + .submit_plc_operation(&prepared) + .await + .unwrap(); + + let rejected = prepare_repo_did( + &runtime_signer(12), + &knot_types::KnotServiceUrl::new("https://knot.nel.pet").unwrap(), + &repo_nonce(112), + ) + .unwrap(); + let http = FakeHttp::new(move |_| Ok(status(StatusCode::BAD_REQUEST, Bytes::new()))); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + assert!(matches!( + atproto.submit_plc_operation(&rejected).await, + Err(AtprotoError::PlcSubmit { status, .. }) if status.get() == 400 + )); + } + + #[tokio::test] + async fn a_did_web_document_verification_covers_present_absent_and_missing() { + let signing = signer(9); + let doc = signing.clone(); + let http = FakeHttp::new(move |request| { + assert_eq!( + request.url.as_str(), + "https://limpet.olaren.dev/.well-known/did.json" + ); + Ok(ok(did_doc(DocSpec { + id: "did:web:limpet.olaren.dev", + signing: &doc, + handle: "nel.pet", + pds: "https://pds.oyster.cafe", + method: MethodKind::Multikey, + }))) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + atproto + .verify_did_web_publishes_key( + &repo_did("did:web:limpet.olaren.dev"), + &PublicKeyBytes::from_bytes(sec1(&signing)), + ) + .await + .unwrap(); + + let published = signer(9); + let http = FakeHttp::new(move |_| { + Ok(ok(did_doc(DocSpec { + id: "did:web:limpet.olaren.dev", + signing: &published, + handle: "nel.pet", + pds: "https://pds.oyster.cafe", + method: MethodKind::Multikey, + }))) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let error = atproto + .verify_did_web_publishes_key( + &repo_did("did:web:limpet.olaren.dev"), + &PublicKeyBytes::from_bytes(sec1(&signer(3))), + ) + .await + .unwrap_err(); + assert!(matches!( + error, + AtprotoError::Resolve(ResolveError::ExpectedKeyAbsent { .. }) + )); + + let http = FakeHttp::new(|_| Ok(status(StatusCode::NOT_FOUND, Bytes::new()))); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let error = atproto + .verify_did_web_publishes_key( + &repo_did("did:web:limpet.olaren.dev"), + &PublicKeyBytes::from_bytes(vec![1, 2, 3]), + ) + .await + .unwrap_err(); + assert!(matches!( + error, + AtprotoError::Resolve(ResolveError::Status { status }) if status.get() == 404 + )); + } + + #[tokio::test] + async fn a_pointer_record_is_published_to_the_subjects_pds_over_service_auth() { + let doc_signer = signer(9); + let knot_key = runtime_signer(21); + let knot_public = knot_runtime::Signer::public_key(&knot_key); + let http = FakeHttp::new(move |request| { + if request.url.host_str() == Some("plc.directory") { + return Ok(ok(squid_doc(&doc_signer))); + } + assert_eq!(request.method, http::Method::POST); + assert_eq!( + request.url.as_str(), + "https://pds.oyster.cafe/xrpc/com.atproto.repo.putRecord" + ); + let bearer = request + .headers + .get(http::header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.strip_prefix("Bearer ")) + .expect("request includes a bearer service token"); + let parsed = knot_types::service_auth::parse_jwt(bearer).unwrap(); + assert_eq!(parsed.claims().iss.as_str(), KNOT); + assert_eq!(parsed.claims().aud.as_str(), "did:web:pds.oyster.cafe"); + assert_eq!( + parsed.claims().lxm.as_ref().unwrap().as_str(), + "com.atproto.repo.putRecord" + ); + assert!(parsed.claims().jti.is_some()); + let key = knot_types::service_auth::PublicKey::from_k256_bytes(knot_public.as_bytes()) + .unwrap(); + knot_types::service_auth::verify_signature(&parsed, &key) + .expect("token is signed by the knot key"); + let body: serde_json::Value = + serde_json::from_slice(request.body.as_ref().unwrap()).unwrap(); + assert_eq!(body["repo"], SQUID); + assert_eq!(body["collection"], "sh.tangled.knot.member"); + assert_eq!(body["rkey"], POINTER_RKEY); + assert_eq!(body["record"]["$type"], "sh.tangled.knot.member"); + assert_eq!(body["record"]["subject"], "did:plc:lyna"); + assert_eq!(body["record"]["domain"], "knot.nel.pet"); + let receipt = serde_json::json!({ + "uri": format!("at://{SQUID}/sh.tangled.knot.member/{POINTER_RKEY}"), + "cid": POINTER_CID, + }); + Ok(ok(Bytes::from(serde_json::to_vec(&receipt).unwrap()))) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let rkey = Rkey::new_owned(POINTER_RKEY).unwrap(); + let receipt = atproto + .publish_pointer( + &ServiceAuth::new(&knot_key, &entropy(31)), + &did(SQUID), + &rkey, + &member_pointer(), + ) + .await + .unwrap(); + assert_eq!( + receipt.uri.as_str(), + format!("at://{SQUID}/sh.tangled.knot.member/{POINTER_RKEY}") + ); + assert_eq!(receipt.cid.as_str(), POINTER_CID); + } + + async fn rejected_put_record(st: StatusCode) -> AtprotoError { + let doc_signer = signer(9); + let http = FakeHttp::new(move |request| { + if request.url.host_str() == Some("plc.directory") { + return Ok(ok(squid_doc(&doc_signer))); + } + Ok(status(st, Bytes::new())) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let rkey = Rkey::new_owned(POINTER_RKEY).unwrap(); + atproto + .publish_pointer( + &ServiceAuth::new(&runtime_signer(22), &entropy(31)), + &did(SQUID), + &rkey, + &member_pointer(), + ) + .await + .unwrap_err() + } + + #[tokio::test] + async fn a_rejected_put_record_is_a_typed_error_and_transient_only_on_server_failure() { + let server_failure = rejected_put_record(StatusCode::BAD_GATEWAY).await; + assert!(matches!( + server_failure, + AtprotoError::PutRecord { status, .. } if status.get() == 502 + )); + assert!(server_failure.is_transient()); + + let rate_limited = rejected_put_record(StatusCode::TOO_MANY_REQUESTS).await; + assert!(matches!( + rate_limited, + AtprotoError::PutRecord { status, .. } if status.get() == 429 + )); + assert!(rate_limited.is_transient()); + + let client_failure = rejected_put_record(StatusCode::BAD_REQUEST).await; + assert!(matches!( + client_failure, + AtprotoError::PutRecord { status, .. } if status.get() == 400 + )); + assert!(!client_failure.is_transient()); + } + + #[tokio::test] + async fn a_malformed_put_record_receipt_is_a_typed_error() { + let doc_signer = signer(9); + let http = FakeHttp::new(move |request| { + if request.url.host_str() == Some("plc.directory") { + return Ok(ok(squid_doc(&doc_signer))); + } + Ok(ok(Bytes::from_static(b"not json"))) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let rkey = Rkey::new_owned(POINTER_RKEY).unwrap(); + let error = atproto + .publish_pointer( + &ServiceAuth::new(&runtime_signer(23), &entropy(31)), + &did(SQUID), + &rkey, + &member_pointer(), + ) + .await + .unwrap_err(); + assert!(matches!(error, AtprotoError::MalformedReceipt(_))); + } + + #[tokio::test] + async fn a_pointer_to_an_insecure_pds_endpoint_is_refused() { + let doc_signer = signer(9); + let http = FakeHttp::new(move |request| { + assert_eq!( + request.url.host_str(), + Some("plc.directory"), + "no request may reach the insecure PDS" + ); + Ok(ok(did_doc(DocSpec { + id: SQUID, + signing: &doc_signer, + handle: "nel.pet", + pds: "http://pds.oyster.cafe", + method: MethodKind::Multikey, + }))) + }); + let atproto = Atproto::new(http, clock(), knot_did(KNOT), plc()); + let rkey = Rkey::new_owned(POINTER_RKEY).unwrap(); + let error = atproto + .publish_pointer( + &ServiceAuth::new(&runtime_signer(24), &entropy(31)), + &did(SQUID), + &rkey, + &member_pointer(), + ) + .await + .unwrap_err(); + assert!(matches!( + error, + AtprotoError::Resolve(ResolveError::InsecureScheme { .. }) + )); + } +} diff --git a/knot2/crates/knot-atproto/src/pointer.rs b/knot2/crates/knot-atproto/src/pointer.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-atproto/src/pointer.rs @@ -0,0 +1,166 @@ +use knot_types::{AccountDid, AtUri, Cid, Collection, Nsid, Rkey, ServiceDid}; +use serde::{Deserialize, Serialize}; + +use crate::AtprotoError; +use crate::resolve::PdsEndpoint; + +pub fn put_record_method() -> Nsid { + Nsid::new_static("com.atproto.repo.putRecord").expect("literal method nsid parses") +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PointerReceipt { + pub uri: AtUri, + pub cid: Cid, +} + +pub(crate) fn pds_service_did(pds: &PdsEndpoint) -> Result { + let bad = || AtprotoError::BadPdsEndpoint { + pds: pds.url().as_str().to_string(), + }; + let host = pds + .url() + .host_str() + .filter(|host| !host.is_empty()) + .ok_or_else(bad)?; + let authority = pds + .url() + .port() + .map_or_else(|| host.to_string(), |port| format!("{host}%3A{port}")); + let msid = std::iter::once(authority) + .chain( + pds.url() + .path_segments() + .into_iter() + .flatten() + .filter(|segment| !segment.is_empty()) + .map(str::to_string), + ) + .collect::>() + .join(":"); + ServiceDid::new(format!("did:web:{msid}")).map_err(|_| bad()) +} + +#[derive(Serialize)] +struct PutRecordInput<'a, R: Serialize> { + repo: &'a AccountDid, + collection: &'static str, + rkey: &'a Rkey, + record: &'a R, +} + +pub(crate) fn put_record_body( + subject: &AccountDid, + rkey: &Rkey, + record: &R, +) -> Result, AtprotoError> { + serde_json::to_vec(&PutRecordInput { + repo: subject, + collection: R::NSID, + rkey, + record, + }) + .map_err(|error| AtprotoError::PointerEncode(error.to_string())) +} + +#[derive(Deserialize)] +struct PutRecordOutput { + uri: String, + cid: String, +} + +pub(crate) fn receipt_from_response(body: &[u8]) -> Result { + let output: PutRecordOutput = serde_json::from_slice(body) + .map_err(|error| AtprotoError::MalformedReceipt(error.to_string()))?; + let uri = AtUri::new_owned(&output.uri) + .map_err(|error| AtprotoError::MalformedReceipt(error.to_string()))?; + let cid = Cid::new_owned(output.cid.as_bytes()) + .map_err(|error| AtprotoError::MalformedReceipt(error.to_string())) + .and_then(|cid: Cid| { + cid.is_valid().then_some(cid).ok_or_else(|| { + AtprotoError::MalformedReceipt(format!("cid {:?} doesn't parse", output.cid)) + }) + })?; + Ok(PointerReceipt { uri, cid }) +} + +#[cfg(test)] +mod tests { + use super::*; + use url::Url; + + fn service_did_of(value: &str) -> Result { + PdsEndpoint::new(Url::parse(value).unwrap()) + .map_err(|_| AtprotoError::BadPdsEndpoint { + pds: value.to_string(), + }) + .and_then(|pds| pds_service_did(&pds)) + } + + struct PdsCase { + url: &'static str, + expect: fn(&Result) -> bool, + } + + const PDS_CASES: &[PdsCase] = &[ + PdsCase { + url: "https://pds.oyster.cafe", + expect: |r| matches!(r, Ok(did) if did.as_str() == "did:web:pds.oyster.cafe"), + }, + PdsCase { + url: "https://pds.oyster.cafe:8443", + expect: |r| matches!(r, Ok(did) if did.as_str() == "did:web:pds.oyster.cafe%3A8443"), + }, + PdsCase { + url: "https://shared.host/account-pds", + expect: |r| matches!(r, Ok(did) if did.as_str() == "did:web:shared.host:account-pds"), + }, + PdsCase { + url: "unix:/run/pds.sock", + expect: |r| matches!(r, Err(AtprotoError::BadPdsEndpoint { .. })), + }, + ]; + + #[test] + fn pds_service_did_maps_each_endpoint_shape_to_its_web_did() { + PDS_CASES.iter().for_each(|case| { + let result = service_did_of(case.url); + assert!((case.expect)(&result), "case {:?} got {result:?}", case.url); + }); + } + + #[test] + fn a_receipt_round_trips_its_uri_and_cid() { + let body = serde_json::json!({ + "uri": "at://did:plc:squid/sh.tangled.knot.member/3jzfcijpj2z2a", + "cid": "bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a" + }); + let receipt = receipt_from_response(&serde_json::to_vec(&body).unwrap()).unwrap(); + assert_eq!( + receipt.uri.as_str(), + "at://did:plc:squid/sh.tangled.knot.member/3jzfcijpj2z2a" + ); + assert_eq!( + receipt.cid.as_str(), + "bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a" + ); + } + + #[test] + fn a_garbage_receipt_is_a_typed_error() { + assert!(matches!( + receipt_from_response(b"not json"), + Err(AtprotoError::MalformedReceipt(_)) + )); + let bad_uri = serde_json::json!({ "uri": "http://nope", "cid": "bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a" }); + assert!(matches!( + receipt_from_response(&serde_json::to_vec(&bad_uri).unwrap()), + Err(AtprotoError::MalformedReceipt(_)) + )); + let bad_cid = serde_json::json!({ "uri": "at://did:plc:squid/sh.tangled.knot.member/3jzfcijpj2z2a", "cid": "not-a-cid" }); + assert!(matches!( + receipt_from_response(&serde_json::to_vec(&bad_cid).unwrap()), + Err(AtprotoError::MalformedReceipt(_)) + )); + } +} diff --git a/knot2/crates/knot-atproto/src/pubkeys.rs b/knot2/crates/knot-atproto/src/pubkeys.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-atproto/src/pubkeys.rs @@ -0,0 +1,231 @@ +use base64::Engine; +use base64::engine::general_purpose::STANDARD; +use knot_types::OfferedKey; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(transparent)] +pub(crate) struct Cursor(String); + +impl Cursor { + #[cfg(test)] + pub(crate) fn new(value: impl Into) -> Self { + Self(value.into()) + } + + pub(crate) fn as_str(&self) -> &str { + &self.0 + } +} + +#[derive(Debug, thiserror::Error)] +pub enum KeyParseError { + #[error("public key line is missing algorithm and blob")] + Incomplete, + #[error("public key blob isn't valid base64: {0}")] + Base64(String), + #[error("public key blob is truncated")] + Truncated, + #[error("declared algorithm {declared:?} doesn't match blob's {embedded:?}")] + AlgorithmMismatch { declared: String, embedded: String }, +} + +pub fn parse_authorized_key(line: &str) -> Result { + let parts: Vec<&str> = line.split_whitespace().take(2).collect(); + let [algo, blob_b64] = parts.as_slice() else { + return Err(KeyParseError::Incomplete); + }; + let blob = STANDARD + .decode(blob_b64) + .map_err(|error| KeyParseError::Base64(error.to_string()))?; + let embedded = embedded_algorithm(&blob)?; + if embedded != algo.as_bytes() { + return Err(KeyParseError::AlgorithmMismatch { + declared: (*algo).to_string(), + embedded: String::from_utf8_lossy(embedded).into_owned(), + }); + } + Ok(OfferedKey::from_bytes(blob)) +} + +fn embedded_algorithm(blob: &[u8]) -> Result<&[u8], KeyParseError> { + let length = blob + .get(..4) + .map(|head| u32::from_be_bytes(head.try_into().expect("four bytes")) as usize) + .ok_or(KeyParseError::Truncated)?; + let end = length.checked_add(4).ok_or(KeyParseError::Truncated)?; + blob.get(4..end).ok_or(KeyParseError::Truncated) +} + +#[derive(Deserialize)] +struct ListRecords { + records: Vec, + #[serde(default)] + cursor: Option, +} + +#[derive(Deserialize)] +struct Envelope { + value: KeyRecord, +} + +#[derive(Deserialize)] +struct KeyRecord { + key: String, +} + +pub(crate) struct PubkeyPage { + pub keys: Vec, + pub cursor: Option, +} + +pub(crate) fn offered_page(body: &[u8], max_keys: usize) -> Result { + let listing: ListRecords = serde_json::from_slice(body)?; + let keys = listing + .records + .iter() + .take(max_keys) + .filter_map(|envelope| parse_authorized_key(&envelope.value.key).ok()) + .collect(); + Ok(PubkeyPage { + keys, + cursor: listing.cursor.filter(|cursor| !cursor.as_str().is_empty()), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::*; + + fn ssh_string(bytes: &[u8]) -> Vec { + [&(bytes.len() as u32).to_be_bytes()[..], bytes].concat() + } + + fn wire_blob(material: &[u8]) -> OfferedKey { + OfferedKey::from_bytes([ssh_string(b"ssh-ed25519"), ssh_string(material)].concat()) + } + + struct KeyCase { + name: &'static str, + line: fn() -> String, + expect: fn(&Result) -> bool, + } + + const KEY_CASES: &[KeyCase] = &[ + KeyCase { + name: "genuine ed25519 key with a trailing comment", + line: || ssh_line("ssh-ed25519", &[7u8; 32], "nel@oyster.cafe"), + expect: |r| matches!(r, Ok(key) if *key == wire_blob(&[7u8; 32])), + }, + KeyCase { + name: "the same key material with no comment", + line: || ssh_line("ssh-ed25519", &[7u8; 32], ""), + expect: |r| matches!(r, Ok(key) if *key == wire_blob(&[7u8; 32])), + }, + KeyCase { + name: "bare algorithm with no blob", + line: || "ssh-ed25519".to_string(), + expect: |r| matches!(r, Err(KeyParseError::Incomplete)), + }, + KeyCase { + name: "whitespace-only line", + line: || " ".to_string(), + expect: |r| matches!(r, Err(KeyParseError::Incomplete)), + }, + KeyCase { + name: "blob that isn't base64", + line: || "ssh-ed25519 not-base64!!!".to_string(), + expect: |r| matches!(r, Err(KeyParseError::Base64(_))), + }, + KeyCase { + name: "declared algorithm lying about the blob", + line: || { + let blob = [ssh_string(b"ssh-ed25519"), ssh_string(&[1u8; 32])].concat(); + format!("ssh-rsa {}", STANDARD.encode(blob)) + }, + expect: |r| matches!(r, Err(KeyParseError::AlgorithmMismatch { .. })), + }, + KeyCase { + name: "authorized_keys options prefix", + line: || { + let blob = [ssh_string(b"ssh-ed25519"), ssh_string(&[7u8; 32])].concat(); + format!( + "command=\"true\",no-pty ssh-ed25519 {} nel@oyster.cafe", + STANDARD.encode(blob) + ) + }, + expect: |r| r.is_err(), + }, + KeyCase { + name: "overlong length prefix", + line: || { + let lying = [&u32::MAX.to_be_bytes()[..], b"short"].concat(); + format!("ssh-ed25519 {}", STANDARD.encode(lying)) + }, + expect: |r| matches!(r, Err(KeyParseError::Truncated)), + }, + ]; + + #[test] + fn parse_authorized_key_accepts_genuine_lines_and_rejects_malformed_ones() { + KEY_CASES.iter().for_each(|case| { + let result = parse_authorized_key(&(case.line)()); + assert!( + (case.expect)(&result), + "case {:?} got {result:?}", + case.name + ); + }); + } + + #[test] + fn list_records_yields_every_well_formed_key_and_skips_the_rest() { + let good_one = ssh_line("ssh-ed25519", &[1u8; 32], "one"); + let good_two = ssh_line("ssh-ed25519", &[2u8; 32], "two"); + let body = serde_json::json!({ + "records": [ + { "uri": "at://did:plc:squid/sh.tangled.publicKey/a", "value": { "$type": "sh.tangled.publicKey", "key": good_one, "name": "laptop", "createdAt": "2026-06-08T00:00:00Z" } }, + { "uri": "at://did:plc:squid/sh.tangled.publicKey/b", "value": { "$type": "sh.tangled.publicKey", "key": "garbage line", "name": "broken", "createdAt": "2026-06-08T00:00:00Z" } }, + { "uri": "at://did:plc:squid/sh.tangled.publicKey/c", "value": { "$type": "sh.tangled.publicKey", "key": good_two, "name": "desktop", "createdAt": "2026-06-08T00:00:00Z" } } + ], + "cursor": "c" + }); + let page = offered_page(serde_json::to_vec(&body).unwrap().as_slice(), 100).unwrap(); + assert_eq!(page.keys.len(), 2); + assert_eq!(page.keys[0], parse_authorized_key(&good_one).unwrap()); + assert_eq!(page.keys[1], parse_authorized_key(&good_two).unwrap()); + assert_eq!(page.cursor.as_ref().map(Cursor::as_str), Some("c")); + } + + #[test] + fn a_cursor_serializes_as_a_plain_json_string() { + let cursor = Cursor::new("page-token"); + assert_eq!(cursor.as_str(), "page-token"); + assert_eq!(serde_json::to_string(&cursor).unwrap(), "\"page-token\""); + let parsed: Cursor = serde_json::from_str("\"page-token\"").unwrap(); + assert_eq!(parsed, cursor); + } + + #[test] + fn a_page_is_bounded_by_the_requested_record_limit() { + let records: Vec<_> = (0u32..50) + .map(|seed| { + let mut material = [0u8; 32]; + material[..4].copy_from_slice(&seed.to_be_bytes()); + let line = ssh_line("ssh-ed25519", &material, "k"); + serde_json::json!({ + "uri": "at://did:plc:squid/sh.tangled.publicKey/x", + "value": { "$type": "sh.tangled.publicKey", "key": line, "name": "k", "createdAt": "2026-06-08T00:00:00Z" } + }) + }) + .collect(); + let body = serde_json::json!({ "records": records }); + let page = offered_page(serde_json::to_vec(&body).unwrap().as_slice(), 10).unwrap(); + assert_eq!( + page.keys.len(), + 10, + "page yields at most requested record limit, no matter how many the PDS returns" + ); + } +} diff --git a/knot2/crates/knot-atproto/src/resolve.rs b/knot2/crates/knot-atproto/src/resolve.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-atproto/src/resolve.rs @@ -0,0 +1,586 @@ +use std::borrow::Cow; + +use knot_runtime::PublicKeyBytes; +use knot_types::crypto::{KeyCodec, PublicKey as CryptoKey}; +use knot_types::did_doc::DidDocument; +use knot_types::{AccountDid, Handle, HttpStatus, RepoDid}; +use url::{Host, Url}; + +const LEGACY_K256_KIND: &str = "EcdsaSecp256k1VerificationKey2019"; +const LEGACY_P256_KIND: &str = "EcdsaSecp256r1VerificationKey2019"; + +#[derive(Debug, Clone, thiserror::Error)] +pub enum ResolveError { + #[error("unsupported DID method in {value:?}")] + UnsupportedMethod { value: String }, + #[error("DID {value:?} doesn't form a resolvable document location")] + Unresolvable { value: String }, + #[error("DID document fetch returned HTTP {status}")] + Status { status: HttpStatus }, + #[error("network failure resolving DID document: {0}")] + Network(#[from] knot_runtime::NetworkError), + #[error("DID document isn't valid JSON: {0}")] + Malformed(String), + #[error("DID document for {requested:?} claims to describe {document:?}")] + IdMismatch { requested: String, document: String }, + #[error("refusing to resolve over non-https endpoint: {url}")] + InsecureScheme { url: String }, + #[error("refusing to resolve non-public address {host}")] + BlockedHost { host: String }, + #[error("DID document declares no atproto signing key")] + MissingSigningKey, + #[error("DID document signing key is unusable: {0}")] + BadSigningKey(String), + #[error("DID document declares no atproto_pds service endpoint")] + MissingPds, + #[error("atproto_pds endpoint {value:?} isn't valid URL")] + BadPds { value: String }, + #[error("PLC directory {value:?} isn't valid http(s) base URL")] + BadPlcDirectory { value: String }, + #[error("identity {did} recently failed to resolve and is negatively cached")] + RecentlyFailed { did: AccountDid }, + #[error("did:web document for {did} doesn't publish expected signing key")] + ExpectedKeyAbsent { did: RepoDid }, + #[error("handle {handle} has no atproto DNS or well-known record")] + HandleUnresolvable { handle: Handle }, + #[error("handle {handle} resolves to more than one distinct DID")] + HandleAmbiguous { handle: Handle }, + #[error("handle {handle} points at {value:?}, which isn't a valid DID")] + HandleForwardMalformed { handle: Handle, value: String }, + #[error("handle {handle} resolved to {resolved} but that document claims {claimed:?}")] + HandleMismatch { + handle: Handle, + resolved: AccountDid, + claimed: Option, + }, + #[error("handle {handle} recently failed to resolve and is negatively cached")] + HandleRecentlyFailed { handle: Handle }, +} + +impl ResolveError { + pub fn is_transient(&self) -> bool { + match self { + ResolveError::Network(_) => true, + ResolveError::Status { status } => status.is_transient(), + _ => false, + } + } +} + +#[derive(Debug, Clone)] +pub struct PdsEndpoint(Url); + +impl PdsEndpoint { + pub fn new(url: Url) -> Result { + match http_base(&url) { + true => Ok(Self(url)), + false => Err(ResolveError::BadPds { + value: url.as_str().to_string(), + }), + } + } + + pub fn url(&self) -> &Url { + &self.0 + } +} + +#[derive(Debug, Clone)] +pub struct PlcDirectory(Url); + +impl PlcDirectory { + pub fn new(url: Url) -> Result { + match http_base(&url) { + true => Ok(Self(url)), + false => Err(ResolveError::BadPlcDirectory { + value: url.as_str().to_string(), + }), + } + } +} + +fn http_base(url: &Url) -> bool { + matches!(url.scheme(), "http" | "https") + && url.has_host() + && url.username().is_empty() + && url.password().is_none() + && url.query().is_none() + && url.fragment().is_none() +} + +#[derive(Debug, Clone)] +pub struct Identity { + pub did: AccountDid, + pub handles: Vec, + pub signing_key: CryptoKey<'static>, + pub pds: PdsEndpoint, +} + +impl Identity { + pub fn primary_handle(&self) -> Option<&Handle> { + self.handles.first() + } + + pub fn claims_handle(&self, handle: &Handle) -> bool { + self.handles.iter().any(|known| known == handle) + } +} + +pub(crate) fn document_url( + did: &AccountDid, + plc_directory: &PlcDirectory, +) -> Result { + let value = did.as_str(); + if value.strip_prefix("did:plc:").is_some() { + let base = plc_directory.0.as_str().trim_end_matches('/'); + Url::parse(&format!("{base}/{value}")).map_err(|_| ResolveError::Unresolvable { + value: value.to_string(), + }) + } else if let Some(rest) = value.strip_prefix("did:web:") { + web_document_url(rest).ok_or(ResolveError::Unresolvable { + value: value.to_string(), + }) + } else { + Err(ResolveError::UnsupportedMethod { + value: value.to_string(), + }) + } +} + +pub(crate) fn guard_fetch_url(url: &Url) -> Result<(), ResolveError> { + if url.scheme() != "https" { + return Err(ResolveError::InsecureScheme { + url: url.as_str().to_string(), + }); + } + let blocked = match url.host() { + Some(Host::Ipv4(ip)) => knot_runtime::is_blocked_ip(ip.into()).then(|| ip.to_string()), + Some(Host::Ipv6(ip)) => knot_runtime::is_blocked_ip(ip.into()).then(|| ip.to_string()), + _ => None, + }; + match blocked { + Some(host) => Err(ResolveError::BlockedHost { host }), + None => Ok(()), + } +} + +fn web_document_url(rest: &str) -> Option { + let mut segments = rest.split(':'); + let authority = segments.next().filter(|head| !head.is_empty())?; + let host = authority.replace("%3A", ":").replace("%3a", ":"); + let path: Vec<&str> = segments.collect(); + let tail = if path.is_empty() { + ".well-known/did.json".to_string() + } else if path.iter().any(|segment| segment.is_empty()) { + return None; + } else { + format!("{}/did.json", path.join("/")) + }; + Url::parse(&format!("https://{host}/{tail}")).ok() +} + +pub(crate) fn identity_from_document( + did: &AccountDid, + body: &[u8], +) -> Result { + let document: DidDocument = + serde_json::from_slice(body).map_err(|error| ResolveError::Malformed(error.to_string()))?; + if AccountDid::new(document.id.as_str()).ok().as_ref() != Some(did) { + return Err(ResolveError::IdMismatch { + requested: did.as_str().to_string(), + document: document.id.as_str().to_string(), + }); + } + let signing_key = atproto_signing_key(&document)?; + let pds_endpoint = document.pds_endpoint().ok_or(ResolveError::MissingPds)?; + let pds = Url::parse(pds_endpoint.as_str()) + .map_err(|_| ResolveError::BadPds { + value: pds_endpoint.as_str().to_string(), + }) + .and_then(PdsEndpoint::new)?; + let handles = document + .handles() + .iter() + .filter_map(|found| Handle::new_owned(found.as_str()).ok()) + .collect(); + Ok(Identity { + did: did.clone(), + handles, + signing_key, + pds, + }) +} + +pub(crate) fn web_document_url_for(did: &RepoDid) -> Result { + let rest = + did.as_str() + .strip_prefix("did:web:") + .ok_or_else(|| ResolveError::UnsupportedMethod { + value: did.as_str().to_string(), + })?; + web_document_url(rest).ok_or_else(|| ResolveError::Unresolvable { + value: did.as_str().to_string(), + }) +} + +pub(crate) fn document_publishes_key( + did: &RepoDid, + body: &[u8], + expected: &PublicKeyBytes, +) -> Result<(), ResolveError> { + let document: DidDocument = + serde_json::from_slice(body).map_err(|error| ResolveError::Malformed(error.to_string()))?; + if document.id.as_str() != did.as_str() { + return Err(ResolveError::IdMismatch { + requested: did.as_str().to_string(), + document: document.id.as_str().to_string(), + }); + } + let methods = document + .verification_method + .as_ref() + .ok_or(ResolveError::MissingSigningKey)?; + let published = methods + .iter() + .filter_map(|method| method_key(method).ok()) + .any(|key| { + matches!(key.codec, KeyCodec::Secp256k1) && key.bytes.as_ref() == expected.as_bytes() + }); + if published { + Ok(()) + } else { + Err(ResolveError::ExpectedKeyAbsent { did: did.clone() }) + } +} + +fn supported_kind(kind: &str) -> bool { + matches!(kind, "Multikey" | LEGACY_K256_KIND | LEGACY_P256_KIND) +} + +fn method_key( + method: &knot_types::did_doc::VerificationMethod, +) -> Result, String> { + let multibase = method + .public_key_multibase + .as_ref() + .ok_or_else(|| "verification method lacks publicKeyMultibase".to_string())? + .as_ref(); + match method.r#type.as_ref() { + "Multikey" => CryptoKey::decode_owned(multibase).map_err(|error| error.to_string()), + LEGACY_K256_KIND => legacy_key(KeyCodec::Secp256k1, multibase), + LEGACY_P256_KIND => legacy_key(KeyCodec::P256, multibase), + other => Err(format!("unsupported verification method type {other:?}")), + } +} + +fn legacy_key(codec: KeyCodec, multibase: &str) -> Result, String> { + let encoded = multibase + .strip_prefix('z') + .ok_or_else(|| format!("legacy key {multibase:?} isn't base58btc multibase"))?; + let bytes = bs58::decode(encoded) + .into_vec() + .map_err(|error| error.to_string())?; + Ok(CryptoKey { + codec, + bytes: Cow::Owned(bytes), + }) +} + +fn atproto_signing_key(document: &DidDocument) -> Result, ResolveError> { + let method = document + .verification_method + .as_ref() + .and_then(|methods| { + methods.iter().find(|method| { + let id: &str = method.id.as_ref(); + id.ends_with("#atproto") + && supported_kind(method.r#type.as_ref()) + && method.public_key_multibase.is_some() + }) + }) + .ok_or(ResolveError::MissingSigningKey)?; + method_key(method).map_err(ResolveError::BadSigningKey) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::*; + use bytes::Bytes; + + struct UrlCase { + did: &'static str, + expect: fn(&Result) -> bool, + } + + const URL_CASES: &[UrlCase] = &[ + UrlCase { + did: "did:plc:squid", + expect: |r| matches!(r, Ok(u) if u.as_str() == "https://plc.directory/did:plc:squid"), + }, + UrlCase { + did: "did:web:nel.pet", + expect: |r| matches!(r, Ok(u) if u.as_str() == "https://nel.pet/.well-known/did.json"), + }, + UrlCase { + did: "did:web:nel.pet:repos:squid", + expect: |r| matches!(r, Ok(u) if u.as_str() == "https://nel.pet/repos/squid/did.json"), + }, + UrlCase { + did: "did:web:nel.pet%3A8443", + expect: |r| matches!(r, Ok(u) if u.as_str() == "https://nel.pet:8443/.well-known/did.json"), + }, + UrlCase { + did: "did:key:zabc", + expect: |r| matches!(r, Err(ResolveError::UnsupportedMethod { .. })), + }, + ]; + + #[test] + fn document_url_maps_each_did_method_to_its_document_location() { + URL_CASES.iter().for_each(|case| { + let result = document_url(&did(case.did), &plc()); + assert!((case.expect)(&result), "case {:?} got {result:?}", case.did); + }); + } + + fn body(value: serde_json::Value) -> Bytes { + Bytes::from(serde_json::to_vec(&value).unwrap()) + } + + fn sample_key() -> String { + knot_types::crypto::multikey(0xe7, &sec1(&signer(5))) + } + + fn legacy_document(kind: &str, multibase: &str) -> Bytes { + body(serde_json::json!({ + "id": SQUID, + "alsoKnownAs": ["at://nel.pet"], + "verificationMethod": [{ + "id": format!("{SQUID}#atproto"), + "type": kind, + "controller": SQUID, + "publicKeyMultibase": multibase + }], + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": "https://pds.oyster.cafe" + }] + })) + } + + #[test] + fn a_complete_document_yields_a_full_identity() { + let body = did_doc(DocSpec { + id: SQUID, + signing: &signer(5), + handle: "nel.pet", + pds: "https://pds.oyster.cafe", + method: MethodKind::Multikey, + }); + let identity = identity_from_document(&did(SQUID), &body).unwrap(); + assert_eq!(identity.pds.url().as_str(), "https://pds.oyster.cafe/"); + assert_eq!(identity.primary_handle().unwrap().as_str(), "nel.pet"); + } + + #[test] + fn the_atproto_verification_method_wins_over_a_decoy_first_key() { + let decoy = sec1(&signer(3)); + let real = sec1(&signer(5)); + let document = body(serde_json::json!({ + "id": SQUID, + "alsoKnownAs": ["at://nel.pet"], + "verificationMethod": [ + { + "id": format!("{SQUID}#extra"), + "type": "Multikey", + "controller": SQUID, + "publicKeyMultibase": knot_types::crypto::multikey(0xe7, &decoy) + }, + { + "id": format!("{SQUID}#atproto"), + "type": "Multikey", + "controller": SQUID, + "publicKeyMultibase": knot_types::crypto::multikey(0xe7, &real) + } + ], + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": "https://pds.oyster.cafe" + }] + })); + let identity = identity_from_document(&did(SQUID), &document).unwrap(); + assert_eq!(identity.signing_key.bytes.as_ref(), real.as_slice()); + assert_ne!(identity.signing_key.bytes.as_ref(), decoy.as_slice()); + } + + #[test] + fn a_legacy_secp256k1_verification_method_resolves() { + let body = did_doc(DocSpec { + id: SQUID, + signing: &signer(5), + handle: "nel.pet", + pds: "https://pds.oyster.cafe", + method: MethodKind::LegacyK256, + }); + let identity = identity_from_document(&did(SQUID), &body).unwrap(); + assert_eq!( + identity.signing_key.bytes.as_ref(), + sec1(&signer(5)).as_slice() + ); + assert!(matches!(identity.signing_key.codec, KeyCodec::Secp256k1)); + } + + #[test] + fn document_publishes_key_matches_only_on_codec_and_bytes() { + let published = did_doc(DocSpec { + id: SQUID, + signing: &signer(5), + handle: "nel.pet", + pds: "https://pds.oyster.cafe", + method: MethodKind::LegacyK256, + }); + document_publishes_key( + &RepoDid::new(SQUID).unwrap(), + &published, + &PublicKeyBytes::from_bytes(sec1(&signer(5))), + ) + .unwrap(); + + let foreign = did_doc(DocSpec { + id: SQUID, + signing: &signer(5), + handle: "nel.pet", + pds: "https://pds.oyster.cafe", + method: MethodKind::LegacyP256, + }); + let error = document_publishes_key( + &RepoDid::new(SQUID).unwrap(), + &foreign, + &PublicKeyBytes::from_bytes(sec1(&signer(5))), + ) + .unwrap_err(); + assert!( + matches!(error, ResolveError::ExpectedKeyAbsent { .. }), + "the same bytes under a foreign codec mustn't satisfy publishes_key, got {error:?}" + ); + } + + struct DocCase { + name: &'static str, + requested: &'static str, + body: fn() -> Bytes, + expect: fn(&Result) -> bool, + } + + const DOC_CASES: &[DocCase] = &[ + DocCase { + name: "document declares no atproto_pds service", + requested: SQUID, + body: || { + body(serde_json::json!({ + "id": SQUID, + "verificationMethod": [{ + "id": format!("{SQUID}#atproto"), + "type": "Multikey", + "controller": SQUID, + "publicKeyMultibase": sample_key() + }] + })) + }, + expect: |r| matches!(r, Err(ResolveError::MissingPds)), + }, + DocCase { + name: "document declares no signing key", + requested: SQUID, + body: || { + body(serde_json::json!({ + "id": SQUID, + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": "https://pds.oyster.cafe" + }] + })) + }, + expect: |r| matches!(r, Err(ResolveError::MissingSigningKey)), + }, + DocCase { + name: "body isn't valid json", + requested: SQUID, + body: || Bytes::from_static(b"not json"), + expect: |r| matches!(r, Err(ResolveError::Malformed(_))), + }, + DocCase { + name: "legacy key without a multibase prefix", + requested: SQUID, + body: || { + legacy_document( + "EcdsaSecp256k1VerificationKey2019", + &bs58::encode(sec1(&signer(5))).into_string(), + ) + }, + expect: |r| matches!(r, Err(ResolveError::BadSigningKey(_))), + }, + DocCase { + name: "unsupported verification method type", + requested: SQUID, + body: || { + legacy_document( + "JsonWebKey2020", + &format!("z{}", bs58::encode(sec1(&signer(5))).into_string()), + ) + }, + expect: |r| matches!(r, Err(ResolveError::MissingSigningKey)), + }, + DocCase { + name: "document has only a non-atproto method", + requested: SQUID, + body: || { + body(serde_json::json!({ + "id": SQUID, + "verificationMethod": [{ + "id": format!("{SQUID}#extra"), + "type": "Multikey", + "controller": SQUID, + "publicKeyMultibase": knot_types::crypto::multikey(0xe7, &sec1(&signer(3))) + }], + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": "https://pds.oyster.cafe" + }] + })) + }, + expect: |r| matches!(r, Err(ResolveError::MissingSigningKey)), + }, + DocCase { + name: "knot repo did:plc isn't resolvable as an account", + requested: "did:plc:anemone", + body: || { + did_doc(DocSpec { + id: "did:plc:anemone", + signing: &signer(5), + handle: "nel.pet", + pds: "https://knot.oyster.cafe/repo/anemone", + method: MethodKind::None, + }) + }, + expect: |r| matches!(r, Err(ResolveError::MissingSigningKey)), + }, + ]; + + #[test] + fn identity_from_document_rejects_every_underspecified_document() { + DOC_CASES.iter().for_each(|case| { + let result = identity_from_document(&did(case.requested), &(case.body)()); + assert!( + (case.expect)(&result), + "case {:?} got {result:?}", + case.name + ); + }); + } +} diff --git a/knot2/crates/knot-atproto/src/test_support.rs b/knot2/crates/knot-atproto/src/test_support.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-atproto/src/test_support.rs @@ -0,0 +1,199 @@ +use std::borrow::Cow; +use std::sync::{Arc, Mutex}; + +use base64::Engine; +use base64::engine::general_purpose::{STANDARD, URL_SAFE_NO_PAD}; +use bytes::Bytes; +use http::StatusCode; +use k256::ecdsa::{Signature, SigningKey, signature::Signer}; +use knot_runtime::{HttpResponse, K256Signer, ManualClock, SeededEntropy, UnixMicros}; +use knot_types::crypto::{KeyCodec, PublicKey as CryptoKey}; +use knot_types::{AccountDid, KnotId, Nsid, OwnerDid, RepoDid, RepoRkey}; +use serde_json::Value; +use url::Url; + +use crate::{MintNonce, ServiceJwt}; + +pub(crate) const SQUID: &str = "did:plc:squid"; +pub(crate) const LIMPET: &str = "did:plc:limpet"; +pub(crate) const KNOT: &str = "did:web:nel.pet"; +pub(crate) const METHOD: &str = "sh.tangled.knot.addMember"; + +pub(crate) fn did(value: &str) -> AccountDid { + AccountDid::new(value).unwrap() +} + +pub(crate) fn repo_did(value: &str) -> RepoDid { + RepoDid::new(value).unwrap() +} + +pub(crate) fn owner_did(value: &str) -> OwnerDid { + OwnerDid::new(value).unwrap() +} + +pub(crate) fn knot_did(value: &str) -> KnotId { + KnotId::new(value).unwrap() +} + +pub(crate) fn handle(value: &str) -> knot_types::Handle { + knot_types::Handle::new_owned(value).unwrap() +} + +pub(crate) fn member_method() -> Nsid { + Nsid::new_owned(METHOD).unwrap() +} + +pub(crate) fn plc() -> crate::PlcDirectory { + crate::PlcDirectory::new(Url::parse("https://plc.directory/").unwrap()).unwrap() +} + +pub(crate) fn clock() -> ManualClock { + ManualClock::new(UnixMicros::new(1_000_000_000)) +} + +pub(crate) fn signer(seed: u8) -> SigningKey { + SigningKey::from_bytes(&[seed; 32].into()).unwrap() +} + +pub(crate) fn sec1(signing: &SigningKey) -> Vec { + signing + .verifying_key() + .to_encoded_point(true) + .as_bytes() + .to_vec() +} + +pub(crate) fn k256_public(signing: &SigningKey) -> CryptoKey<'static> { + CryptoKey { + codec: KeyCodec::Secp256k1, + bytes: Cow::Owned(sec1(signing)), + } +} + +pub(crate) enum MethodKind { + Multikey, + LegacyK256, + LegacyP256, + None, +} + +pub(crate) struct DocSpec<'a> { + pub id: &'a str, + pub signing: &'a SigningKey, + pub handle: &'a str, + pub pds: &'a str, + pub method: MethodKind, +} + +pub(crate) fn did_doc(spec: DocSpec) -> Bytes { + let raw = sec1(spec.signing); + let legacy = || format!("z{}", bs58::encode(&raw).into_string()); + let method_entry: Option<(&'static str, String)> = match spec.method { + MethodKind::Multikey => Some(("Multikey", knot_types::crypto::multikey(0xe7, &raw))), + MethodKind::LegacyK256 => Some(("EcdsaSecp256k1VerificationKey2019", legacy())), + MethodKind::LegacyP256 => Some(("EcdsaSecp256r1VerificationKey2019", legacy())), + MethodKind::None => None, + }; + let verification_method: Value = match method_entry { + Some((kind, multibase)) => serde_json::json!([{ + "id": format!("{}#atproto", spec.id), + "type": kind, + "controller": spec.id, + "publicKeyMultibase": multibase, + }]), + None => serde_json::json!([]), + }; + let body = serde_json::json!({ + "id": spec.id, + "alsoKnownAs": [format!("at://{}", spec.handle)], + "verificationMethod": verification_method, + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": spec.pds, + }] + }); + Bytes::from(serde_json::to_vec(&body).unwrap()) +} + +pub(crate) fn ssh_line(algo: &str, material: &[u8], comment: &str) -> String { + let ssh_string = |bytes: &[u8]| [&(bytes.len() as u32).to_be_bytes()[..], bytes].concat(); + let blob = [ssh_string(algo.as_bytes()), ssh_string(material)].concat(); + format!("{algo} {} {comment}", STANDARD.encode(blob)) +} + +pub(crate) fn list_body(lines: &[String], cursor: Option<&str>) -> Bytes { + let records: Vec<_> = lines + .iter() + .enumerate() + .map(|(index, line)| { + serde_json::json!({ + "uri": format!("at://{SQUID}/sh.tangled.publicKey/{index}"), + "value": { "$type": "sh.tangled.publicKey", "key": line, "name": "k", "createdAt": "2026-06-08T00:00:00Z" } + }) + }) + .collect(); + let cursor_field: Value = cursor.map_or(Value::Null, |value| serde_json::json!(value)); + let body = serde_json::json!({ "records": records, "cursor": cursor_field }); + Bytes::from(serde_json::to_vec(&body).unwrap()) +} + +pub(crate) fn ok(body: Bytes) -> HttpResponse { + status(StatusCode::OK, body) +} + +pub(crate) fn status(status: StatusCode, body: Bytes) -> HttpResponse { + HttpResponse { + status, + headers: http::HeaderMap::new(), + body, + } +} + +pub(crate) fn mint(signing: &SigningKey, claims: &Value) -> ServiceJwt { + mint_with_header(signing, br#"{"alg":"ES256K","typ":"JWT"}"#, claims) +} + +pub(crate) fn mint_with_header(signing: &SigningKey, header: &[u8], claims: &Value) -> ServiceJwt { + let header_b64 = URL_SAFE_NO_PAD.encode(header); + let payload = URL_SAFE_NO_PAD.encode(serde_json::to_vec(claims).unwrap()); + let signing_input = format!("{header_b64}.{payload}"); + let signature: Signature = signing.sign(signing_input.as_bytes()); + ServiceJwt::new(format!( + "{signing_input}.{}", + URL_SAFE_NO_PAD.encode(signature.to_bytes()) + )) + .expect("minted token is structurally a JWT") +} + +pub(crate) fn runtime_signer(seed: u64) -> K256Signer { + K256Signer::generate(&SeededEntropy::new(seed)) +} + +pub(crate) fn entropy(seed: u64) -> SeededEntropy { + SeededEntropy::new(seed) +} + +pub(crate) fn repo_nonce(seed: u64) -> MintNonce { + MintNonce::mint( + &entropy(seed), + &owner_did("did:plc:nel"), + &RepoRkey::new("anemone").unwrap(), + ) +} + +pub(crate) fn member_pointer() -> knot_lexicons::sh_tangled::knot::member::Member { + knot_lexicons::sh_tangled::knot::member::Member { + created_at: knot_types::Datetime::raw_str("2026-06-11T00:00:00Z"), + domain: "knot.nel.pet".into(), + subject: knot_types::Did::new_owned("did:plc:lyna").unwrap(), + extra_data: None, + } +} + +pub(crate) type UrlLog = Arc>>; + +pub(crate) fn recorder() -> (UrlLog, UrlLog) { + let urls: UrlLog = Arc::new(Mutex::new(Vec::new())); + (urls.clone(), urls) +} diff --git a/knot2/crates/knot-atproto/tests/fuzz_smoke.rs b/knot2/crates/knot-atproto/tests/fuzz_smoke.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-atproto/tests/fuzz_smoke.rs @@ -0,0 +1,15 @@ +use proptest::prelude::*; + +proptest! { + #![proptest_config(ProptestConfig::with_cases(1024))] + + #[test] + fn the_pubkey_parsers_never_panic(data in proptest::collection::vec(any::(), 0..4096)) { + knot_atproto::fuzz::pubkey(&data); + } + + #[test] + fn the_did_document_decoder_never_panics(data in proptest::collection::vec(any::(), 0..4096)) { + knot_atproto::fuzz::did_document(&data); + } +} diff --git a/knot2/crates/knot-atproto/tests/security.rs b/knot2/crates/knot-atproto/tests/security.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-atproto/tests/security.rs @@ -0,0 +1,62 @@ +use std::io::{Read, Write}; +use std::net::TcpListener; + +use knot_runtime::{HttpLimits, HttpRequest, HttpTransport, NetworkError, ReqwestHttp}; +use url::Url; + +#[tokio::test] +async fn the_transport_refuses_a_loopback_target() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + let http = ReqwestHttp::new(HttpLimits::default()).unwrap(); + let url = Url::parse(&format!("http://127.0.0.1:{port}/")).unwrap(); + let error = http.execute(HttpRequest::get(url)).await.unwrap_err(); + assert!( + matches!(error, NetworkError::Blocked { .. }), + "got {error:?}" + ); + drop(listener); +} + +#[tokio::test] +async fn a_hostname_resolving_only_to_loopback_is_refused_at_dns() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + let http = ReqwestHttp::new(HttpLimits::default()).unwrap(); + let url = Url::parse(&format!("https://localhost:{port}/")).unwrap(); + let error = http.execute(HttpRequest::get(url)).await.unwrap_err(); + assert!( + matches!( + error, + NetworkError::Connect(_) | NetworkError::Request(_) | NetworkError::Timeout(_) + ), + "hostname whose only addresses are loopback must fail to connect, got {error:?}" + ); + drop(listener); +} + +#[tokio::test] +async fn a_redirect_to_an_internal_host_is_not_followed() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + std::thread::spawn(move || { + if let Ok((mut stream, _)) = listener.accept() { + let _ = stream.read(&mut [0u8; 1024]); + let _ = stream.write_all( + b"HTTP/1.1 302 Found\r\nLocation: http://169.254.169.254/\r\nContent-Length: 0\r\n\r\n", + ); + } + }); + let limits = HttpLimits { + block_private_addresses: false, + ..HttpLimits::default() + }; + let http = ReqwestHttp::new(limits).unwrap(); + let url = Url::parse(&format!("http://{addr}/")).unwrap(); + let response = http.execute(HttpRequest::get(url)).await.unwrap(); + assert_eq!( + response.status.as_u16(), + 302, + "302 is surfaced verbatim, redirect to internal host is never followed" + ); +} diff --git a/knot2/crates/knot-bench/benches/advert.rs b/knot2/crates/knot-bench/benches/advert.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-bench/benches/advert.rs @@ -0,0 +1,30 @@ +use divan::Bencher; +use divan::counter::ItemsCount; +use knot_bench::{RefCount, build_many_refs}; + +#[global_allocator] +static ALLOC: divan::AllocProfiler = divan::AllocProfiler::system(); + +const GRADES: &[u32] = &[16, 256, 4096]; + +fn main() { + divan::main(); +} + +#[divan::bench(args = GRADES)] +fn advert_uncached(bencher: Bencher, refs: u32) { + let built = build_many_refs(RefCount::new(refs)); + let count = built.repo().references().unwrap().len(); + bencher + .counter(ItemsCount::new(count)) + .bench_local(|| built.repo().references().unwrap()); +} + +#[divan::bench(args = GRADES)] +fn advert_cached(bencher: Bencher, refs: u32) { + let built = build_many_refs(RefCount::new(refs)); + let count = built.repo().advertised_refs().unwrap().len(); + bencher + .counter(ItemsCount::new(count)) + .bench_local(|| built.repo().advertised_refs().unwrap()); +} diff --git a/knot2/crates/knot-bench/benches/cob.rs b/knot2/crates/knot-bench/benches/cob.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-bench/benches/cob.rs @@ -0,0 +1,27 @@ +use divan::Bencher; +use divan::counter::ItemsCount; +use knot_bench::{ChangeCount, RepoCount, build_linear_cob, build_registry_checkpointed}; + +#[global_allocator] +static ALLOC: divan::AllocProfiler = divan::AllocProfiler::system(); + +const GRADES: &[u32] = &[64, 512, 4096]; +const REPO_GRADES: &[u64] = &[256, 1024, 4096]; + +fn main() { + divan::main(); +} + +#[divan::bench(args = GRADES)] +fn fold(bencher: Bencher, changes: u32) { + let cob = build_linear_cob(ChangeCount::new(changes)); + bencher + .counter(ItemsCount::new(changes as usize)) + .bench_local(|| cob.fold()); +} + +#[divan::bench(args = REPO_GRADES)] +fn registry_write_checkpointed(bencher: Bencher, repos: u64) { + let writer = build_registry_checkpointed(RepoCount::new(repos)); + bencher.bench_local(|| writer.probe()); +} diff --git a/knot2/crates/knot-bench/benches/coldstart.rs b/knot2/crates/knot-bench/benches/coldstart.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-bench/benches/coldstart.rs @@ -0,0 +1,54 @@ +use divan::Bencher; +use divan::counter::ItemsCount; +use knot_bench::{OpenLatency, RepoCount, build_registry, replay_boot}; +use knot_types::RepoDid; + +#[global_allocator] +static ALLOC: divan::AllocProfiler = divan::AllocProfiler::system(); + +const FABRIC_OPEN_MICROS: u64 = 200; + +fn repo_counts() -> Vec { + let Ok(raw) = std::env::var("KNOT_BENCH_REPOS") else { + return vec![1, 1000]; + }; + let counts: Vec = raw + .split(',') + .map(str::trim) + .filter(|entry| !entry.is_empty()) + .map(|entry| { + entry + .parse::() + .unwrap_or_else(|_| panic!("KNOT_BENCH_REPOS entry {entry:?} is not a repo count")) + }) + .collect(); + assert!( + !counts.is_empty(), + "KNOT_BENCH_REPOS is set but lists no repo counts" + ); + counts +} + +fn main() { + divan::main(); +} + +#[divan::bench(args = repo_counts())] +fn boot(bencher: Bencher, repos: u64) { + let registry = build_registry(RepoCount::new(repos)); + bencher + .counter(ItemsCount::new(registry.dids().len())) + .bench_local(|| registry.index().rebuild().unwrap()); +} + +#[divan::bench(args = repo_counts())] +fn boot_fabric(bencher: Bencher, repos: u64) { + let registry = build_registry(RepoCount::new(repos)); + let dids: Vec = registry.dids().to_vec(); + bencher + .counter(ItemsCount::new(dids.len())) + .bench_local(|| { + let index = registry.index(); + replay_boot(&index, &dids, OpenLatency::micros(FABRIC_OPEN_MICROS)); + }); +} diff --git a/knot2/crates/knot-bench/benches/pack.rs b/knot2/crates/knot-bench/benches/pack.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-bench/benches/pack.rs @@ -0,0 +1,257 @@ +use std::time::Duration; + +use divan::Bencher; +use divan::counter::{BytesCount, ItemsCount}; +use knot_bench::{ChurnCount, CommitCount, HistorySpec, PathCount, build_history}; +use knot_git::{Filter, Haves, PackBudget, Repo, Wants}; +use knot_pack::{ + HaveOids, PackLimits, ReceiveCommand, ReceiveGuard, RefDecision, WantOids, count_expanded, + local_pack, receive_pack_guarded, upload_archive, write_expanded, write_pack, +}; +use knot_types::{ObjectCount, Oid}; + +#[global_allocator] +static ALLOC: divan::AllocProfiler = divan::AllocProfiler::system(); + +const GRADES: &[u32] = &[64, 256, 1024]; +const CAP: u64 = 512 * 1024 * 1024; + +fn spec_for(commits: u32) -> HistorySpec { + HistorySpec { + commits: CommitCount::new(commits), + paths: PathCount::new(commits.max(64)), + churn: ChurnCount::new(8), + } +} + +fn pkt(payload: &[u8]) -> Vec { + let mut out = format!("{:04x}", payload.len() + 4).into_bytes(); + out.extend_from_slice(payload); + out +} + +struct AllowAll; + +impl ReceiveGuard for AllowAll { + fn authorize(&self, _staged: &Repo, commands: &[ReceiveCommand]) -> Vec { + commands.iter().map(|_| RefDecision::Allow).collect() + } +} + +fn main() { + divan::main(); +} + +#[divan::bench(args = GRADES)] +fn select(bencher: Bencher, commits: u32) { + let history = build_history(spec_for(commits)); + let tips = history.tips(); + let count = history + .repo() + .select_pack_objects_filtered( + Wants::new(&tips), + Haves::new(&[]), + Filter::None, + PackBudget::unbounded(), + ) + .unwrap() + .send + .len(); + bencher.counter(ItemsCount::new(count)).bench_local(|| { + history + .repo() + .select_pack_objects_filtered( + Wants::new(&tips), + Haves::new(&[]), + Filter::None, + PackBudget::unbounded(), + ) + .unwrap() + }); +} + +#[divan::bench(args = GRADES)] +fn clone(bencher: Bencher, commits: u32) { + let history = build_history(spec_for(commits)); + let wants = WantOids::new(history.tips()); + let no_haves = HaveOids::default(); + let bytes = local_pack(history.repo(), &wants, &no_haves, CAP) + .unwrap() + .len(); + bencher + .counter(BytesCount::new(bytes)) + .bench_local(|| local_pack(history.repo(), &wants, &no_haves, CAP).unwrap()); +} + +#[divan::bench(args = GRADES)] +fn full_clone_manual(bencher: Bencher, commits: u32) { + let history = build_history(spec_for(commits)); + let tips = history.tips(); + let dir = history.repo().objects_dir(); + let count = history + .repo() + .select_pack_objects_filtered( + Wants::new(&tips), + Haves::new(&[]), + Filter::None, + PackBudget::unbounded(), + ) + .unwrap() + .send + .len(); + bencher.counter(ItemsCount::new(count)).bench_local(|| { + let send = history + .repo() + .select_pack_objects_filtered( + Wants::new(&tips), + Haves::new(&[]), + Filter::None, + PackBudget::unbounded(), + ) + .unwrap() + .send; + write_pack( + &dir, + send, + None, + &mut std::io::sink(), + history.repo().object_format().kind(), + ) + .unwrap(); + }); +} + +#[divan::bench(args = GRADES)] +fn full_clone_expanding(bencher: Bencher, commits: u32) { + let history = build_history(spec_for(commits)); + let tips = history.tips(); + let dir = history.repo().objects_dir(); + let far = Duration::from_secs(3600); + let roots = history + .repo() + .clone_roots(&tips, PackBudget::unbounded()) + .unwrap(); + let kind = history.repo().object_format().kind(); + let count = count_expanded(&dir, roots, ObjectCount::new(usize::MAX), far, kind) + .unwrap() + .len(); + bencher.counter(ItemsCount::new(count)).bench_local(|| { + let roots = history + .repo() + .clone_roots(&tips, PackBudget::unbounded()) + .unwrap(); + let pack = count_expanded(&dir, roots, ObjectCount::new(usize::MAX), far, kind).unwrap(); + write_expanded(pack, &mut std::io::sink()).unwrap(); + }); +} + +#[divan::bench(args = GRADES)] +fn fetch(bencher: Bencher, commits: u32) { + let history = build_history(spec_for(commits)); + let tips = history.tips(); + let walk = history + .repo() + .rev_walk(Wants::new(&tips), Haves::new(&[])) + .unwrap(); + let haves = HaveOids::new(vec![walk[walk.len() / 2]]); + let wants = WantOids::new(tips); + let bytes = local_pack(history.repo(), &wants, &haves, CAP) + .unwrap() + .len(); + bencher + .counter(BytesCount::new(bytes)) + .bench_local(|| local_pack(history.repo(), &wants, &haves, CAP).unwrap()); +} + +#[divan::bench(args = GRADES)] +fn push(bencher: Bencher, commits: u32) { + let history = build_history(spec_for(commits)); + let tips = history.tips(); + let pack = local_pack( + history.repo(), + &WantOids::new(tips.clone()), + &HaveOids::default(), + CAP, + ) + .unwrap(); + let walk = history + .repo() + .rev_walk(Wants::new(&tips), Haves::new(&[])) + .unwrap(); + let stride = walk.len().max(1) / 8 + 1; + let branch_tips: Vec = walk.iter().step_by(stride).copied().collect(); + let request = build_receive_request(&branch_tips, &pack); + let limits = PackLimits::default(); + bencher + .counter(ItemsCount::new(branch_tips.len())) + .with_inputs(fresh_target) + .bench_local_values(|target| { + receive_pack_guarded( + target.repo(), + &request, + &limits, + &AllowAll, + &|_| {}, + &knot_pack::default_catalog().reject, + ) + .unwrap() + }); +} + +#[divan::bench(args = GRADES)] +fn archive(bencher: Bencher, commits: u32) { + let history = build_history(spec_for(commits)); + let request = build_archive_request(history.tip()); + let bytes = upload_archive(history.repo(), &request).unwrap().len(); + bencher + .counter(BytesCount::new(bytes)) + .bench_local(|| upload_archive(history.repo(), &request).unwrap()); +} + +struct FreshTarget { + _dir: tempfile::TempDir, + repo: Repo, +} + +impl FreshTarget { + fn repo(&self) -> &Repo { + &self.repo + } +} + +fn fresh_target() -> FreshTarget { + let dir = tempfile::tempdir().expect("tempdir"); + let repo = Repo::create(dir.path().join("target.git")).expect("create target"); + FreshTarget { _dir: dir, repo } +} + +fn build_receive_request(branch_tips: &[Oid], pack: &[u8]) -> Vec { + let null = Oid::null().to_hex(); + let commands: Vec> = branch_tips + .iter() + .enumerate() + .map(|(index, tip)| { + let line = format!("{null} {} refs/heads/b{index}", tip.to_hex()); + match index { + 0 => { + let mut payload = line.into_bytes(); + payload.push(0); + payload.extend_from_slice(b"report-status\n"); + pkt(&payload) + } + _ => pkt(format!("{line}\n").as_bytes()), + } + }) + .collect(); + let mut request: Vec = commands.concat(); + request.extend_from_slice(b"0000"); + request.extend_from_slice(pack); + request +} + +fn build_archive_request(tip: Oid) -> Vec { + let mut request = pkt(b"argument --format=tar"); + request.extend_from_slice(&pkt(format!("argument {}", tip.to_hex()).as_bytes())); + request.extend_from_slice(b"0000"); + request +} diff --git a/knot2/crates/knot-bench/benches/projection.rs b/knot2/crates/knot-bench/benches/projection.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-bench/benches/projection.rs @@ -0,0 +1,37 @@ +use divan::Bencher; +use divan::counter::ItemsCount; +use knot_bench::{RepoCount, RosterCount, build_collaborator_roster, build_registry}; + +#[global_allocator] +static ALLOC: divan::AllocProfiler = divan::AllocProfiler::system(); + +const ROSTER_GRADES: &[u32] = &[64, 512, 4096]; +const REGISTRY_GRADES: &[u64] = &[64, 4096]; + +fn main() { + divan::main(); +} + +#[divan::bench(args = ROSTER_GRADES)] +fn collaborator_refresh(bencher: Bencher, collaborators: u32) { + let built = build_collaborator_roster(RosterCount::new(collaborators)); + let index = built.index(); + index.rebuild().expect("rebuild"); + index + .ensure_collaborators(built.repo()) + .expect("first fold"); + bencher + .counter(ItemsCount::new(collaborators as usize)) + .bench_local(|| index.refresh_collaborators(built.repo()).expect("refresh")); +} + +#[divan::bench(args = REGISTRY_GRADES)] +fn resolve(bencher: Bencher, repos: u64) { + let registry = build_registry(RepoCount::new(repos)); + let index = registry.index(); + index.rebuild().expect("rebuild"); + let (owner, rkey) = registry.alias(repos / 2); + bencher + .counter(ItemsCount::new(1usize)) + .bench_local(|| index.resolve_repo(&owner, &rkey)); +} diff --git a/knot2/crates/knot-bench/src/fixtures.rs b/knot2/crates/knot-bench/src/fixtures.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-bench/src/fixtures.rs @@ -0,0 +1,657 @@ +use std::path::{Path, PathBuf}; + +use knot_cob::{CobError, CobHome, CobId, CobStore}; +use knot_cobs::{ + CollaboratorsChange, Grant, MembersChange, MembersCob, Registration, RegistryChange, + RepoRegistryCob, add_member, register_repo, +}; +use knot_git::{ + EntryKind, Identity, Layout, NewCommit, RefUpdate, Repo, StagedAction, StagedChange, +}; +use knot_index::Index; +use knot_runtime::{K256Signer, SeededEntropy}; +use knot_types::{ + AccountDid, AuthorName, Email, KnotId, Oid, OwnerDid, RefName, RepoDid, RepoName, RepoRkey, + UnixSeconds, +}; +use tempfile::TempDir; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CommitCount(u32); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PathCount(u32); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ChurnCount(u32); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ChangeCount(u32); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RepoCount(u64); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RefCount(u32); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RosterCount(u32); + +impl CommitCount { + pub fn new(value: u32) -> Self { + Self(value.max(1)) + } + fn get(self) -> u32 { + self.0 + } +} + +impl PathCount { + pub fn new(value: u32) -> Self { + Self(value.max(1)) + } + fn get(self) -> u32 { + self.0 + } +} + +impl ChurnCount { + pub fn new(value: u32) -> Self { + Self(value.max(1)) + } + fn get(self) -> u32 { + self.0 + } +} + +impl ChangeCount { + pub fn new(value: u32) -> Self { + Self(value.max(1)) + } + fn get(self) -> u32 { + self.0 + } +} + +impl RepoCount { + pub fn new(value: u64) -> Self { + Self(value.max(1)) + } + pub fn get(self) -> u64 { + self.0 + } +} + +impl RefCount { + pub fn new(value: u32) -> Self { + Self(value.max(1)) + } + fn get(self) -> u32 { + self.0 + } +} + +impl RosterCount { + pub fn new(value: u32) -> Self { + Self(value.max(1)) + } + fn get(self) -> u32 { + self.0 + } +} + +#[derive(Debug, Clone, Copy)] +pub struct HistorySpec { + pub commits: CommitCount, + pub paths: PathCount, + pub churn: ChurnCount, +} + +const CONTENT_BYTES: usize = 128; +const GENESIS_SECONDS: i64 = 1_700_000_000; + +fn splitmix(seed: u64) -> u64 { + let z = seed.wrapping_add(0x9e37_79b9_7f4a_7c15); + let z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + let z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + z ^ (z >> 31) +} + +knot_types::scalar_newtype! { + struct PathIndex(u32); + struct Revision(u32); +} + +fn blob_content(path_index: PathIndex, revision: Revision) -> Vec { + let seed = splitmix(u64::from(path_index.get()) ^ u64::from(revision.get()).rotate_left(32)); + (0..CONTENT_BYTES) + .scan(seed, |state, _| { + *state = splitmix(*state); + Some((*state & 0xff) as u8) + }) + .collect() +} + +fn path_at(path_index: PathIndex) -> knot_types::RepoPath { + let path_index = path_index.get(); + knot_types::RepoPath::new(format!( + "src/m{:04}/f{:06}.dat", + path_index / 256, + path_index + )) + .expect("generated fixture path is well-formed") +} + +fn identity(revision: Revision) -> Identity { + Identity { + name: AuthorName::new("nel"), + email: Email::new("nel@oyster.cafe"), + time: UnixSeconds::new(GENESIS_SECONDS + i64::from(revision.get())), + offset_seconds: 0, + } +} + +fn put(path_index: PathIndex, revision: Revision) -> StagedChange { + StagedChange { + path: path_at(path_index), + action: StagedAction::Put { + content: blob_content(path_index, revision), + kind: EntryKind::Blob, + }, + } +} + +fn churn_indices(revision: Revision, spec: HistorySpec) -> impl Iterator { + let span = u64::from(spec.paths.get()); + let base = u64::from(revision.get()).wrapping_mul(u64::from(spec.churn.get())); + (0..spec.churn.get()) + .map(move |offset| PathIndex::new(((base + u64::from(offset)) % span) as u32)) +} + +pub struct BuiltHistory { + _dir: TempDir, + repo: Repo, + tip: Oid, +} + +impl BuiltHistory { + pub fn repo(&self) -> &Repo { + &self.repo + } + pub fn tip(&self) -> Oid { + self.tip + } + pub fn tips(&self) -> Vec { + vec![self.tip] + } +} + +pub fn build_history(spec: HistorySpec) -> BuiltHistory { + let dir = tempfile::tempdir().expect("tempdir"); + let repo = Repo::create(dir.path().join("repo.git")).expect("create repo"); + let tip = write_history(&repo, spec); + BuiltHistory { + _dir: dir, + repo, + tip, + } +} + +pub fn write_history(repo: &Repo, spec: HistorySpec) -> Oid { + let empty_tree = Oid::from(repo.git().empty_tree().id().detach()); + + let genesis_changes: Vec = (0..spec.paths.get()) + .map(|index| put(PathIndex::new(index), Revision::new(0))) + .collect(); + let genesis_tree = repo + .write_staged_tree(empty_tree, &genesis_changes) + .expect("genesis tree"); + let genesis_commit = repo + .write_commit(&NewCommit { + tree: genesis_tree, + parents: Vec::new(), + author: identity(Revision::new(0)), + committer: identity(Revision::new(0)), + message: "genesis".to_string(), + extra_headers: Vec::new(), + }) + .expect("genesis commit"); + + let (_, tip) = (1..spec.commits.get()) + .try_fold( + (genesis_tree, genesis_commit), + |(prev_tree, prev_commit), revision| -> Result<(Oid, Oid), knot_git::GitError> { + let revision = Revision::new(revision); + let changes: Vec = churn_indices(revision, spec) + .map(|index| put(index, revision)) + .collect(); + let tree = repo.write_staged_tree(prev_tree, &changes)?; + let commit = repo.write_commit(&NewCommit { + tree, + parents: vec![prev_commit], + author: identity(revision), + committer: identity(revision), + message: format!("revision {}", revision.get()), + extra_headers: Vec::new(), + })?; + Ok((tree, commit)) + }, + ) + .expect("commit chain"); + + let main = RefName::new("refs/heads/main").expect("main ref name"); + repo.update_ref(&RefUpdate::Create { + name: main.clone(), + new: tip, + }) + .expect("create main"); + repo.set_head(&main).expect("set head"); + + tip +} + +pub struct BuiltRefs { + _dir: TempDir, + repo: Repo, +} + +impl BuiltRefs { + pub fn repo(&self) -> &Repo { + &self.repo + } +} + +pub fn build_many_refs(count: RefCount) -> BuiltRefs { + let dir = tempfile::tempdir().expect("tempdir"); + let repo = Repo::create(dir.path().join("repo.git")).expect("create repo"); + let tip = write_history( + &repo, + HistorySpec { + commits: CommitCount::new(1), + paths: PathCount::new(1), + churn: ChurnCount::new(1), + }, + ); + (0..count.get()).for_each(|index| { + let name = RefName::new(format!("refs/heads/branch{index:06}")).expect("ref name"); + repo.update_ref(&RefUpdate::Create { name, new: tip }) + .expect("create ref"); + }); + BuiltRefs { _dir: dir, repo } +} + +fn synthetic_account(seed: u64) -> AccountDid { + AccountDid::new(format!("did:plc:acct{seed:012}")).expect("account did") +} + +fn synthetic_repo_did(index: u64) -> RepoDid { + RepoDid::new(format!("did:plc:repo{index:012}")).expect("repo did") +} + +fn registry_owner() -> OwnerDid { + OwnerDid::new("did:plc:nel").expect("owner did") +} + +fn knot_home() -> CobHome { + CobHome::from(&KnotId::new("did:web:knot.nel.pet").expect("knot did")) +} + +fn registration(index: u64) -> Registration { + let rkey = format!("repo{index:012}"); + Registration { + owner: registry_owner(), + rkey: RepoRkey::new(&rkey).expect("rkey"), + name: RepoName::new(&rkey).expect("repo name"), + repo: synthetic_repo_did(index), + created_at: UnixSeconds::new(GENESIS_SECONDS + index as i64), + } +} + +fn grant(seed: u64) -> Grant { + Grant { + subject: synthetic_account(seed), + added_by: synthetic_account(0), + created_at: UnixSeconds::new(GENESIS_SECONDS), + } +} + +pub struct BuiltRegistry { + _dir: TempDir, + meta_path: PathBuf, + layout: Layout, + dids: Vec, +} + +impl BuiltRegistry { + pub fn index(&self) -> Index { + Index::new(&self.meta_path, self.layout.clone()) + } + pub fn dids(&self) -> &[RepoDid] { + &self.dids + } + pub fn alias(&self, index: u64) -> (OwnerDid, RepoRkey) { + let reg = registration(index); + (reg.owner, reg.rkey) + } +} + +fn seed_members(meta_path: &Path, signer: &dyn knot_runtime::Signer) { + let meta = Repo::open(meta_path).expect("open meta"); + let store = CobStore::new(&meta); + store + .create( + &knot_home(), + &knot_cobs::MembersChange::Add(grant(1)), + signer, + UnixSeconds::new(GENESIS_SECONDS), + ) + .expect("seed members"); +} + +pub fn build_registry(repos: RepoCount) -> BuiltRegistry { + let dir = tempfile::tempdir().expect("tempdir"); + let meta_path = dir.path().join("meta.git"); + Repo::create(&meta_path).expect("create meta"); + let layout = Layout::new(dir.path().join("repos")); + let signer = K256Signer::generate(&SeededEntropy::new(7)); + + seed_members(&meta_path, &signer); + + let meta = Repo::open(&meta_path).expect("open meta"); + let store = CobStore::new(&meta); + let registry_object = store + .create( + &knot_home(), + &RegistryChange::Register(registration(0)), + &signer, + UnixSeconds::new(GENESIS_SECONDS), + ) + .expect("create registry") + .object; + (1..repos.get()).for_each(|index| { + store + .update( + &knot_home(), + registry_object, + &RegistryChange::Register(registration(index)), + &signer, + UnixSeconds::new(GENESIS_SECONDS + index as i64), + ) + .expect("register repo"); + }); + + let dids: Vec = (0..repos.get()).map(synthetic_repo_did).collect(); + + dids.iter().enumerate().for_each(|(index, did)| { + let git = layout.create(did).expect("create repo dir"); + let collab = CobStore::new(&git); + collab + .create( + &CobHome::from(did), + &CollaboratorsChange::Add(grant(index as u64 + 2)), + &signer, + UnixSeconds::new(GENESIS_SECONDS), + ) + .expect("seed collaborator"); + }); + + BuiltRegistry { + _dir: dir, + meta_path, + layout, + dids, + } +} + +pub struct BuiltRoster { + _dir: TempDir, + meta_path: PathBuf, + layout: Layout, + repo: RepoDid, +} + +impl BuiltRoster { + pub fn index(&self) -> Index { + Index::new(&self.meta_path, self.layout.clone()) + } + pub fn repo(&self) -> &RepoDid { + &self.repo + } +} + +pub fn build_collaborator_roster(collaborators: RosterCount) -> BuiltRoster { + let dir = tempfile::tempdir().expect("tempdir"); + let meta_path = dir.path().join("meta.git"); + Repo::create(&meta_path).expect("create meta"); + let layout = Layout::new(dir.path().join("repos")); + let signer = K256Signer::generate(&SeededEntropy::new(9)); + + seed_members(&meta_path, &signer); + + let repo = synthetic_repo_did(0); + let git = layout.create(&repo).expect("create repo dir"); + let store = CobStore::new(&git); + let home = CobHome::from(&repo); + let object = store + .create( + &home, + &CollaboratorsChange::Add(grant(2)), + &signer, + UnixSeconds::new(GENESIS_SECONDS), + ) + .expect("seed collaborator") + .object; + (1..collaborators.get()).for_each(|index| { + store + .update( + &home, + object, + &CollaboratorsChange::Add(grant(u64::from(index) + 2)), + &signer, + UnixSeconds::new(GENESIS_SECONDS + i64::from(index)), + ) + .expect("add collaborator"); + }); + + BuiltRoster { + _dir: dir, + meta_path, + layout, + repo, + } +} + +pub struct BuiltRegistryWriter { + _dir: TempDir, + meta_path: PathBuf, + object: CobId, + signer: K256Signer, +} + +impl BuiltRegistryWriter { + pub fn probe(&self) { + let meta = Repo::open(&self.meta_path).expect("reopen meta"); + let store = CobStore::new(&meta); + register_repo( + &store, + &knot_home(), + self.object, + registration(0), + &self.signer, + UnixSeconds::new(GENESIS_SECONDS), + ) + .expect("idempotent re-register folds registry"); + } + + pub fn full_fold(&self) { + let meta = Repo::open(&self.meta_path).expect("reopen meta"); + let store = CobStore::new(&meta); + store + .get::(self.object) + .expect("full fold of registry"); + } +} + +pub fn build_registry_checkpointed(repos: RepoCount) -> BuiltRegistryWriter { + let dir = tempfile::tempdir().expect("tempdir"); + let meta_path = dir.path().join("meta.git"); + Repo::create(&meta_path).expect("create meta"); + let signer = K256Signer::generate(&SeededEntropy::new(13)); + + let meta = Repo::open(&meta_path).expect("open meta"); + let store = CobStore::new(&meta); + let object = store + .create( + &knot_home(), + &RegistryChange::Register(registration(0)), + &signer, + UnixSeconds::new(GENESIS_SECONDS), + ) + .expect("create registry") + .object; + (1..repos.get()).for_each(|index| { + register_repo( + &store, + &knot_home(), + object, + registration(index), + &signer, + UnixSeconds::new(GENESIS_SECONDS + index as i64), + ) + .expect("register repo"); + }); + + BuiltRegistryWriter { + _dir: dir, + meta_path, + object, + signer, + } +} + +pub struct BuiltMembersWriter { + _dir: TempDir, + meta_path: PathBuf, + object: CobId, + signer: K256Signer, +} + +impl BuiltMembersWriter { + pub fn probe(&self) { + let meta = Repo::open(&self.meta_path).expect("reopen meta"); + let store = CobStore::new(&meta); + store + .update_maybe_checkpointed::( + &knot_home(), + self.object, + &self.signer, + UnixSeconds::new(GENESIS_SECONDS), + |roster| { + Ok(if roster.contains(&grant(0).subject) { + None + } else { + Some(MembersChange::Add(grant(0))) + }) + }, + ) + .expect("idempotent re-add folds the bounded suffix"); + } + + pub fn full_fold(&self) { + let meta = Repo::open(&self.meta_path).expect("reopen meta"); + let store = CobStore::new(&meta); + store + .get::(self.object) + .expect("full fold of members"); + } +} + +pub fn build_members_checkpointed(members: RosterCount) -> BuiltMembersWriter { + let dir = tempfile::tempdir().expect("tempdir"); + let meta_path = dir.path().join("meta.git"); + Repo::create(&meta_path).expect("create meta"); + let signer = K256Signer::generate(&SeededEntropy::new(17)); + + let meta = Repo::open(&meta_path).expect("open meta"); + let store = CobStore::new(&meta); + let object = store + .create( + &knot_home(), + &MembersChange::Add(grant(0)), + &signer, + UnixSeconds::new(GENESIS_SECONDS), + ) + .expect("create members") + .object; + (1..members.get()).for_each(|index| { + add_member( + &store, + &knot_home(), + object, + grant(u64::from(index) + 1), + &signer, + UnixSeconds::new(GENESIS_SECONDS + i64::from(index)), + ) + .expect("add member"); + }); + + BuiltMembersWriter { + _dir: dir, + meta_path, + object, + signer, + } +} + +pub struct BuiltLinearCob { + _dir: TempDir, + repo: Repo, + object: CobId, +} + +impl BuiltLinearCob { + pub fn fold(&self) -> usize { + let store = CobStore::new(&self.repo); + store + .get::(self.object) + .expect("fold registry") + .state() + .len() + } +} + +pub fn build_linear_cob(changes: ChangeCount) -> BuiltLinearCob { + let dir = tempfile::tempdir().expect("tempdir"); + let meta_path = dir.path().join("meta.git"); + Repo::create(&meta_path).expect("create meta"); + let signer = K256Signer::generate(&SeededEntropy::new(11)); + + let meta = Repo::open(&meta_path).expect("open meta"); + let store = CobStore::new(&meta); + let object = store + .create( + &knot_home(), + &RegistryChange::Register(registration(0)), + &signer, + UnixSeconds::new(GENESIS_SECONDS), + ) + .expect("create registry") + .object; + (1..changes.get()).for_each(|index| { + store + .update( + &knot_home(), + object, + &RegistryChange::Register(registration(u64::from(index))), + &signer, + UnixSeconds::new(GENESIS_SECONDS + i64::from(index)), + ) + .expect("append change"); + }); + + BuiltLinearCob { + _dir: dir, + repo: Repo::open(&meta_path).expect("reopen meta"), + object, + } +} diff --git a/knot2/crates/knot-bench/src/latency.rs b/knot2/crates/knot-bench/src/latency.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-bench/src/latency.rs @@ -0,0 +1,32 @@ +use std::time::Duration; + +use knot_index::Index; +use knot_types::RepoDid; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct OpenLatency(Duration); + +impl OpenLatency { + pub fn micros(value: u64) -> Self { + Self(Duration::from_micros(value)) + } + pub fn zero() -> Self { + Self(Duration::ZERO) + } + fn stall(self) { + if !self.0.is_zero() { + std::thread::sleep(self.0); + } + } +} + +pub fn replay_boot(index: &Index, repos: &[RepoDid], per_open: OpenLatency) { + index.refresh_members().expect("fold members"); + index.refresh_registry().expect("fold registry"); + repos.iter().for_each(|repo| { + per_open.stall(); + index + .refresh_collaborators(repo) + .expect("fabric-boot repo must fold, otherwise bench measures nothing"); + }); +} diff --git a/knot2/crates/knot-bench/src/lib.rs b/knot2/crates/knot-bench/src/lib.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-bench/src/lib.rs @@ -0,0 +1,11 @@ +mod fixtures; +mod latency; + +pub use fixtures::{ + BuiltHistory, BuiltLinearCob, BuiltMembersWriter, BuiltRefs, BuiltRegistry, + BuiltRegistryWriter, BuiltRoster, ChangeCount, ChurnCount, CommitCount, HistorySpec, PathCount, + RefCount, RepoCount, RosterCount, build_collaborator_roster, build_history, build_linear_cob, + build_many_refs, build_members_checkpointed, build_registry, build_registry_checkpointed, + write_history, +}; +pub use latency::{OpenLatency, replay_boot}; diff --git a/knot2/crates/knot-bench/tests/boot.rs b/knot2/crates/knot-bench/tests/boot.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-bench/tests/boot.rs @@ -0,0 +1,26 @@ +use knot_bench::{RepoCount, build_registry}; +use knot_index::Resolved; + +#[test] +fn the_registry_fixture_boots_without_folding_collaborators() { + let registry = build_registry(RepoCount::new(8)); + let index = registry.index(); + index.rebuild().unwrap(); + + registry.dids().iter().for_each(|repo| { + assert_eq!( + index.is_collaborator(repo, &knot_types::AccountDid::new("did:plc:nel").unwrap()), + Resolved::Warming, + "rebuild must not fold any collaborator COB, so every repo reads warming until first access" + ); + }); + + let first = ®istry.dids()[0]; + index.ensure_collaborators(first).unwrap(); + assert!( + !index + .is_collaborator(first, &knot_types::AccountDid::new("did:plc:nel").unwrap()) + .is_warming(), + "repo folds on first access" + ); +} diff --git a/knot2/crates/knot-bench/tests/gate.rs b/knot2/crates/knot-bench/tests/gate.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-bench/tests/gate.rs @@ -0,0 +1,77 @@ +#![cfg(feature = "instrument")] + +use knot_bench::{ChurnCount, CommitCount, HistorySpec, PathCount, build_history}; +use knot_git::instrument::measure; +use knot_git::{Filter, PackBudget}; +use knot_pack::upload_pack; +use knot_types::Oid; + +fn gate_spec() -> HistorySpec { + HistorySpec { + commits: CommitCount::new(32), + paths: PathCount::new(64), + churn: ChurnCount::new(4), + } +} + +const SELECTION_ODB_READS: u64 = 129; +const SERVER_FETCH_ODB_READS: u64 = 2; + +fn pkt(payload: &[u8]) -> Vec { + let mut out = format!("{:04x}", payload.len() + 4).into_bytes(); + out.extend_from_slice(payload); + out +} + +fn fetch_request(want: Oid) -> Vec { + let mut request = pkt(b"command=fetch\n"); + request.extend_from_slice(b"0001"); + request.extend(pkt(format!("want {}\n", want.to_hex()).as_bytes())); + request.extend(pkt(b"done\n")); + request.extend_from_slice(b"0000"); + request +} + +#[test] +fn a_single_selection_walk_has_an_exact_odb_read_count() { + let history = build_history(gate_spec()); + let tips = history.tips(); + let (_selection, reads) = measure(|| { + history + .repo() + .select_pack_objects_filtered(&tips, &[], Filter::None, PackBudget::unbounded()) + .unwrap() + }); + assert_eq!( + reads.get(), + SELECTION_ODB_READS, + "the selection walk made {} explicit object loads through Repo::load_object. \ + The gate pins this at {SELECTION_ODB_READS}. A lower count means the single-pass \ + commit-walk fix landed. A higher count is a regression. The counter records loads \ + on the calling thread only. gix's internal rev-walk decodes never reach load_object \ + and stay uncounted. Update the constant only when the change is deliberate", + reads.get() + ); +} + +#[test] +fn the_upload_pack_server_path_has_an_exact_odb_read_count() { + let history = build_history(gate_spec()); + let walk = history.repo().rev_walk(&history.tips(), &[]).unwrap(); + let hidden = walk + .iter() + .copied() + .find(|commit| *commit != history.tip()) + .expect("multi-commit history has a non-tip commit to want"); + let request = fetch_request(hidden); + let (_response, reads) = measure(|| upload_pack(history.repo(), &request).unwrap()); + assert_eq!( + reads.get(), + SERVER_FETCH_ODB_READS, + "server fetch made {} Repo::load_object calls, gate pins {SERVER_FETCH_ODB_READS}. \ + No-haves fetch enumerates inside gix, so only the want check and root peel reach \ + load_object. A count near the old manual-walk figure means the full-clone fast path \ + stopped firing", + reads.get() + ); +} diff --git a/knot2/crates/knot-bench/tests/members_gate.rs b/knot2/crates/knot-bench/tests/members_gate.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-bench/tests/members_gate.rs @@ -0,0 +1,44 @@ +#![cfg(feature = "instrument")] + +use knot_bench::{RosterCount, build_members_checkpointed}; +use knot_cob::instrument::measure; + +#[test] +fn a_checkpointed_member_write_reads_a_bounded_object_set() { + let small = build_members_checkpointed(RosterCount::new(512)); + let large = build_members_checkpointed(RosterCount::new(1024)); + + let (_, full_small) = measure(|| small.full_fold()); + let (_, full_large) = measure(|| large.full_fold()); + let (_, write_small) = measure(|| small.probe()); + let (_, write_large) = measure(|| large.probe()); + + assert_eq!( + full_small.get(), + 2 * 512, + "full fold reads every change's commit and payload, so twice the member count" + ); + assert_eq!( + full_large.get(), + 2 * 1024, + "full fold the checkpoint replaces is linear in member count" + ); + assert_eq!( + write_small.get(), + write_large.get(), + "a checkpointed member write reads only the bounded suffix, the same object count at \ + 512 members as at 1024" + ); + assert!( + write_large.get() <= 2 * 256, + "per-write read set is bounded by twice the snapshot stride, not the member count, \ + was {}", + write_large.get() + ); + assert!( + write_large.get() < full_large.get() / 3, + "checkpoint cuts per-write object reads far below the full fold, {} vs {}", + write_large.get(), + full_large.get() + ); +} diff --git a/knot2/crates/knot-bench/tests/registry_gate.rs b/knot2/crates/knot-bench/tests/registry_gate.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-bench/tests/registry_gate.rs @@ -0,0 +1,44 @@ +#![cfg(feature = "instrument")] + +use knot_bench::{RepoCount, build_registry_checkpointed}; +use knot_cob::instrument::measure; + +#[test] +fn a_checkpointed_registry_write_reads_a_bounded_object_set() { + let small = build_registry_checkpointed(RepoCount::new(512)); + let large = build_registry_checkpointed(RepoCount::new(1024)); + + let (_, full_small) = measure(|| small.full_fold()); + let (_, full_large) = measure(|| large.full_fold()); + let (_, write_small) = measure(|| small.probe()); + let (_, write_large) = measure(|| large.probe()); + + assert_eq!( + full_small.get(), + 2 * 512, + "full fold reads every change's commit and payload, so twice the change count" + ); + assert_eq!( + full_large.get(), + 2 * 1024, + "full fold the checkpoint replaces is linear in repo count" + ); + assert_eq!( + write_small.get(), + write_large.get(), + "checkpointed write reads only the bounded suffix, the same object count at 512 \ + repos as at 1024" + ); + assert!( + write_large.get() <= 2 * 256, + "per-write read set is bounded by twice the snapshot stride, not the repo count, \ + was {}", + write_large.get() + ); + assert!( + write_large.get() < full_large.get() / 3, + "checkpoint cuts per-write object reads far below the full fold, {} vs {}", + write_large.get(), + full_large.get() + ); +} diff --git a/knot2/crates/knot-cache/src/expiring.rs b/knot2/crates/knot-cache/src/expiring.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-cache/src/expiring.rs @@ -0,0 +1,388 @@ +use std::collections::HashMap; +use std::collections::hash_map::Entry as Slot; +use std::hash::Hash; +use std::sync::Mutex; + +use knot_runtime::UnixMicros; + +knot_types::scalar_newtype! { + pub struct GroupQuota(usize); + pub struct TotalQuota(usize); +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Quotas { + pub per_group: GroupQuota, + pub total: TotalQuota, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Rejected { + Total, + Group, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Admitted { + Inserted, + Occupied(V), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Occupancy { + Keep, + Extend, +} + +struct Entry { + group: G, + value: V, + expires_at: UnixMicros, +} + +struct Inner { + entries: HashMap>, + group_counts: HashMap, +} + +pub struct Expiring { + inner: Mutex>, + quotas: Quotas, +} + +impl Expiring +where + K: Eq + Hash + Clone, + G: Eq + Hash + Clone, +{ + pub fn new(quotas: Quotas) -> Self { + Self { + inner: Mutex::new(Inner { + entries: HashMap::new(), + group_counts: HashMap::new(), + }), + quotas, + } + } + + fn lock(&self) -> std::sync::MutexGuard<'_, Inner> { + self.inner + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + pub fn prune(&self, now: UnixMicros) -> Vec { + prune_locked(&mut self.lock(), now) + } + + pub fn admit( + &self, + key: K, + group: G, + value: V, + expires_at: UnixMicros, + now: UnixMicros, + ) -> Result, Rejected> + where + V: Clone + PartialEq, + { + self.enter(key, group, value, expires_at, now, Occupancy::Keep) + } + + pub fn admit_or_renew( + &self, + key: K, + group: G, + value: V, + expires_at: UnixMicros, + now: UnixMicros, + ) -> Result, Rejected> + where + V: Clone + PartialEq, + { + self.enter(key, group, value, expires_at, now, Occupancy::Extend) + } + + fn enter( + &self, + key: K, + group: G, + value: V, + expires_at: UnixMicros, + now: UnixMicros, + occupied: Occupancy, + ) -> Result, Rejected> + where + V: Clone + PartialEq, + { + let mut inner = self.lock(); + let key = match inner.entries.entry(key) { + Slot::Occupied(mut held) if held.get().expires_at > now => { + let entry = held.get_mut(); + if occupied == Occupancy::Extend && entry.value == value { + entry.expires_at = expires_at; + } + return Ok(Admitted::Occupied(entry.value.clone())); + } + Slot::Occupied(held) => { + let (key, entry) = held.remove_entry(); + release_group(&mut inner.group_counts, &entry.group); + key + } + Slot::Vacant(free) => free.into_key(), + }; + // This is lazy on purpose! + // Pruning traverses every entry so we shouldn't do it on every + // admission, better to do this O(1) quota check. + if self.rejection(&inner, &group).is_some() { + prune_locked(&mut inner, now); + } + match self.rejection(&inner, &group) { + Some(rejected) => Err(rejected), + None => { + *inner.group_counts.entry(group.clone()).or_insert(0) += 1; + inner.entries.insert( + key, + Entry { + group, + value, + expires_at, + }, + ); + Ok(Admitted::Inserted) + } + } + } + + fn rejection(&self, inner: &Inner, group: &G) -> Option { + let held = inner.group_counts.get(group).copied().unwrap_or(0); + match ( + held >= self.quotas.per_group.get(), + inner.entries.len() >= self.quotas.total.get(), + ) { + (true, _) => Some(Rejected::Group), + (_, true) => Some(Rejected::Total), + (false, false) => None, + } + } + + pub fn remove(&self, key: &K) -> Option { + remove_locked(&mut self.lock(), key) + } + + pub fn get(&self, key: &K, now: UnixMicros) -> Option + where + V: Clone, + { + let inner = self.lock(); + inner + .entries + .get(key) + .filter(|entry| entry.expires_at > now) + .map(|entry| entry.value.clone()) + } + + pub fn len(&self) -> usize { + self.lock().entries.len() + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn group_len(&self, group: &G) -> usize { + self.lock().group_counts.get(group).copied().unwrap_or(0) + } +} + +fn prune_locked(inner: &mut Inner, now: UnixMicros) -> Vec +where + K: Eq + Hash + Clone, + G: Eq + Hash, +{ + let expired: Vec = inner + .entries + .iter() + .filter(|(_, entry)| entry.expires_at <= now) + .map(|(key, _)| key.clone()) + .collect(); + expired.iter().for_each(|key| { + remove_locked(inner, key); + }); + expired +} + +fn remove_locked(inner: &mut Inner, key: &K) -> Option +where + K: Eq + Hash, + G: Eq + Hash, +{ + let entry = inner.entries.remove(key)?; + release_group(&mut inner.group_counts, &entry.group); + Some(entry.value) +} + +fn release_group(counts: &mut HashMap, group: &G) { + if let Some(count) = counts.get_mut(group) { + *count = count.saturating_sub(1); + if *count == 0 { + counts.remove(group); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn at(micros: u64) -> UnixMicros { + UnixMicros::new(micros) + } + + fn store(per_group: usize, total: usize) -> Expiring<&'static str, &'static str, u8> { + Expiring::new(Quotas { + per_group: GroupQuota::new(per_group), + total: TotalQuota::new(total), + }) + } + + #[test] + fn a_slot_stays_occupied_until_it_expires_and_is_then_free_to_take_over() { + let store = store(8, 8); + assert_eq!( + store.admit("uni", "nel", 1, at(100), at(0)), + Ok(Admitted::Inserted) + ); + assert_eq!( + store.admit("uni", "olaren", 2, at(900), at(50)), + Ok(Admitted::Occupied(1)), + "the caller decides whether an occupied slot is a replay or a renewal" + ); + assert_eq!(store.get(&"uni", at(50)), Some(1)); + assert_eq!( + store.get(&"uni", at(100)), + None, + "a replay guard must keep the original expiry or a replayed token renews its own window" + ); + assert_eq!( + store.admit("uni", "olaren", 2, at(300), at(100)), + Ok(Admitted::Inserted), + "expiry is inclusive so a slot expiring exactly now is available" + ); + assert_eq!( + store.group_len(&"nel"), + 0, + "the takeover releases the old group's count" + ); + assert_eq!(store.group_len(&"olaren"), 1); + } + + #[test] + fn admit_or_renew_extends_only_the_current_holder() { + let store = store(8, 8); + store + .admit_or_renew("uni", "nel", 1, at(100), at(0)) + .unwrap(); + assert_eq!( + store.admit_or_renew("uni", "olaren", 2, at(900), at(50)), + Ok(Admitted::Occupied(1)) + ); + assert_eq!( + store.get(&"uni", at(150)), + None, + "a caller that isn't the holder mustn't extend the lease" + ); + store + .admit_or_renew("uni", "nel", 1, at(300), at(200)) + .unwrap(); + assert_eq!( + store.admit_or_renew("uni", "nel", 1, at(600), at(250)), + Ok(Admitted::Occupied(1)) + ); + assert_eq!( + store.get(&"uni", at(500)), + Some(1), + "renewing under the lock that read the entry leaves no window for a release \ + to drop the slot between the read and the renewal" + ); + } + + #[test] + fn each_quota_bounds_its_own_scope_and_names_itself_when_it_refuses() { + let store = store(1, 1); + store.admit("uni", "nel", 1, at(100), at(0)).unwrap(); + assert_eq!( + store.admit("kelp", "nel", 2, at(100), at(0)), + Err(Rejected::Group), + "a rejection names the group limit first because the caller can act on its own quota" + ); + assert_eq!( + store.admit("kelp", "olaren", 2, at(100), at(0)), + Err(Rejected::Total), + "a distinct group has its own budget but still shares the total" + ); + assert_eq!( + store.admit("kelp", "olaren", 2, at(400), at(200)), + Ok(Admitted::Inserted), + "admit prunes the expired entry and takes the slot it freed" + ); + } + + #[test] + fn releasing_a_slot_by_expiry_or_by_hand_frees_its_group_budget() { + let store = store(2, 8); + store.admit("uni", "nel", 7, at(100), at(0)).unwrap(); + store.admit("kelp", "nel", 8, at(500), at(0)).unwrap(); + assert_eq!(store.prune(at(200)), vec!["uni"]); + assert_eq!(store.len(), 1); + assert_eq!( + store.group_len(&"nel"), + 1, + "pruning decrements the group count rather than rebuilding it" + ); + assert_eq!(store.remove(&"kelp"), Some(8)); + assert_eq!(store.remove(&"kelp"), None); + assert_eq!( + store.group_len(&"nel"), + 0, + "removing the last entry of a group frees its budget" + ); + assert_eq!( + store.admit("whelk", "nel", 9, at(600), at(200)), + Ok(Admitted::Inserted) + ); + } + + #[test] + fn concurrent_admissions_at_the_quota_keep_group_counts_equal_to_what_is_stored() { + let store = std::sync::Arc::new(Expiring::new(Quotas { + per_group: GroupQuota::new(2), + total: TotalQuota::new(3), + })); + let keys = ["uni", "kelp", "whelk", "clam", "conch", "limpet"]; + let groups = ["nel", "olaren"]; + let threads: Vec<_> = (0..8u64) + .map(|thread| { + let store = std::sync::Arc::clone(&store); + std::thread::spawn(move || { + (0..2_000u64).for_each(|round| { + let key = keys[(thread + round) as usize % keys.len()]; + let group = groups[(thread + round) as usize % groups.len()]; + let now = at(round * 10); + let _ = store.admit(key, group, 1u8, at(round * 10 + 40), now); + store.prune(now); + }); + }) + }) + .collect(); + threads + .into_iter() + .for_each(|thread| thread.join().unwrap()); + let counted: usize = groups.iter().map(|group| store.group_len(group)).sum(); + assert_eq!( + counted, + store.len(), + "a group count higher than what is stored locks its group out of every later admission" + ); + } +} diff --git a/knot2/crates/knot-cache/src/lib.rs b/knot2/crates/knot-cache/src/lib.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-cache/src/lib.rs @@ -0,0 +1,634 @@ +mod expiring; + +pub use expiring::{Admitted, Expiring, GroupQuota, Quotas, Rejected, TotalQuota}; + +use std::collections::{BTreeMap, HashMap}; +use std::future::Future; +use std::hash::Hash; +use std::sync::{Arc, Mutex, OnceLock, Weak}; +use std::time::Duration; + +use knot_runtime::{Clock, UnixMicros}; + +knot_types::scalar_newtype! { + pub struct EntryCount(u64); + pub struct Weight(u64); +} + +pub trait Cache: Send + Sync { + fn get(&self, key: &K) -> Option; + fn insert(&self, key: K, value: V); + fn invalidate(&self, key: &K); + fn invalidate_all(&self); + fn entry_count(&self) -> EntryCount; + fn weighted_size(&self) -> Weight; +} + +pub struct Untimed; + +impl Clock for Untimed { + fn now_unix_micros(&self) -> UnixMicros { + UnixMicros::new(0) + } +} + +pub struct Moka { + inner: moka::sync::Cache, +} + +impl Moka +where + K: Hash + Eq + Send + Sync + 'static, + V: Clone + Send + Sync + 'static, +{ + pub fn by_count(max_entries: EntryCount) -> Self { + Self { + inner: moka::sync::Cache::builder() + .max_capacity(max_entries.get()) + .build(), + } + } + + pub fn by_weight(max_weight: Weight, weigh: F) -> Self + where + F: Fn(&V) -> Weight + Send + Sync + 'static, + { + Self { + inner: moka::sync::Cache::builder() + .max_capacity(max_weight.get()) + .weigher(move |_key: &K, value: &V| weigh(value).get().min(u32::MAX as u64) as u32) + .build(), + } + } + + pub fn get_or_try_insert_with(&self, key: K, init: F) -> Result> + where + F: FnOnce() -> Result, + E: Send + Sync + 'static, + { + self.inner.try_get_with(key, init) + } +} + +impl Cache for Moka +where + K: Hash + Eq + Send + Sync + 'static, + V: Clone + Send + Sync + 'static, +{ + fn get(&self, key: &K) -> Option { + self.inner.get(key) + } + + fn insert(&self, key: K, value: V) { + self.inner.insert(key, value); + } + + fn invalidate(&self, key: &K) { + self.inner.invalidate(key); + } + + fn invalidate_all(&self) { + self.inner.invalidate_all(); + } + + fn entry_count(&self) -> EntryCount { + EntryCount::new(self.inner.entry_count()) + } + + fn weighted_size(&self) -> Weight { + Weight::new(self.inner.weighted_size()) + } +} + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +struct Tick(u64); + +impl Tick { + fn issue(&mut self) -> Tick { + let issued = *self; + self.0 = self.0.saturating_add(1); + issued + } +} + +struct Node { + value: V, + tick: Tick, + weight: u64, + expires_at: Option, +} + +struct LruInner { + by_key: HashMap>, + order: BTreeMap, + next: Tick, + total_weight: u64, +} + +type Weigh = Arc Weight + Send + Sync>; + +pub struct Lru { + inner: Mutex>, + max_entries: Option, + max_weight: Option, + weigh: Option>, + ttl: Option, + clock: C, +} + +impl Lru +where + K: Hash + Eq + Clone, + V: Clone, +{ + pub fn by_count(max_entries: EntryCount) -> Self { + Self::build(Some(max_entries), None, None, None, Untimed) + } + + pub fn by_weight(max_weight: Weight, weigh: F) -> Self + where + F: Fn(&V) -> Weight + Send + Sync + 'static, + { + Self::build(None, Some(max_weight), Some(Arc::new(weigh)), None, Untimed) + } +} + +impl Lru +where + K: Hash + Eq + Clone, + V: Clone, + C: Clock, +{ + pub fn by_count_with_ttl(max_entries: EntryCount, ttl: Duration, clock: C) -> Self { + Self::build(Some(max_entries), None, None, Some(ttl), clock) + } + + pub fn by_weight_with_ttl(max_weight: Weight, ttl: Duration, clock: C, weigh: F) -> Self + where + F: Fn(&V) -> Weight + Send + Sync + 'static, + { + Self::build( + None, + Some(max_weight), + Some(Arc::new(weigh)), + Some(ttl), + clock, + ) + } + + pub fn with_entry_cap(mut self, max_entries: EntryCount) -> Self { + self.max_entries = Some(max_entries); + self + } + + fn build( + max_entries: Option, + max_weight: Option, + weigh: Option>, + ttl: Option, + clock: C, + ) -> Self { + Self { + inner: Mutex::new(LruInner { + by_key: HashMap::new(), + order: BTreeMap::new(), + next: Tick(0), + total_weight: 0, + }), + max_entries, + max_weight, + weigh, + ttl, + clock, + } + } + + fn weight_of(&self, value: &V) -> u64 { + self.weigh.as_ref().map_or(1, |weigh| weigh(value).get()) + } + + fn lock(&self) -> std::sync::MutexGuard<'_, LruInner> { + self.inner + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } +} + +impl Cache for Lru +where + K: Hash + Eq + Clone + Send + Sync + 'static, + V: Clone + Send + Sync + 'static, + C: Clock, +{ + fn get(&self, key: &K) -> Option { + let mut guard = self.lock(); + let inner = &mut *guard; + let (stale, expires_at) = inner + .by_key + .get(key) + .map(|node| (node.tick, node.expires_at))?; + if expires_at.is_some_and(|at| at.get() <= self.clock.now_unix_micros().get()) { + if let Some(node) = inner.by_key.remove(key) { + inner.total_weight = inner.total_weight.saturating_sub(node.weight); + } + inner.order.remove(&stale); + return None; + } + let fresh = inner.next.issue(); + inner.order.remove(&stale); + inner.order.insert(fresh, key.clone()); + let node = inner.by_key.get_mut(key).expect("hit is still present"); + node.tick = fresh; + Some(node.value.clone()) + } + + fn insert(&self, key: K, value: V) { + let expires_at = self.ttl.map(|ttl| { + UnixMicros::new( + self.clock + .now_unix_micros() + .get() + .saturating_add(ttl.as_micros() as u64), + ) + }); + let weight = self.weight_of(&value); + let mut guard = self.lock(); + let inner = &mut *guard; + if let Some(previous) = inner.by_key.remove(&key) { + inner.order.remove(&previous.tick); + inner.total_weight = inner.total_weight.saturating_sub(previous.weight); + } + let fresh = inner.next.issue(); + inner.order.insert(fresh, key.clone()); + inner.total_weight = inner.total_weight.saturating_add(weight); + inner.by_key.insert( + key, + Node { + value, + tick: fresh, + weight, + expires_at, + }, + ); + while self + .max_entries + .is_some_and(|max| inner.by_key.len() as u64 > max.get()) + || self + .max_weight + .is_some_and(|max| inner.total_weight > max.get()) + { + let Some((_, evicted)) = inner.order.pop_first() else { + break; + }; + if let Some(node) = inner.by_key.remove(&evicted) { + inner.total_weight = inner.total_weight.saturating_sub(node.weight); + } + } + } + + fn invalidate(&self, key: &K) { + let mut guard = self.lock(); + if let Some(node) = guard.by_key.remove(key) { + guard.order.remove(&node.tick); + guard.total_weight = guard.total_weight.saturating_sub(node.weight); + } + } + + fn invalidate_all(&self) { + let mut guard = self.lock(); + guard.by_key.clear(); + guard.order.clear(); + guard.total_weight = 0; + } + + fn entry_count(&self) -> EntryCount { + EntryCount::new(self.lock().by_key.len() as u64) + } + + fn weighted_size(&self) -> Weight { + Weight::new(self.lock().total_weight) + } +} + +pub struct Noop; + +impl Cache for Noop +where + K: Send + Sync + 'static, + V: Send + Sync + 'static, +{ + fn get(&self, _key: &K) -> Option { + None + } + + fn insert(&self, _key: K, _value: V) {} + + fn invalidate(&self, _key: &K) {} + + fn invalidate_all(&self) {} + + fn entry_count(&self) -> EntryCount { + EntryCount::new(0) + } + + fn weighted_size(&self) -> Weight { + Weight::new(0) + } +} + +pub struct Filled { + pub value: V, + pub fresh: bool, +} + +type DeterministicHasher = std::hash::BuildHasherDefault; + +pub trait AsyncCache: Send + Sync { + fn get(&self, key: &K) -> impl Future> + Send; + fn entry_count(&self) -> EntryCount; +} + +pub struct MokaFuture { + inner: moka::future::Cache, +} + +impl MokaFuture +where + K: Hash + Eq + Send + Sync + 'static, + V: Clone + Send + Sync + 'static, +{ + pub fn by_count(max_entries: EntryCount) -> Self { + Self { + inner: moka::future::Cache::builder() + .max_capacity(max_entries.get()) + .build_with_hasher(DeterministicHasher::default()), + } + } + + pub async fn get_or_fill_if(&self, key: K, refill_if: P, fill: Fut) -> Filled + where + Fut: Future + Send, + P: FnMut(&V) -> bool + Send, + { + let entry = self + .inner + .entry(key) + .or_insert_with_if(fill, refill_if) + .await; + Filled { + fresh: entry.is_fresh(), + value: entry.into_value(), + } + } + + pub async fn run_pending_tasks(&self) { + self.inner.run_pending_tasks().await; + } +} + +impl AsyncCache for MokaFuture +where + K: Hash + Eq + Send + Sync + 'static, + V: Clone + Send + Sync + 'static, +{ + fn get(&self, key: &K) -> impl Future> + Send { + self.inner.get(key) + } + + fn entry_count(&self) -> EntryCount { + EntryCount::new(self.inner.entry_count()) + } +} + +pub trait Reclaimable: Send + Sync { + fn footprint(&self) -> Weight; + fn reclaim(&self); +} + +impl Reclaimable for Moka +where + K: Hash + Eq + Send + Sync + 'static, + V: Clone + Send + Sync + 'static, +{ + fn footprint(&self) -> Weight { + self.weighted_size() + } + + fn reclaim(&self) { + self.invalidate_all(); + } +} + +impl Reclaimable for Lru +where + K: Hash + Eq + Clone + Send + Sync + 'static, + V: Clone + Send + Sync + 'static, + C: Clock, +{ + fn footprint(&self) -> Weight { + self.weighted_size() + } + + fn reclaim(&self) { + self.invalidate_all(); + } +} + +#[derive(Default)] +struct Registry { + caches: Mutex>>, +} + +impl Registry { + fn lock(&self) -> std::sync::MutexGuard<'_, Vec>> { + self.caches + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + fn register(&self, cache: &Arc) { + let erased: Arc = cache.clone(); + let weak = Arc::downgrade(&erased); + let mut caches = self.lock(); + caches.retain(|entry| entry.strong_count() > 0); + caches.push(weak); + } + + fn reclaim_largest(&self) -> Option { + let largest = self + .lock() + .iter() + .filter_map(Weak::upgrade) + .max_by_key(|cache| cache.footprint().get())?; + let freed = largest.footprint(); + (freed.get() > 0).then(|| { + largest.reclaim(); + freed + }) + } +} + +fn global_registry() -> &'static Registry { + static REGISTRY: OnceLock = OnceLock::new(); + REGISTRY.get_or_init(Registry::default) +} + +pub fn register(cache: &Arc) { + global_registry().register(cache); +} + +pub fn reclaim_largest() -> Option { + global_registry().reclaim_largest() +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use knot_runtime::ManualClock; + + use super::*; + + #[test] + fn a_no_ttl_cache_retains_until_invalidated() { + let cache: Moka = Moka::by_count(EntryCount::new(16)); + cache.insert(1, 9); + cache.insert(2, 10); + assert_eq!(cache.get(&1), Some(9)); + cache.invalidate(&1); + assert_eq!(cache.get(&1), None); + assert_eq!(cache.get(&2), Some(10)); + } + + #[test] + fn the_same_clock_sequence_yields_the_same_hits_and_misses() { + let run = || { + let clock = Arc::new(ManualClock::new(UnixMicros::new(0))); + let cache: Lru = Lru::by_count_with_ttl( + EntryCount::new(16), + Duration::from_secs(5), + Arc::clone(&clock), + ); + cache.insert(1, 100); + let before = cache.get(&1); + clock.advance(Duration::from_secs(6)); + let after = cache.get(&1); + (before, after) + }; + assert_eq!(run(), run()); + } + + #[test] + fn get_or_try_insert_with_serves_the_first_value_without_recomputing() { + let cache: Moka = Moka::by_count(EntryCount::new(16)); + let first = cache.get_or_try_insert_with(1, || Ok::(7)); + assert_eq!(first.unwrap(), 7); + let second = cache.get_or_try_insert_with(1, || Ok::(99)); + assert_eq!(second.unwrap(), 7); + } + + #[test] + fn get_or_try_insert_with_propagates_the_error_and_stores_nothing() { + let cache: Moka = Moka::by_count(EntryCount::new(16)); + let failed = cache.get_or_try_insert_with(1, || Err::(9)); + assert_eq!(*failed.unwrap_err(), 9); + assert_eq!(cache.get(&1), None); + } + + #[test] + fn the_lru_evicts_the_least_recently_used_key() { + let cache: Lru = Lru::by_count(EntryCount::new(3)); + cache.insert(0, 0); + cache.insert(1, 1); + cache.insert(2, 2); + assert_eq!(cache.get(&0), Some(0)); + cache.insert(3, 3); + assert_eq!( + cache.get(&1), + None, + "the least recently used key is evicted" + ); + assert_eq!(cache.get(&0), Some(0), "the touched key survives"); + assert_eq!(cache.get(&3), Some(3)); + assert_eq!(cache.entry_count().get(), 3); + } + + #[test] + fn the_weighted_lru_evicts_oldest_until_it_fits_the_byte_budget() { + let cache: Lru> = Lru::by_weight(Weight::new(8), |value: &Vec| { + Weight::new(value.len() as u64) + }); + cache.insert(0, vec![0u8; 5]); + cache.insert(1, vec![0u8; 5]); + assert_eq!(cache.get(&0), None, "the oldest entry is evicted to fit"); + assert_eq!(cache.get(&1), Some(vec![0u8; 5])); + assert_eq!(cache.weighted_size().get(), 5); + } + + #[test] + fn the_entry_cap_bounds_a_flood_of_zero_weight_entries() { + let cache: Lru> = Lru::by_weight(Weight::new(1_000_000), |value: &Vec| { + Weight::new(value.len() as u64) + }) + .with_entry_cap(EntryCount::new(4)); + (0..64).for_each(|nonce| cache.insert(nonce, Vec::new())); + assert!(cache.entry_count().get() <= 4); + } + + #[test] + fn the_lru_expires_entries_on_the_injected_clock() { + let clock = Arc::new(ManualClock::new(UnixMicros::new(0))); + let cache: Lru = Lru::by_count_with_ttl( + EntryCount::new(8), + Duration::from_secs(1), + Arc::clone(&clock), + ); + cache.insert(1, 5); + assert_eq!(cache.get(&1), Some(5)); + clock.advance(Duration::from_secs(2)); + assert_eq!(cache.get(&1), None); + assert_eq!(cache.entry_count().get(), 0); + } + + #[test] + fn the_governor_sheds_the_largest_registered_cache() { + let weigh = |value: &Vec| Weight::new(value.len() as u64); + let small: Arc>> = Arc::new(Lru::by_weight(Weight::new(1_000_000), weigh)); + let big: Arc>> = Arc::new(Lru::by_weight(Weight::new(1_000_000), weigh)); + small.insert(0, vec![0u8; 10]); + big.insert(0, vec![0u8; 100]); + let registry = Registry::default(); + registry.register(&small); + registry.register(&big); + let freed = registry + .reclaim_largest() + .expect("a registered cache is shed"); + assert_eq!(freed.get(), 100, "the largest footprint is reclaimed"); + assert_eq!(big.entry_count().get(), 0, "the largest cache is emptied"); + assert_eq!( + small.entry_count().get(), + 1, + "the smaller cache is untouched" + ); + } + + #[test] + fn a_registered_cache_with_nothing_to_reclaim_is_not_shed() { + let registry = Registry::default(); + let cache: Arc>> = Arc::new(Lru::by_count(EntryCount::new(8))); + registry.register(&cache); + assert_eq!( + registry.reclaim_largest(), + None, + "an empty registered cache reports nothing to shed" + ); + } + + #[test] + fn the_noop_cache_never_retains() { + let cache: &dyn Cache = &Noop; + cache.insert(1, 2); + assert_eq!(cache.get(&1), None); + assert_eq!(cache.entry_count().get(), 0); + } +} diff --git a/knot2/crates/knot-cob/src/backend.rs b/knot2/crates/knot-cob/src/backend.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-cob/src/backend.rs @@ -0,0 +1,384 @@ +use std::collections::BTreeMap; + +use gix::bstr::{BStr, ByteSlice as _}; +use gix::date::Time; +use knot_git::Repo; +use knot_runtime::{Signature, Signer}; +use knot_types::{ActorId, ChangeId, CobId, Oid, RefName, TypeName, UnixSeconds}; + +use crate::change::{Change, CobHome, Payload}; +use crate::error::CobError; +use crate::graph::ChangeGraph; +use crate::object::HistoryModel; + +const COBS_PREFIX: &str = "refs/cobs/"; +const CHECKPOINTS_PREFIX: &str = "refs/cob-checkpoints/"; +const TYPE_HEADER: &str = "cob-type"; +const SIG_HEADER: &str = "cob-sig"; +const AUTHOR_HEADER: &str = "cob-author"; +const PAYLOAD_BLOB: &str = "payload"; +pub(crate) const MAX_GRAPH_CHANGES: usize = 100_000; +const REBUILD_CHANGE_BYTES: u64 = 2560; +const REBUILD_FOLD_DIVISOR: u64 = 4; +// two bazillion +const REBUILD_UNMEASURED_CHANGES: usize = 2_000_000; + +pub(crate) fn rebuild_change_limit_for(available: Option) -> usize { + let derived = match available { + Some(available) => { + usize::try_from(available.get() / REBUILD_FOLD_DIVISOR / REBUILD_CHANGE_BYTES) + .unwrap_or(usize::MAX) + } + None => REBUILD_UNMEASURED_CHANGES, + }; + derived.max(MAX_GRAPH_CHANGES) +} + +pub(crate) fn rebuild_graph_limit() -> usize { + rebuild_change_limit_for(knot_resource::available_bytes()) +} + +pub(crate) fn cob_ref_name(type_name: &TypeName, object: CobId) -> Result { + let raw = format!( + "{COBS_PREFIX}{}/{}", + type_name.as_str(), + object.oid().to_hex() + ); + RefName::new(raw.as_str()).map_err(|_| CobError::RefName(raw)) +} + +pub(crate) fn checkpoint_ref_name( + type_name: &TypeName, + object: CobId, +) -> Result { + let raw = format!( + "{CHECKPOINTS_PREFIX}{}/{}", + type_name.as_str(), + object.oid().to_hex() + ); + RefName::new(raw.as_str()).map_err(|_| CobError::RefName(raw)) +} + +pub fn parse_cob_ref(refname: &str) -> Option<(TypeName, CobId)> { + let (nsid, oid) = refname.strip_prefix(COBS_PREFIX)?.rsplit_once('/')?; + let type_name = TypeName::new(nsid).ok()?; + let object = Oid::from_hex(oid).ok().map(CobId::new)?; + Some((type_name, object)) +} + +pub(crate) fn resolve_tip( + repo: &Repo, + type_name: &TypeName, + object: CobId, +) -> Result, CobError> { + let name = cob_ref_name(type_name, object)?; + Ok(repo.find_ref(&name)?) +} + +pub(crate) fn list_objects(repo: &Repo, type_name: &TypeName) -> Result, CobError> { + let prefix = format!("{COBS_PREFIX}{}/", type_name.as_str()); + Ok(repo + .references()? + .into_iter() + .filter_map(|record| { + let rest = record.name.as_str().strip_prefix(&prefix)?; + Oid::from_hex(rest).ok().map(CobId::new) + }) + .collect()) +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn write_change( + home: &CobHome, + repo: &Repo, + type_name: &TypeName, + payload: &[u8], + parents: &[ChangeId], + object: Option, + signer: &dyn Signer, + timestamp: UnixSeconds, +) -> Result { + let git = repo.git(); + let payload_oid = git + .write_blob(payload) + .map_err(|error| CobError::Write(error.to_string()))? + .detach(); + let revision = git + .write_object(build_tree(payload_oid)) + .map_err(|error| CobError::Write(error.to_string()))? + .detach(); + let author = ActorId::from_secp256k1(signer.public_key().as_bytes()); + let revision_oid = Oid::from(revision); + let binding = crate::change::object_binding(parents, object); + let signing = crate::change::signing_bytes( + home, + revision_oid, + parents, + type_name, + &author, + timestamp, + binding, + ); + let signature = signer.sign(&signing); + if !crate::change::verify_signature( + home, + revision_oid, + parents, + type_name, + &author, + timestamp, + object, + signature.as_bytes(), + ) { + return Err(CobError::SelfCheck(type_name.clone())); + } + let commit = gix::objs::Commit { + tree: revision, + parents: parents + .iter() + .map(|parent| parent.oid().object_id()) + .collect(), + author: knot_identity(timestamp), + committer: knot_identity(timestamp), + encoding: None, + message: Vec::new().into(), + extra_headers: vec![ + (TYPE_HEADER.into(), type_name.as_str().into()), + (AUTHOR_HEADER.into(), author.as_str().into()), + ( + SIG_HEADER.into(), + knot_types::lowercase_hex(signature.as_bytes()).into(), + ), + ], + }; + let id = git + .write_object(commit) + .map_err(|error| CobError::Write(error.to_string()))? + .detach(); + Ok(ChangeId::new(Oid::from(id))) +} + +pub(crate) fn load_graph( + repo: &Repo, + type_name: &TypeName, + object: CobId, + history: HistoryModel, + limit: usize, +) -> Result<(ChangeGraph, ChangeId), CobError> { + let tip = resolve_tip(repo, type_name, object)?.ok_or(CobError::NoSuchObject(object))?; + let changes = collect(repo, ChangeId::new(tip), object, limit, None)?; + check_full_shape(&changes, object, history)?; + Ok((ChangeGraph::new(object, changes), ChangeId::new(tip))) +} + +pub(crate) fn check_full_shape( + changes: &BTreeMap, + object: CobId, + history: HistoryModel, +) -> Result<(), CobError> { + let root_id = ChangeId::new(object.oid()); + let root = changes.get(&root_id).ok_or(CobError::DetachedTip(object))?; + if !root.parents.is_empty() { + return Err(CobError::RootNotGenesis(object)); + } + if let Some(stray) = changes + .values() + .find(|change| change.id != root_id && change.parents.is_empty()) + { + return Err(CobError::MultipleRoots { + object, + stray: stray.id, + }); + } + check_no_forbidden_merge(changes, object, history) +} + +pub(crate) fn check_delta_shape( + changes: &BTreeMap, + object: CobId, + since: ChangeId, + history: HistoryModel, +) -> Result<(), CobError> { + check_no_forbidden_merge(changes, object, history)?; + let descends = changes + .values() + .any(|change| change.parents.contains(&since)); + if !changes.is_empty() && !descends { + return Err(CobError::DivergedTip { object, since }); + } + Ok(()) +} + +fn check_no_forbidden_merge( + changes: &BTreeMap, + object: CobId, + history: HistoryModel, +) -> Result<(), CobError> { + let forbidden_merge = (history == HistoryModel::Linear) + .then(|| changes.values().find(|change| change.parents.len() > 1)) + .flatten(); + match forbidden_merge { + Some(merge) => Err(CobError::ForkedHistory { + object, + change: merge.id, + }), + None => Ok(()), + } +} + +pub(crate) fn collect( + repo: &Repo, + tip: ChangeId, + object: CobId, + limit: usize, + stop: Option, +) -> Result, CobError> { + let mut frontier = vec![tip]; + let mut seen: BTreeMap = BTreeMap::new(); + let mut overflowed = false; + let walk = { + let mut step = || -> Option> { + let head = frontier.pop()?; + if Some(head) == stop || seen.contains_key(&head) { + return Some(Ok(())); + } + if seen.len() >= limit { + overflowed = true; + return None; + } + match read_change(repo, head) { + Ok(change) => { + frontier.extend(change.parents.iter().copied()); + seen.insert(head, change); + Some(Ok(())) + } + Err(error) => Some(Err(error)), + } + }; + std::iter::from_fn(&mut step).try_for_each(|outcome| outcome) + }; + walk?; + if overflowed { + return Err(CobError::HistoryTooLong(object)); + } + Ok(seen) +} + +pub(crate) fn read_change(repo: &Repo, id: ChangeId) -> Result { + let oid = id.oid(); + let malformed = |reason: String| CobError::MalformedChange { oid, reason }; + #[cfg(feature = "instrument")] + crate::instrument::record_read(); + let data = repo + .git() + .find_object(oid.object_id()) + .map_err(|error| malformed(error.to_string()))? + .detach() + .data; + let commit = gix::objs::CommitRef::from_bytes(&data, repo.git().object_hash()) + .map_err(|error| malformed(error.to_string()))?; + let revision = Oid::from(commit.tree()); + let parents = commit + .parents() + .map(|parent| ChangeId::new(Oid::from(parent))) + .collect(); + let timestamp = UnixSeconds::new( + commit + .time() + .map_err(|error| malformed(error.to_string()))? + .seconds, + ); + let type_raw = commit + .extra_headers() + .find(TYPE_HEADER) + .ok_or_else(|| malformed("missing cob-type header".into()))?; + let type_name = TypeName::new( + type_raw + .to_str() + .map_err(|error| malformed(error.to_string()))?, + ) + .map_err(|error| malformed(error.to_string()))?; + let author_raw = commit + .extra_headers() + .find(AUTHOR_HEADER) + .ok_or_else(|| malformed("missing cob-author header".into()))?; + let author = ActorId::new( + author_raw + .to_str() + .map_err(|error| malformed(error.to_string()))?, + ) + .map_err(|error| malformed(error.to_string()))?; + let signature = commit + .extra_headers() + .find(SIG_HEADER) + .ok_or_else(|| malformed("missing cob-sig header".into())) + .and_then(|raw| { + knot_types::decode_hex(raw).ok_or_else(|| malformed("cob-sig isn't valid hex".into())) + })?; + let payload = read_payload(repo, revision)?; + Ok(Change { + id, + revision, + parents, + type_name, + author, + signature: Signature::from_bytes(signature), + payload: Payload::new(payload), + timestamp, + }) +} + +fn read_payload(repo: &Repo, revision: Oid) -> Result, CobError> { + let malformed = |reason: String| CobError::MalformedChange { + oid: revision, + reason, + }; + #[cfg(feature = "instrument")] + crate::instrument::record_read(); + let data = repo + .git() + .find_object(revision.object_id()) + .map_err(|error| malformed(error.to_string()))? + .detach() + .data; + let tree = gix::objs::TreeRef::from_bytes(&data, repo.git().object_hash()) + .map_err(|error| malformed(error.to_string()))?; + let payload_oid = + entry_oid(&tree, PAYLOAD_BLOB).ok_or_else(|| malformed("missing payload blob".into()))?; + let payload = repo + .git() + .find_object(payload_oid) + .map_err(|error| malformed(error.to_string()))? + .detach() + .data; + Ok(payload) +} + +fn entry_oid(tree: &gix::objs::TreeRef<'_>, name: &str) -> Option { + tree.entries + .iter() + .find(|entry| entry.filename == BStr::new(name)) + .map(|entry| entry.oid.to_owned()) +} + +fn build_tree(payload_oid: gix::ObjectId) -> gix::objs::Tree { + gix::objs::Tree { + entries: vec![blob_entry(PAYLOAD_BLOB, payload_oid)], + } +} + +fn blob_entry(name: &str, oid: gix::ObjectId) -> gix::objs::tree::Entry { + gix::objs::tree::Entry { + mode: gix::objs::tree::EntryKind::Blob.into(), + filename: name.into(), + oid, + } +} + +fn knot_identity(time: UnixSeconds) -> gix::actor::Signature { + gix::actor::Signature { + name: "knot".into(), + email: "noreply@knot".into(), + time: Time::new(time.get(), 0), + } +} diff --git a/knot2/crates/knot-cob/src/change.rs b/knot2/crates/knot-cob/src/change.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-cob/src/change.rs @@ -0,0 +1,361 @@ +use k256::ecdsa::signature::Verifier as _; +use k256::ecdsa::{Signature as K256Signature, VerifyingKey}; +use knot_runtime::Signature; +use knot_types::crypto::PublicKey; +use knot_types::{ActorId, ChangeId, CobId, KnotId, Oid, RepoDid, TypeName, UnixSeconds}; +use serde::Serialize; +use serde::de::DeserializeOwned; + +use crate::error::PayloadError; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CobHome { + Repo(RepoDid), + Knot(KnotId), +} + +impl CobHome { + pub fn as_str(&self) -> &str { + match self { + CobHome::Repo(did) => did.as_str(), + CobHome::Knot(knot) => knot.as_str(), + } + } + + fn kind(&self) -> &'static str { + match self { + CobHome::Repo(_) => "repo", + CobHome::Knot(_) => "knot", + } + } +} + +impl From<&RepoDid> for CobHome { + fn from(did: &RepoDid) -> Self { + CobHome::Repo(did.clone()) + } +} + +impl From<&KnotId> for CobHome { + fn from(knot: &KnotId) -> Self { + CobHome::Knot(knot.clone()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Payload(Vec); + +impl Payload { + pub fn new(bytes: Vec) -> Self { + Self(bytes) + } + + pub fn as_bytes(&self) -> &[u8] { + &self.0 + } +} + +pub trait ChangePayload: Serialize + DeserializeOwned + Sized { + const TYPE: &'static str; + + fn type_name() -> TypeName { + TypeName::new(Self::TYPE).expect("ChangePayload::TYPE must be valid nsid") + } + + fn encode(&self) -> Result, PayloadError> { + serde_ipld_dagcbor::to_vec(self).map_err(|error| PayloadError::Encode(error.to_string())) + } + + fn decode(bytes: &[u8]) -> Result { + serde_ipld_dagcbor::from_slice(bytes) + .map_err(|error| PayloadError::Decode(error.to_string())) + } +} + +const SIGNING_CONTEXT: &str = "sh.tangled.knot.cob.change.v1"; + +#[derive(Serialize)] +struct SignedChange<'a> { + context: &'static str, + #[serde(rename = "homeKind")] + home_kind: &'static str, + home: &'a str, + revision: Oid, + parents: &'a [ChangeId], + #[serde(rename = "typeName")] + type_name: &'a TypeName, + author: &'a ActorId, + timestamp: UnixSeconds, + #[serde(skip_serializing_if = "Option::is_none")] + object: Option, +} + +pub(crate) fn signing_bytes( + home: &CobHome, + revision: Oid, + parents: &[ChangeId], + type_name: &TypeName, + author: &ActorId, + timestamp: UnixSeconds, + object: Option, +) -> Vec { + let view = SignedChange { + context: SIGNING_CONTEXT, + home_kind: home.kind(), + home: home.as_str(), + revision, + parents, + type_name, + author, + timestamp, + object, + }; + serde_ipld_dagcbor::to_vec(&view).expect("change signing view always encodes") +} + +pub(crate) fn object_binding(parents: &[ChangeId], object: Option) -> Option { + if parents.is_empty() { None } else { object } +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn verify_signature( + home: &CobHome, + revision: Oid, + parents: &[ChangeId], + type_name: &TypeName, + author: &ActorId, + timestamp: UnixSeconds, + object: Option, + signature: &[u8], +) -> bool { + let Ok(public) = PublicKey::decode(author.as_str()) else { + return false; + }; + let Ok(verifying) = public.to_k256() else { + return false; + }; + let Ok(signature) = K256Signature::from_slice(signature) else { + return false; + }; + let message = signing_bytes( + home, + revision, + parents, + type_name, + author, + timestamp, + object_binding(parents, object), + ); + VerifyingKey::from(&verifying) + .verify(&message, &signature) + .is_ok() +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Change { + pub id: ChangeId, + pub revision: Oid, + pub parents: Vec, + pub type_name: TypeName, + pub author: ActorId, + pub signature: Signature, + pub payload: Payload, + pub timestamp: UnixSeconds, +} + +impl Change { + pub fn payload(&self) -> &[u8] { + self.payload.as_bytes() + } + + pub fn sort_key(&self) -> (UnixSeconds, ChangeId) { + (self.timestamp, self.id) + } + + pub fn verify(&self, home: &CobHome, expected_author: &ActorId, object: Option) -> bool { + &self.author == expected_author + && verify_signature( + home, + self.revision, + &self.parents, + &self.type_name, + &self.author, + self.timestamp, + object, + self.signature.as_bytes(), + ) + } +} + +#[cfg(test)] +mod tests { + use knot_runtime::{K256Signer, SeededEntropy, Signer}; + + use super::*; + + fn type_name() -> TypeName { + TypeName::new("sh.tangled.test.tag").unwrap() + } + + fn cob_home() -> CobHome { + CobHome::from(&RepoDid::new("did:plc:squid").unwrap()) + } + + fn signed_change( + signer: &K256Signer, + revision: Oid, + parents: Vec, + timestamp: i64, + ) -> Change { + let author = ActorId::from_secp256k1(signer.public_key().as_bytes()); + let timestamp = UnixSeconds::new(timestamp); + let bytes = signing_bytes( + &cob_home(), + revision, + &parents, + &type_name(), + &author, + timestamp, + object_binding(&parents, None), + ); + Change { + id: ChangeId::new(Oid::null()), + revision, + parents, + type_name: type_name(), + author, + signature: signer.sign(&bytes), + payload: Payload::new(Vec::new()), + timestamp, + } + } + + #[test] + fn signing_bytes_stay_byte_stable() { + let author = { + let mut compressed = [0u8; 33]; + compressed[0] = 0x02; + compressed[1] = 0x09; + ActorId::from_secp256k1(&compressed) + }; + let parents = vec![ChangeId::new( + Oid::from_hex("2222222222222222222222222222222222222222").unwrap(), + )]; + let object = Some(CobId::new( + Oid::from_hex("3333333333333333333333333333333333333333").unwrap(), + )); + let bytes = signing_bytes( + &cob_home(), + Oid::from_hex("1111111111111111111111111111111111111111").unwrap(), + &parents, + &type_name(), + &author, + UnixSeconds::new(1_700_000_000), + object_binding(&parents, object), + ); + assert_eq!( + knot_types::lowercase_hex(&bytes), + "a964686f6d656d6469643a706c633a737175696466617574686f7278317a513373684e317652664257527847397234564c3251796466474e675955715a5a385836743971535359774b636a644a50666f626a65637478283333333333333333333333333333333333333333333333333333333333333333333333333333333367636f6e74657874781d73682e74616e676c65642e6b6e6f742e636f622e6368616e67652e763167706172656e74738178283232323232323232323232323232323232323232323232323232323232323232323232323232323268686f6d654b696e64647265706f687265766973696f6e78283131313131313131313131313131313131313131313131313131313131313131313131313131313168747970654e616d657373682e74616e676c65642e746573742e7461676974696d657374616d701a6553f100" + ); + } + + #[test] + fn verify_binds_the_author() { + let signer = K256Signer::generate(&SeededEntropy::new(13)); + let stranger = K256Signer::generate(&SeededEntropy::new(14)); + let revision = Oid::from_hex("6666666666666666666666666666666666666666").unwrap(); + let change = signed_change(&signer, revision, Vec::new(), 1); + let stranger_actor = ActorId::from_secp256k1(stranger.public_key().as_bytes()); + assert!(change.verify(&cob_home(), &change.author, None)); + assert!( + !change.verify(&cob_home(), &stranger_actor, None), + "a valid signature under an unexpected expected-author is refused" + ); + let foreign = Change { + author: stranger_actor, + ..change + }; + assert!( + !foreign.verify(&cob_home(), &foreign.author, None), + "an author swapped to a stranger fails its own signature check" + ); + } + + #[test] + fn verify_rejects_a_tampered_transplanted_or_rehomed_change() { + let signer = K256Signer::generate(&SeededEntropy::new(10)); + let revision = Oid::from_hex("1111111111111111111111111111111111111111").unwrap(); + let genuine = signed_change(&signer, revision, Vec::new(), 1); + assert!( + genuine.verify(&cob_home(), &genuine.author, None), + "genuine root change verifies" + ); + + let mutators: Vec Change> = vec![ + |change| { + let mut bytes = change.signature.as_bytes().to_vec(); + bytes[0] ^= 0xff; + Change { + signature: Signature::from_bytes(bytes), + ..change + } + }, + |change| Change { + parents: vec![ChangeId::new( + Oid::from_hex("3333333333333333333333333333333333333333").unwrap(), + )], + ..change + }, + |change| Change { + timestamp: UnixSeconds::new(9_999_999), + ..change + }, + ]; + mutators.into_iter().for_each(|mutate| { + let broken = mutate(genuine.clone()); + assert!( + !broken.verify(&cob_home(), &broken.author, None), + "a tampered or transplanted change fails verification" + ); + }); + + let other_home = CobHome::from(&RepoDid::new("did:plc:limpet").unwrap()); + assert!( + !genuine.verify(&other_home, &genuine.author, None), + "change signed for one repo mustn't verify under another repo's home" + ); + + let author = ActorId::from_secp256k1(signer.public_key().as_bytes()); + let parent = + ChangeId::new(Oid::from_hex("5555555555555555555555555555555555555555").unwrap()); + let home = CobId::new(Oid::from_hex("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").unwrap()); + let elsewhere = + CobId::new(Oid::from_hex("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb").unwrap()); + let timestamp = UnixSeconds::new(1); + let bytes = signing_bytes( + &cob_home(), + revision, + &[parent], + &type_name(), + &author, + timestamp, + object_binding(&[parent], Some(home)), + ); + let bound = Change { + id: ChangeId::new(Oid::null()), + revision, + parents: vec![parent], + type_name: type_name(), + author, + signature: signer.sign(&bytes), + payload: Payload::new(Vec::new()), + timestamp, + }; + assert!(bound.verify(&cob_home(), &bound.author, Some(home))); + assert!( + !bound.verify(&cob_home(), &bound.author, Some(elsewhere)), + "a non-root change is bound to its object" + ); + assert!(!bound.verify(&cob_home(), &bound.author, None)); + } +} diff --git a/knot2/crates/knot-cob/src/error.rs b/knot2/crates/knot-cob/src/error.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-cob/src/error.rs @@ -0,0 +1,53 @@ +use knot_types::{ChangeId, CobId, Oid, TypeName}; + +#[derive(Debug, thiserror::Error)] +pub enum PayloadError { + #[error("dag-cbor encode failed: {0}")] + Encode(String), + #[error("dag-cbor decode failed: {0}")] + Decode(String), +} + +#[derive(Debug, thiserror::Error)] +pub enum CobError { + #[error(transparent)] + Git(#[from] knot_git::GitError), + #[error(transparent)] + Payload(#[from] PayloadError), + #[error("object {0} does not exist")] + NoSuchObject(CobId), + #[error("object {0} tip does not descend from its root change")] + DetachedTip(CobId), + #[error("object {object} tip does not descend from indexed tip {since}")] + DivergedTip { object: CobId, since: ChangeId }, + #[error("object {0} is not rooted at genesis change")] + RootNotGenesis(CobId), + #[error("object {object} contains second parentless change {stray}")] + MultipleRoots { object: CobId, stray: ChangeId }, + #[error("authoritative object {object} has forked history at merge change {change}")] + ForkedHistory { object: CobId, change: ChangeId }, + #[error("change {change} is not validly signed by the owning identity")] + UnverifiedChange { change: ChangeId }, + #[error("concurrent write moved object {object} past expected tip {expected}")] + StaleTip { object: CobId, expected: ChangeId }, + #[error("object {0} exceeded its compare-and-swap retry budget under contention")] + Contended(CobId), + #[error("change {oid} is malformed: {reason}")] + MalformedChange { oid: Oid, reason: String }, + #[error("change {change} payload does not decode: {reason}")] + UndecodableChange { change: ChangeId, reason: String }, + #[error("change {change} is a {found} change in {expected} object")] + UnexpectedChangeType { + change: ChangeId, + expected: TypeName, + found: TypeName, + }, + #[error("produced unverifiable signature for {0} change")] + SelfCheck(TypeName), + #[error("change graph for {0} exceeds load bound")] + HistoryTooLong(CobId), + #[error("'{0}' is not usable collaborative-object ref name")] + RefName(String), + #[error("writing git object failed: {0}")] + Write(String), +} diff --git a/knot2/crates/knot-cob/src/graph.rs b/knot2/crates/knot-cob/src/graph.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-cob/src/graph.rs @@ -0,0 +1,130 @@ +use std::cmp::Reverse; +use std::collections::{BTreeMap, BTreeSet, BinaryHeap}; + +use knot_types::{ChangeId, CobId, UnixSeconds}; + +use crate::change::Change; + +#[derive(Debug)] +pub struct ChangeGraph { + root: CobId, + changes: BTreeMap, +} + +impl ChangeGraph { + pub(crate) fn new(root: CobId, changes: BTreeMap) -> Self { + Self { root, changes } + } + + pub fn root(&self) -> CobId { + self.root + } + + pub fn len(&self) -> usize { + self.changes.len() + } + + pub fn is_empty(&self) -> bool { + self.changes.is_empty() + } + + pub fn causal_order(&self) -> Vec { + order(&self.changes) + } + + pub(crate) fn into_ordered(self) -> Vec { + let ordered = order(&self.changes); + let mut changes = self.changes; + ordered + .into_iter() + .map(|id| changes.remove(&id).expect("ordered id is in graph")) + .collect() + } +} + +fn order(changes: &BTreeMap) -> Vec { + let mut indegree: BTreeMap = changes + .values() + .map(|change| { + let present = change + .parents + .iter() + .filter(|parent| changes.contains_key(*parent)) + .count(); + (change.id, present) + }) + .collect(); + let children: BTreeMap> = + changes.values().fold(BTreeMap::new(), |mut acc, change| { + change + .parents + .iter() + .filter(|parent| changes.contains_key(*parent)) + .for_each(|parent| acc.entry(*parent).or_default().push(change.id)); + acc + }); + let mut ready: BinaryHeap> = changes + .values() + .filter(|change| indegree[&change.id] == 0) + .map(|change| Reverse(change.sort_key())) + .collect(); + std::iter::from_fn(move || { + let Reverse((_, id)) = ready.pop()?; + children.get(&id).into_iter().flatten().for_each(|child| { + let degree = indegree + .get_mut(child) + .expect("every child has an indegree entry"); + *degree -= 1; + if *degree == 0 { + ready.push(Reverse(changes[child].sort_key())); + } + }); + Some(id) + }) + .collect() +} + +#[derive(Debug)] +pub struct History { + root: ChangeId, + changes: Vec, +} + +impl History { + pub(crate) fn new(root: ChangeId, changes: Vec) -> Self { + Self { root, changes } + } + + pub fn root(&self) -> ChangeId { + self.root + } + + pub fn changes(&self) -> &[Change] { + &self.changes + } + + pub fn len(&self) -> usize { + self.changes.len() + } + + pub fn is_empty(&self) -> bool { + self.changes.is_empty() + } + + pub fn traverse A>(&self, init: A, f: F) -> A { + self.changes.iter().fold(init, f) + } + + pub fn tips(&self) -> Vec { + let referenced: BTreeSet = self + .changes + .iter() + .flat_map(|change| change.parents.iter().copied()) + .collect(); + self.changes + .iter() + .map(|change| change.id) + .filter(|id| !referenced.contains(id)) + .collect() + } +} diff --git a/knot2/crates/knot-cob/src/instrument.rs b/knot2/crates/knot-cob/src/instrument.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-cob/src/instrument.rs @@ -0,0 +1,1 @@ +knot_types::read_counter!(CobReads, record_read, cob_reads, reset_cob_reads, measure); diff --git a/knot2/crates/knot-cob/src/lib.rs b/knot2/crates/knot-cob/src/lib.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-cob/src/lib.rs @@ -0,0 +1,1999 @@ +mod backend; +mod change; +mod error; +mod graph; +#[cfg(feature = "instrument")] +pub mod instrument; +mod object; + +pub use change::{Change, ChangePayload, CobHome, Payload}; +pub use error::{CobError, PayloadError}; +pub use graph::{ChangeGraph, History}; +pub use knot_types::{ActorId, ChangeId, CobId, TypeName}; +pub use object::{Checkpoint, Evaluate, HistoryModel, Object, SnapshotStride, StateSize}; + +pub use backend::parse_cob_ref; + +use knot_git::{RefUpdate, Repo}; +use knot_runtime::Signer; +use knot_types::{Oid, UnixSeconds}; +use serde::Serialize; +use serde::de::DeserializeOwned; + +const MAX_CAS_RETRIES: usize = 16; +const CHECKPOINT_GROWTH_DIVISOR: usize = 16; +// don't ask + +fn checkpoint_stride(snapshot_stride: SnapshotStride, size: StateSize) -> usize { + snapshot_stride + .get() + .max(size.get() / CHECKPOINT_GROWTH_DIVISOR) + .min(backend::MAX_GRAPH_CHANGES) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Created { + pub object: CobId, + pub tip: ChangeId, +} + +#[derive(Debug)] +pub struct Delta { + pub changes: Vec, + pub tip: ChangeId, +} + +const CHECKPOINT_FORMAT: u16 = 3; + +#[derive(Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +struct CheckpointDigest(Oid); + +fn checkpoint_digest( + object: CobId, + tip: ChangeId, + state: &S, +) -> Result { + let bytes = serde_ipld_dagcbor::to_vec(&(object, tip, state)) + .map_err(|error| CobError::Write(error.to_string()))?; + let mut hasher = gix_hash::hasher(gix_hash::Kind::Sha1); + hasher.update(&bytes); + hasher + .try_finalize() + .map(|id| CheckpointDigest(Oid::from(id))) + .map_err(|error| CobError::Write(error.to_string())) +} + +fn encode_checkpoint( + object: CobId, + tip: ChangeId, + state: &S, +) -> Result, CobError> { + let digest = checkpoint_digest(object, tip, state)?; + serde_ipld_dagcbor::to_vec(&(CHECKPOINT_FORMAT, object, tip, digest, state)) + .map_err(|error| CobError::Write(error.to_string())) +} + +fn decode_checkpoint( + object: CobId, + bytes: &[u8], +) -> Option<(ChangeId, S)> { + let (version, decoded_object, tip, digest, state): (u16, CobId, ChangeId, CheckpointDigest, S) = + serde_ipld_dagcbor::from_slice(bytes).ok()?; + (version == CHECKPOINT_FORMAT).then_some(())?; + (decoded_object == object).then_some(())?; + (checkpoint_digest(object, tip, &state).ok()? == digest).then_some(())?; + Some((tip, state)) +} + +pub struct CobStore<'r> { + repo: &'r Repo, +} + +impl<'r> CobStore<'r> { + pub fn new(repo: &'r Repo) -> Self { + Self { repo } + } + + pub fn create( + &self, + home: &CobHome, + payload: &P, + signer: &dyn Signer, + timestamp: UnixSeconds, + ) -> Result { + let type_name = P::type_name(); + let bytes = payload.encode()?; + let tip = backend::write_change( + home, + self.repo, + &type_name, + &bytes, + &[], + None, + signer, + timestamp, + )?; + let object = CobId::new(tip.oid()); + let name = backend::cob_ref_name(&type_name, object)?; + self.repo.update_ref(&RefUpdate::Create { + name, + new: tip.oid(), + })?; + Ok(Created { object, tip }) + } + + pub fn update( + &self, + home: &CobHome, + object: CobId, + payload: &P, + signer: &dyn Signer, + timestamp: UnixSeconds, + ) -> Result { + let type_name = P::type_name(); + let expected = backend::resolve_tip(self.repo, &type_name, object)? + .map(ChangeId::new) + .ok_or(CobError::NoSuchObject(object))?; + self.append( + home, object, &type_name, expected, payload, signer, timestamp, + ) + } + + pub fn extend<'p, P: ChangePayload + 'p>( + &self, + home: &CobHome, + object: CobId, + changes: impl IntoIterator, + signer: &dyn Signer, + ) -> Result, CobError> { + let type_name = P::type_name(); + let expected = backend::resolve_tip(self.repo, &type_name, object)? + .map(ChangeId::new) + .ok_or(CobError::NoSuchObject(object))?; + self.chain(home, object, &type_name, expected, changes, signer) + } + + #[allow(clippy::too_many_arguments)] + fn append( + &self, + home: &CobHome, + object: CobId, + type_name: &TypeName, + expected: ChangeId, + payload: &P, + signer: &dyn Signer, + timestamp: UnixSeconds, + ) -> Result { + self.chain( + home, + object, + type_name, + expected, + std::iter::once((payload, timestamp)), + signer, + ) + .map(|tip| tip.expect("one change chains onto one tip")) + } + + fn chain<'p, P: ChangePayload + 'p>( + &self, + home: &CobHome, + object: CobId, + type_name: &TypeName, + expected: ChangeId, + changes: impl IntoIterator, + signer: &dyn Signer, + ) -> Result, CobError> { + let written = match changes.into_iter().try_fold( + Vec::::new(), + |mut written, (payload, timestamp)| { + let parent = written.last().copied().unwrap_or(expected); + let write = || -> Result { + let bytes = payload.encode()?; + backend::write_change( + home, + self.repo, + type_name, + &bytes, + &[parent], + Some(object), + signer, + timestamp, + ) + }; + match write() { + Ok(tip) => { + written.push(tip); + Ok(written) + } + Err(error) => Err((written, error)), + } + }, + ) { + Ok(written) => written, + Err((partial, error)) => { + partial + .iter() + .try_for_each(|change| self.repo.remove_loose_object(change.oid()))?; + return Err(error); + } + }; + written.last().copied().map_or(Ok(None), |tip| { + self.publish(type_name, object, expected, tip, &written) + .map(Some) + }) + } + + fn publish( + &self, + type_name: &TypeName, + object: CobId, + expected: ChangeId, + tip: ChangeId, + written: &[ChangeId], + ) -> Result { + let name = backend::cob_ref_name(type_name, object)?; + match self.repo.update_ref(&RefUpdate::Update { + name, + old: expected.oid(), + new: tip.oid(), + }) { + Ok(()) => Ok(tip), + Err(error) => { + let actual = backend::resolve_tip(self.repo, type_name, object)?; + if actual != Some(tip.oid()) { + written + .iter() + .try_for_each(|change| self.repo.remove_loose_object(change.oid()))?; + } + match actual { + actual if actual != Some(expected.oid()) => { + Err(CobError::StaleTip { object, expected }) + } + _ => Err(CobError::Git(error)), + } + } + } + } + + pub fn update_with( + &self, + home: &CobHome, + object: CobId, + signer: &dyn Signer, + timestamp: UnixSeconds, + decide: impl Fn(&E::State) -> Result, + ) -> Result + where + E: Evaluate, + D: From, + { + self.update_maybe::(home, object, signer, timestamp, |state| { + decide(state).map(Some) + }) + .map(|tip| tip.expect("update_with always yields change to append")) + } + + pub fn update_maybe( + &self, + home: &CobHome, + object: CobId, + signer: &dyn Signer, + timestamp: UnixSeconds, + decide: impl Fn(&E::State) -> Result, D>, + ) -> Result, D> + where + E: Evaluate, + D: From, + { + let type_name = E::Change::type_name(); + let attempt = || -> Result>, D> { + let (graph, expected) = backend::load_graph( + self.repo, + &type_name, + object, + E::HISTORY, + backend::MAX_GRAPH_CHANGES, + )?; + let (state, _history) = object::evaluate::(graph, &type_name)?; + match decide(&state)? { + None => Ok(Some(None)), + Some(change) => { + match self.append( + home, object, &type_name, expected, &change, signer, timestamp, + ) { + Ok(id) => Ok(Some(Some(id))), + Err(CobError::StaleTip { .. }) => Ok(None), + Err(other) => Err(D::from(other)), + } + } + } + }; + (0..MAX_CAS_RETRIES) + .find_map(|_| attempt().transpose()) + .unwrap_or_else(|| Err(D::from(CobError::Contended(object)))) + } + + pub fn graph(&self, object: CobId) -> Result { + Ok(backend::load_graph( + self.repo, + &E::Change::type_name(), + object, + E::HISTORY, + backend::MAX_GRAPH_CHANGES, + )? + .0) + } + + pub fn get(&self, object: CobId) -> Result, CobError> { + let type_name = E::Change::type_name(); + let (graph, _tip) = backend::load_graph( + self.repo, + &type_name, + object, + E::HISTORY, + backend::MAX_GRAPH_CHANGES, + )?; + let (state, history) = object::evaluate::(graph, &type_name)?; + Ok(Object::new(object, type_name, state, history)) + } + + pub fn verify( + &self, + home: &CobHome, + object: CobId, + owner: &ActorId, + ) -> Result<(), CobError> { + let type_name = E::Change::type_name(); + let (graph, _tip) = backend::load_graph( + self.repo, + &type_name, + object, + E::HISTORY, + backend::MAX_GRAPH_CHANGES, + )?; + graph.into_ordered().into_iter().try_for_each(|change| { + if change.type_name != type_name { + return Err(CobError::UnexpectedChangeType { + change: change.id, + expected: type_name.clone(), + found: change.type_name, + }); + } + change + .verify(home, owner, Some(object)) + .then_some(()) + .ok_or(CobError::UnverifiedChange { change: change.id }) + }) + } + + pub fn list(&self) -> Result, CobError> { + backend::list_objects(self.repo, &E::Change::type_name()) + } + + pub fn changes_since( + &self, + object: CobId, + since: Option, + ) -> Result { + let type_name = E::Change::type_name(); + let tip = backend::resolve_tip(self.repo, &type_name, object)? + .map(ChangeId::new) + .ok_or(CobError::NoSuchObject(object))?; + let collected = + backend::collect(self.repo, tip, object, backend::MAX_GRAPH_CHANGES, since)?; + match since { + None => backend::check_full_shape(&collected, object, E::HISTORY)?, + Some(since) => backend::check_delta_shape(&collected, object, since, E::HISTORY)?, + } + let changes = ChangeGraph::new(object, collected).into_ordered(); + Ok(Delta { changes, tip }) + } + + pub fn update_with_checkpointed( + &self, + home: &CobHome, + object: CobId, + signer: &dyn Signer, + timestamp: UnixSeconds, + decide: impl Fn(&E::State) -> Result, + ) -> Result + where + E: Checkpoint, + E::State: Serialize + DeserializeOwned, + D: From, + { + self.update_maybe_checkpointed::(home, object, signer, timestamp, |state| { + decide(state).map(Some) + }) + .map(|tip| tip.expect("update_with_checkpointed always yields change to append")) + } + + pub fn update_maybe_checkpointed( + &self, + home: &CobHome, + object: CobId, + signer: &dyn Signer, + timestamp: UnixSeconds, + decide: impl Fn(&E::State) -> Result, D>, + ) -> Result, D> + where + E: Checkpoint, + E::State: Serialize + DeserializeOwned, + D: From, + { + let type_name = E::Change::type_name(); + let attempt = || -> Result>, D> { + let (state, expected, suffix) = self.checkpointed_state::(object)?; + match decide(&state)? { + None => Ok(Some(None)), + Some(change) => match self.append( + home, object, &type_name, expected, &change, signer, timestamp, + ) { + Ok(tip) => { + let stride = + checkpoint_stride(E::SNAPSHOT_STRIDE, E::checkpoint_size(&state)); + if suffix.saturating_add(1) >= stride { + let author = ActorId::from_secp256k1(signer.public_key().as_bytes()); + let folded = E::apply(state, change, &author); + if let Err(error) = self.write_checkpoint::(object, tip, &folded) { + tracing::warn!( + cob = type_name.as_str(), + object = %object.oid().to_hex(), + %error, + "checkpoint write failed" + ); + } + } + Ok(Some(Some(tip))) + } + Err(CobError::StaleTip { .. }) => Ok(None), + Err(other) => Err(D::from(other)), + }, + } + }; + (0..MAX_CAS_RETRIES) + .find_map(|_| attempt().transpose()) + .unwrap_or_else(|| Err(D::from(CobError::Contended(object)))) + } + + pub fn materialize(&self, object: CobId) -> Result<(E::State, ChangeId), CobError> + where + E: Checkpoint, + E::State: Serialize + DeserializeOwned, + { + self.checkpointed_state::(object) + .map(|(state, tip, _)| (state, tip)) + } + + fn checkpointed_state(&self, object: CobId) -> Result<(E::State, ChangeId, usize), CobError> + where + E: Checkpoint, + E::State: Serialize + DeserializeOwned, + { + let type_name = E::Change::type_name(); + let tip = backend::resolve_tip(self.repo, &type_name, object)? + .map(ChangeId::new) + .ok_or(CobError::NoSuchObject(object))?; + match self.load_checkpoint::(object)? { + Some((checkpoint_tip, state)) if checkpoint_tip == tip => Ok((state, tip, 0)), + Some((checkpoint_tip, state)) => { + match self.changes_since::(object, Some(checkpoint_tip)) { + Ok(delta) => { + let folded = object::fold_changes::(state, &delta.changes, &type_name)?; + Ok((folded, tip, delta.changes.len())) + } + Err(_) => self + .full_state::(object) + .map(|state| (state, tip, usize::MAX)), + } + } + None => self + .full_state::(object) + .map(|state| (state, tip, usize::MAX)), + } + } + + fn full_state(&self, object: CobId) -> Result { + let type_name = E::Change::type_name(); + let (graph, _tip) = backend::load_graph( + self.repo, + &type_name, + object, + E::HISTORY, + backend::rebuild_graph_limit(), + )?; + object::evaluate::(graph, &type_name).map(|(state, _history)| state) + } + + fn load_checkpoint(&self, object: CobId) -> Result, CobError> + where + E: Checkpoint, + E::State: DeserializeOwned + Serialize, + { + let name = backend::checkpoint_ref_name(&E::Change::type_name(), object)?; + let Some(blob) = self.repo.find_ref(&name)? else { + return Ok(None); + }; + let Ok(bytes) = self.repo.read_blob(blob) else { + return Ok(None); + }; + Ok(decode_checkpoint::(object, &bytes)) + } + + fn write_checkpoint( + &self, + object: CobId, + tip: ChangeId, + state: &E::State, + ) -> Result<(), CobError> + where + E: Checkpoint, + E::State: Serialize, + { + let bytes = encode_checkpoint(object, tip, state)?; + let blob = Oid::from( + self.repo + .git() + .write_blob(&bytes) + .map_err(|error| CobError::Write(error.to_string()))? + .detach(), + ); + let name = backend::checkpoint_ref_name(&E::Change::type_name(), object)?; + let update = match self.repo.find_ref(&name)? { + Some(old) => RefUpdate::Update { + name, + old, + new: blob, + }, + None => RefUpdate::Create { name, new: blob }, + }; + self.repo.update_ref(&update).map_err(CobError::from) + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + + use knot_git::{Layout, Repo}; + use knot_runtime::{K256Signer, SeededEntropy, Signature, Signer}; + use knot_types::{RepoDid, UnixSeconds}; + use proptest::prelude::*; + use serde::{Deserialize, Serialize}; + use tempfile::TempDir; + + use super::*; + + #[derive(Debug, Serialize, Deserialize)] + #[serde(tag = "op", content = "subject")] + enum Tag { + Add(String), + Remove(String), + } + + impl ChangePayload for Tag { + const TYPE: &'static str = "sh.tangled.test.tag"; + } + + #[test] + fn checkpoint_encoding_stays_byte_stable() { + let object = CobId::new(Oid::from_hex("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").unwrap()); + let tip = ChangeId::new(Oid::from_hex("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb").unwrap()); + let mut state = BTreeSet::new(); + state.insert("kelp".to_string()); + state.insert("squid".to_string()); + let bytes = encode_checkpoint(object, tip, &state).unwrap(); + assert_eq!( + knot_types::lowercase_hex(&bytes), + "850378286161616161616161616161616161616161616161616161616161616161616161616161616161616178286262626262626262626262626262626262626262626262626262626262626262626262626262626278283861623531376164633234616530313138383962643634343930663836633562646333333632333782646b656c70657371756964" + ); + } + + struct Tags; + + impl Evaluate for Tags { + type State = BTreeSet; + type Change = Tag; + + const HISTORY: HistoryModel = HistoryModel::Convergent; + + fn initial() -> Self::State { + BTreeSet::new() + } + + fn apply(mut state: Self::State, change: Self::Change, _author: &ActorId) -> Self::State { + match change { + Tag::Add(subject) => { + state.insert(subject); + } + Tag::Remove(subject) => { + state.remove(&subject); + } + } + state + } + } + + #[derive(Debug, Serialize, Deserialize)] + #[serde(tag = "op", content = "subject")] + enum NameChange { + Claim(String), + } + + impl ChangePayload for NameChange { + const TYPE: &'static str = "sh.tangled.test.name"; + } + + struct Names; + + impl Evaluate for Names { + type State = BTreeSet; + type Change = NameChange; + + const HISTORY: HistoryModel = HistoryModel::Linear; + + fn initial() -> Self::State { + BTreeSet::new() + } + + fn apply(mut state: Self::State, change: Self::Change, _author: &ActorId) -> Self::State { + match change { + NameChange::Claim(subject) => { + state.insert(subject); + } + } + state + } + } + + impl Checkpoint for Names { + const SNAPSHOT_STRIDE: SnapshotStride = SnapshotStride::new(4); + fn checkpoint_size(state: &Self::State) -> StateSize { + StateSize::new(state.len()) + } + } + + #[derive(Debug)] + enum NameError { + Taken, + Cob(CobError), + } + + impl From for NameError { + fn from(error: CobError) -> Self { + NameError::Cob(error) + } + } + + fn claim( + store: &CobStore, + object: CobId, + key: &K256Signer, + who: &str, + when: i64, + ) -> Result, NameError> { + store.update_maybe_checkpointed::( + &cob_home(), + object, + key, + at(when), + |state| match state.contains(who) { + true => Err(NameError::Taken), + false => Ok(Some(NameChange::Claim(who.to_string()))), + }, + ) + } + + fn appended(result: Result, NameError>) -> ChangeId { + match result { + Ok(Some(id)) => id, + Ok(None) => panic!("distinct claim should append a change"), + Err(NameError::Taken) => panic!("name was unexpectedly already taken"), + Err(NameError::Cob(error)) => panic!("checkpointed write failed: {error:?}"), + } + } + + fn fixture() -> (TempDir, Repo) { + let dir = tempfile::tempdir().unwrap(); + let layout = Layout::new(dir.path()); + let repo = layout + .create(&RepoDid::new("did:plc:squid").unwrap()) + .unwrap(); + (dir, repo) + } + + fn signer(seed: u64) -> K256Signer { + K256Signer::generate(&SeededEntropy::new(seed)) + } + + fn cob_home() -> CobHome { + CobHome::from(&RepoDid::new("did:plc:squid").unwrap()) + } + + fn at(seconds: i64) -> UnixSeconds { + UnixSeconds::new(seconds) + } + + fn tag_root(repo: &Repo, key: &K256Signer, subject: &str, when: i64) -> ChangeId { + backend::write_change( + &cob_home(), + repo, + &Tag::type_name(), + &Tag::Add(subject.into()).encode().unwrap(), + &[], + None, + key, + at(when), + ) + .unwrap() + } + + fn tag_child( + repo: &Repo, + key: &K256Signer, + subject: &str, + parents: &[ChangeId], + object: CobId, + when: i64, + ) -> ChangeId { + backend::write_change( + &cob_home(), + repo, + &Tag::type_name(), + &Tag::Add(subject.into()).encode().unwrap(), + parents, + Some(object), + key, + at(when), + ) + .unwrap() + } + + fn publish(repo: &Repo, object: CobId, tip: ChangeId) { + let name = backend::cob_ref_name(&Tag::type_name(), object).unwrap(); + repo.update_ref(&RefUpdate::Create { + name, + new: tip.oid(), + }) + .unwrap(); + } + + fn forked_tag_object( + repo: &Repo, + key: &K256Signer, + left: &str, + right: &str, + merge: &str, + ) -> CobId { + let root = tag_root(repo, key, "base", 1); + let object = CobId::new(root.oid()); + let left = tag_child(repo, key, left, &[root], object, 2); + let right = tag_child(repo, key, right, &[root], object, 3); + let merge = tag_child(repo, key, merge, &[left, right], object, 4); + publish(repo, object, merge); + object + } + + fn linear_chain(repo: &Repo, key: &K256Signer, subjects: &[&str]) -> CobId { + let (first, rest) = subjects.split_first().expect("chain needs a root subject"); + let root = tag_root(repo, key, first, 1); + let object = CobId::new(root.oid()); + let tip = rest + .iter() + .enumerate() + .fold(root, |parent, (index, subject)| { + tag_child(repo, key, subject, &[parent], object, index as i64 + 2) + }); + publish(repo, object, tip); + object + } + + #[test] + fn extend_chains_the_same_tip_as_sequential_updates() { + let (_sequential_dir, sequential_repo) = fixture(); + let (_batched_dir, batched_repo) = fixture(); + let key = signer(3); + let changes = [ + NameChange::Claim("kelp".into()), + NameChange::Claim("squid".into()), + NameChange::Claim("whelk".into()), + ]; + let sequential = CobStore::new(&sequential_repo); + let batched = CobStore::new(&batched_repo); + let root = |store: &CobStore| { + store + .create(&cob_home(), &NameChange::Claim("uni".into()), &key, at(0)) + .unwrap() + .object + }; + let one_by_one = root(&sequential); + changes.iter().zip(1..).for_each(|(change, when)| { + sequential + .update(&cob_home(), one_by_one, change, &key, at(when)) + .unwrap(); + }); + let in_one_go = root(&batched); + let tip = batched + .extend( + &cob_home(), + in_one_go, + changes.iter().zip((1..).map(at)), + &key, + ) + .unwrap(); + + let resolved = |repo: &Repo, object| { + backend::resolve_tip(repo, &NameChange::type_name(), object) + .unwrap() + .map(ChangeId::new) + }; + assert_eq!(one_by_one, in_one_go); + assert_eq!(tip, resolved(&batched_repo, in_one_go)); + assert_eq!(tip, resolved(&sequential_repo, one_by_one)); + assert_eq!( + sequential.materialize::(one_by_one).unwrap().0, + batched.materialize::(in_one_go).unwrap().0 + ); + assert_eq!( + batched + .extend::(&cob_home(), in_one_go, std::iter::empty(), &key) + .unwrap(), + None + ); + assert_eq!(tip, resolved(&batched_repo, in_one_go)); + } + + #[test] + fn checkpointed_writes_match_a_full_fold_and_leave_a_snapshot() { + let (_dir, repo) = fixture(); + let store = CobStore::new(&repo); + let key = signer(1); + let object = store + .create( + &cob_home(), + &NameChange::Claim("name0000".into()), + &key, + at(0), + ) + .unwrap() + .object; + + let total = Names::SNAPSHOT_STRIDE.get() * 3; + (1..total).for_each(|index| { + appended(claim( + &store, + object, + &key, + &format!("name{index:04}"), + index as i64, + )); + }); + + let folded = store.get::(object).unwrap(); + assert_eq!( + folded.state().len(), + total, + "every checkpointed write landed and full fold agrees" + ); + let snapshot = backend::checkpoint_ref_name(&NameChange::type_name(), object).unwrap(); + assert!( + repo.find_ref(&snapshot).unwrap().is_some(), + "snapshot ref was written once stride was crossed" + ); + } + + #[test] + fn checkpoint_stride_grows_with_state_so_large_cobs_snapshot_less_often() { + assert_eq!( + checkpoint_stride(SnapshotStride::new(256), StateSize::new(0)), + 256 + ); + assert_eq!( + checkpoint_stride( + SnapshotStride::new(256), + StateSize::new(256 * CHECKPOINT_GROWTH_DIVISOR) + ), + 256, + "up to stride*divisor entries the fixed floor governs, so small cobs snapshot exactly as before" + ); + assert_eq!( + checkpoint_stride(SnapshotStride::new(256), StateSize::new(1_600_000)), + 100_000, + "a large cob snapshots at a fixed fraction of its size, so total snapshot bytes stay linear in the change count" + ); + assert_eq!( + checkpoint_stride(SnapshotStride::new(256), StateSize::new(4_000_000)), + backend::MAX_GRAPH_CHANGES, + "the stride stops at the serving limit so the boot tail always folds incrementally, never overflowing into a full rebuild" + ); + } + + #[test] + fn a_large_checkpointed_cob_bounds_its_boot_suffix_to_a_fraction_of_its_size() { + let (_dir, repo) = fixture(); + let store = CobStore::new(&repo); + let key = signer(2); + let object = store + .create( + &cob_home(), + &NameChange::Claim("name0000".into()), + &key, + at(0), + ) + .unwrap() + .object; + + let total = Names::SNAPSHOT_STRIDE.get() * CHECKPOINT_GROWTH_DIVISOR * 4; + (1..total).for_each(|index| { + claim( + &store, + object, + &key, + &format!("name{index:04}"), + index as i64, + ) + .unwrap(); + }); + + assert_eq!( + store.get::(object).unwrap().state().len(), + total, + "full fold still agrees after adaptive snapshotting" + ); + + let (_state, _tip, suffix) = store.checkpointed_state::(object).unwrap(); + let bound = checkpoint_stride(Names::SNAPSHOT_STRIDE, StateSize::new(total)); + assert!( + suffix <= bound && bound < total, + "boot replays only the {suffix}-change tail since the last snapshot, bounded by the adaptive stride {bound}, never the whole {total}-change history" + ); + } + + #[test] + fn a_corrupt_checkpoint_never_serves_a_wrong_answer_and_heals_at_the_live_tip() { + type Forge = fn(CobId, Oid, &BTreeSet) -> Vec; + let cases: &[Forge] = &[ + |_object, _tip, _state| b"not a valid checkpoint envelope".to_vec(), + |object, tip, state| { + serde_ipld_dagcbor::to_vec(&( + CHECKPOINT_FORMAT + 1, + object.oid().to_hex(), + tip.to_hex(), + checkpoint_digest(object, ChangeId::new(tip), state).unwrap(), + state, + )) + .unwrap() + }, + |object, tip, state| { + let mut tampered = state.clone(); + tampered.insert("intruder".to_string()); + serde_ipld_dagcbor::to_vec(&( + CHECKPOINT_FORMAT, + object.oid().to_hex(), + tip.to_hex(), + checkpoint_digest(object, ChangeId::new(tip), state).unwrap(), + tampered, + )) + .unwrap() + }, + ]; + + cases.iter().enumerate().for_each(|(index, forge)| { + let (_dir, repo) = fixture(); + let store = CobStore::new(&repo); + let key = signer(200 + index as u64); + let object = store + .create(&cob_home(), &NameChange::Claim("squid".into()), &key, at(0)) + .unwrap() + .object; + (1..=Names::SNAPSHOT_STRIDE.get()).for_each(|step| { + claim(&store, object, &key, &format!("name{step:04}"), step as i64).unwrap(); + }); + + let real_state = store.get::(object).unwrap().state().clone(); + let real_tip = backend::resolve_tip(&repo, &NameChange::type_name(), object) + .unwrap() + .unwrap(); + let forged = forge(object, real_tip, &real_state); + + let snapshot = backend::checkpoint_ref_name(&NameChange::type_name(), object).unwrap(); + let live = repo.find_ref(&snapshot).unwrap().expect("snapshot exists"); + let blob = Oid::from(repo.git().write_blob(&forged).unwrap().detach()); + repo.update_ref(&RefUpdate::Update { + name: snapshot.clone(), + old: live, + new: blob, + }) + .unwrap(); + + let duplicate = claim(&store, object, &key, "squid", 100); + assert!( + matches!(duplicate, Err(NameError::Taken)), + "corrupt checkpoint falls back to real state instead of waving a duplicate through" + ); + appended(claim(&store, object, &key, "intruder", 101)); + + let healed = repo.find_ref(&snapshot).unwrap().unwrap(); + let bytes = repo.read_blob(healed).unwrap(); + let (tip, state) = decode_checkpoint::>(object, &bytes) + .expect("healed snapshot decodes"); + let live_tip = backend::resolve_tip(&repo, &NameChange::type_name(), object) + .unwrap() + .unwrap(); + assert_eq!( + tip, + ChangeId::new(live_tip), + "healed snapshot sits at the live tip" + ); + assert_eq!( + &state, + store.get::(object).unwrap().state(), + "healed snapshot matches the full fold" + ); + }); + } + + #[test] + fn a_checkpoint_envelope_is_bound_to_its_object_tip_and_format() { + let (_dir, repo) = fixture(); + let store = CobStore::new(&repo); + let key = signer(6); + let a = store + .create(&cob_home(), &NameChange::Claim("squid".into()), &key, at(0)) + .unwrap(); + let b = store + .create( + &cob_home(), + &NameChange::Claim("anemone".into()), + &key, + at(1), + ) + .unwrap(); + let state = BTreeSet::from(["squid".to_string()]); + + let bytes = encode_checkpoint(a.object, a.tip, &state).unwrap(); + assert!( + decode_checkpoint::>(b.object, &bytes).is_none(), + "checkpoint minted for one object mustn't decode under another" + ); + assert_eq!( + decode_checkpoint::>(a.object, &bytes), + Some((a.tip, state.clone())), + "checkpoint decodes to the tip and state its digest commits to under its own object" + ); + + let wrong_tip = ChangeId::new(Oid::from_hex(&"b".repeat(40)).unwrap()); + let swapped_tip = serde_ipld_dagcbor::to_vec(&( + CHECKPOINT_FORMAT, + a.object.oid().to_hex(), + wrong_tip.oid().to_hex(), + checkpoint_digest(a.object, a.tip, &state).unwrap(), + state.clone(), + )) + .unwrap(); + assert!( + decode_checkpoint::>(a.object, &swapped_tip).is_none(), + "checkpoint whose tip is swapped away from the one its digest covers mustn't decode" + ); + + let future_version = serde_ipld_dagcbor::to_vec(&( + CHECKPOINT_FORMAT + 1, + a.object.oid().to_hex(), + a.tip.oid().to_hex(), + checkpoint_digest(a.object, a.tip, &state).unwrap(), + state, + )) + .unwrap(); + assert!( + decode_checkpoint::>(a.object, &future_version).is_none(), + "envelope whose format version isn't the current one mustn't decode" + ); + } + + #[test] + fn the_rebuild_walk_floors_at_the_serving_limit_and_scales_above_it_with_headroom() { + assert_eq!( + backend::rebuild_change_limit_for(None), + 2_000_000, + "an unmeasurable host rebuilds up to the generous fixed ceiling" + ); + let available = |bytes| Some(knot_resource::AvailableBytes::new(bytes)); + assert_eq!( + backend::rebuild_change_limit_for(available(64 * 1024 * 1024)), + backend::MAX_GRAPH_CHANGES, + "a squeezed host floors the rebuild walk at the serving limit, never beneath it" + ); + assert_eq!( + backend::rebuild_change_limit_for(available(0)), + backend::MAX_GRAPH_CHANGES, + "a host with no measured headroom still heals at least as deep as it serves" + ); + let one_gib = backend::rebuild_change_limit_for(available(1024 * 1024 * 1024)); + assert!( + (100_000..150_000).contains(&one_gib), + "on a 1 GiB host the fold budget tracks the measured per-change cost near the serving limit, was {one_gib}" + ); + let roomy = backend::rebuild_change_limit_for(available(64 * 1024 * 1024 * 1024)); + assert_eq!(roomy, 64 * 1024 * 1024 * 1024 / 4 / 2560); + assert!( + roomy > backend::MAX_GRAPH_CHANGES, + "a roomy host rebuilds far past the serving limit, was {roomy}" + ); + } + + #[test] + fn tag_lifecycle_roundtrips_appends_reloads_and_keeps_signatures() { + let (_dir, repo) = fixture(); + let store = CobStore::new(&repo); + let key = signer(1); + + let created = store + .create(&cob_home(), &Tag::Add("nel".into()), &key, at(1)) + .unwrap(); + let fresh = store.get::(created.object).unwrap(); + assert_eq!(fresh.id(), created.object); + assert_eq!(fresh.state(), &BTreeSet::from(["nel".to_string()])); + assert_eq!(fresh.history().len(), 1); + assert_eq!(fresh.history().root(), created.tip); + let root = backend::read_change(&repo, ChangeId::new(created.object.oid())).unwrap(); + assert!(root.verify(&cob_home(), &root.author, None)); + + store + .update( + &cob_home(), + created.object, + &Tag::Add("olaren".into()), + &key, + at(2), + ) + .unwrap(); + let tip = store + .update( + &cob_home(), + created.object, + &Tag::Remove("nel".into()), + &key, + at(3), + ) + .unwrap(); + + let object = store.get::(created.object).unwrap(); + assert_eq!(object.state(), &BTreeSet::from(["olaren".to_string()])); + assert_eq!(object.history().len(), 3); + assert_eq!(store.list::().unwrap(), vec![created.object]); + let order = object.history().traverse(Vec::new(), |mut acc, change| { + acc.push(change.timestamp.get()); + acc + }); + assert_eq!(order, vec![1, 2, 3]); + assert_eq!(object.history().tips(), vec![tip]); + + let reloaded = store.get::(created.object).unwrap(); + assert_eq!(object.state(), reloaded.state()); + let graph = store.graph::(created.object).unwrap(); + let again = store.graph::(created.object).unwrap(); + assert_eq!(graph.len(), again.len()); + assert_eq!(graph.causal_order(), again.causal_order()); + + let head = backend::read_change(&repo, tip).unwrap(); + assert!(head.verify(&cob_home(), &head.author, Some(created.object))); + assert!(!head.verify(&cob_home(), &head.author, None)); + assert!(!head.verify( + &cob_home(), + &head.author, + Some(CobId::new(knot_types::Oid::null())) + )); + + let absent = CobId::new(knot_types::Oid::from_hex(&"0".repeat(40)).unwrap()); + assert!(matches!( + store.get::(absent), + Err(CobError::NoSuchObject(_)) + )); + } + + #[test] + fn undecodable_change_poisons_the_object() { + let (_dir, repo) = fixture(); + let key = signer(5); + let nsid = Tag::type_name(); + + let root = backend::write_change( + &cob_home(), + &repo, + &nsid, + &Tag::Add("nel".into()).encode().unwrap(), + &[], + None, + &key, + at(1), + ) + .unwrap(); + let object = CobId::new(root.oid()); + let garbage = backend::write_change( + &cob_home(), + &repo, + &nsid, + &[0xff, 0xff, 0xff], + &[root], + Some(object), + &key, + at(2), + ) + .unwrap(); + let name = backend::cob_ref_name(&nsid, object).unwrap(); + repo.update_ref(&RefUpdate::Create { + name, + new: garbage.oid(), + }) + .unwrap(); + + assert!(matches!( + CobStore::new(&repo).get::(object), + Err(CobError::UndecodableChange { .. }) + )); + } + + #[test] + fn merge_is_last_writer_wins_with_no_tombstone() { + let key = signer(40); + let nsid = Tag::type_name(); + let resolve = |add_seconds: i64, remove_seconds: i64| -> bool { + let (_dir, repo) = fixture(); + let root = backend::write_change( + &cob_home(), + &repo, + &nsid, + &Tag::Add("seed".into()).encode().unwrap(), + &[], + None, + &key, + at(1), + ) + .unwrap(); + let object = CobId::new(root.oid()); + let add = backend::write_change( + &cob_home(), + &repo, + &nsid, + &Tag::Add("nel".into()).encode().unwrap(), + &[root], + Some(object), + &key, + at(add_seconds), + ) + .unwrap(); + let remove = backend::write_change( + &cob_home(), + &repo, + &nsid, + &Tag::Remove("nel".into()).encode().unwrap(), + &[root], + Some(object), + &key, + at(remove_seconds), + ) + .unwrap(); + let merge = backend::write_change( + &cob_home(), + &repo, + &nsid, + &Tag::Add("merged".into()).encode().unwrap(), + &[add, remove], + Some(object), + &key, + at(100), + ) + .unwrap(); + let name = backend::cob_ref_name(&nsid, object).unwrap(); + repo.update_ref(&RefUpdate::Create { + name, + new: merge.oid(), + }) + .unwrap(); + CobStore::new(&repo) + .get::(object) + .unwrap() + .into_state() + .contains("nel") + }; + + assert!(resolve(3, 2), "later add resurrects removed element"); + assert!(!resolve(2, 3), "later remove wins under last-writer-wins"); + } + + #[test] + fn tip_not_descending_from_root_is_detached() { + let (_dir, repo) = fixture(); + let key = signer(22); + let nsid = Tag::type_name(); + let root = backend::write_change( + &cob_home(), + &repo, + &nsid, + &Tag::Add("nel".into()).encode().unwrap(), + &[], + None, + &key, + at(1), + ) + .unwrap(); + let object = CobId::new(root.oid()); + let unrelated = backend::write_change( + &cob_home(), + &repo, + &nsid, + &Tag::Add("olaren".into()).encode().unwrap(), + &[], + None, + &key, + at(2), + ) + .unwrap(); + let name = backend::cob_ref_name(&nsid, object).unwrap(); + repo.update_ref(&RefUpdate::Create { + name, + new: unrelated.oid(), + }) + .unwrap(); + + assert!(matches!( + CobStore::new(&repo).get::(object), + Err(CobError::DetachedTip(_)) + )); + } + + #[test] + fn deep_history_loads_and_orders_without_overflow() { + let count: i64 = 8_000; + let (_dir, repo) = fixture(); + let key = signer(23); + let nsid = Tag::type_name(); + let payload = Tag::Add("nel".into()).encode().unwrap(); + let root = + backend::write_change(&cob_home(), &repo, &nsid, &payload, &[], None, &key, at(0)) + .unwrap(); + let object = CobId::new(root.oid()); + let tip = (1..count).fold(root, |parent, i| { + backend::write_change( + &cob_home(), + &repo, + &nsid, + &payload, + &[parent], + Some(object), + &key, + at(i), + ) + .unwrap() + }); + publish(&repo, object, tip); + let graph = CobStore::new(&repo).graph::(object).unwrap(); + assert_eq!(graph.len(), count as usize); + assert_eq!(graph.causal_order().len(), count as usize); + + let synthetic: u64 = 100_000; + let oid = |index: u64| knot_types::Oid::from_hex(&format!("{index:040x}")).unwrap(); + let actor = ActorId::from_secp256k1(&[0x02; 33]); + let changes: std::collections::BTreeMap = (1..=synthetic) + .map(|index| { + let id = ChangeId::new(oid(index)); + let parents = if index == 1 { + Vec::new() + } else { + vec![ChangeId::new(oid(index - 1))] + }; + let change = Change { + id, + revision: oid(index), + parents, + type_name: Tag::type_name(), + author: actor.clone(), + signature: Signature::from_bytes(Vec::new()), + payload: Payload::new(Vec::new()), + timestamp: UnixSeconds::new(index as i64), + }; + (id, change) + }) + .collect(); + let synthetic_order = ChangeGraph::new(CobId::new(oid(1)), changes).causal_order(); + assert_eq!(synthetic_order.len(), synthetic as usize); + assert_eq!(synthetic_order.first(), Some(&ChangeId::new(oid(1)))); + assert_eq!(synthetic_order.last(), Some(&ChangeId::new(oid(synthetic)))); + } + + #[test] + fn grafted_foreign_genesis_is_refused() { + let (_dir, repo) = fixture(); + let key = signer(31); + let nsid = Tag::type_name(); + let root = backend::write_change( + &cob_home(), + &repo, + &nsid, + &Tag::Add("nel".into()).encode().unwrap(), + &[], + None, + &key, + at(1), + ) + .unwrap(); + let object = CobId::new(root.oid()); + let child = backend::write_change( + &cob_home(), + &repo, + &nsid, + &Tag::Add("olaren".into()).encode().unwrap(), + &[root], + Some(object), + &key, + at(2), + ) + .unwrap(); + let foreign = backend::write_change( + &cob_home(), + &repo, + &nsid, + &Tag::Add("evil".into()).encode().unwrap(), + &[], + None, + &key, + at(3), + ) + .unwrap(); + let merge = backend::write_change( + &cob_home(), + &repo, + &nsid, + &Tag::Add("merge".into()).encode().unwrap(), + &[child, foreign], + Some(object), + &key, + at(4), + ) + .unwrap(); + let name = backend::cob_ref_name(&nsid, object).unwrap(); + repo.update_ref(&RefUpdate::Create { + name, + new: merge.oid(), + }) + .unwrap(); + + assert!(matches!( + CobStore::new(&repo).get::(object), + Err(CobError::MultipleRoots { .. }) + )); + } + + #[test] + fn cob_ref_name_and_parse_cob_ref_are_inverses() { + let nsid = Tag::type_name(); + let object = CobId::new(knot_types::Oid::from_hex(&"a".repeat(40)).unwrap()); + let name = backend::cob_ref_name(&nsid, object).unwrap(); + + assert_eq!(parse_cob_ref(name.as_str()), Some((nsid, object))); + assert_eq!(parse_cob_ref("refs/heads/main"), None); + assert_eq!( + parse_cob_ref("refs/cobs/sh.tangled.test.tag/not-an-oid"), + None + ); + } + + fn loose_commits(repo: &Repo) -> BTreeSet { + std::fs::read_dir(repo.path().join("objects")) + .unwrap() + .filter_map(Result::ok) + .filter(|shard| { + shard + .file_name() + .to_str() + .map(|s| s.len() == 2) + .unwrap_or(false) + }) + .flat_map(|shard| { + let prefix = shard.file_name().to_str().unwrap().to_string(); + std::fs::read_dir(shard.path()) + .unwrap() + .filter_map(Result::ok) + .filter_map(move |entry| { + let rest = entry.file_name().to_str()?.to_string(); + knot_types::Oid::from_hex(&format!("{prefix}{rest}")).ok() + }) + .collect::>() + }) + .filter(|oid| { + repo.git() + .find_object(oid.object_id()) + .map(|object| object.kind == gix::object::Kind::Commit) + .unwrap_or(false) + }) + .collect() + } + + #[test] + fn a_contended_retry_keeps_both_writes_and_leaves_no_dangling_commit() { + let (_dir, repo) = fixture(); + let store = CobStore::new(&repo); + let key = signer(124); + let created = store + .create(&cob_home(), &Tag::Add("base".into()), &key, at(1)) + .unwrap(); + + let injected = std::cell::Cell::new(false); + let mine = store.update_with::( + &cob_home(), + created.object, + &key, + at(3), + |_state| { + if !injected.replace(true) { + store + .update( + &cob_home(), + created.object, + &Tag::Add("intruder".into()), + &key, + at(2), + ) + .unwrap(); + } + Ok(Tag::Add("mine".into())) + }, + ); + assert!( + mine.is_ok(), + "handler retried instead of surfacing StaleTip" + ); + + let state = store.get::(created.object).unwrap().into_state(); + assert!( + state.contains("intruder"), + "concurrent append survives retry" + ); + assert!( + state.contains("mine"), + "retried append isn't lost to stale tip" + ); + + let reachable: BTreeSet = store + .graph::(created.object) + .unwrap() + .causal_order() + .into_iter() + .map(|change| change.oid()) + .collect(); + assert_eq!( + reachable.len(), + 3, + "live chain is base, intruder, then mine" + ); + assert_eq!( + loose_commits(&repo), + reachable, + "abandoned compare-and-swap attempt left no dangling commit behind" + ); + } + + #[test] + fn exhausting_the_retry_budget_is_contended_and_leaves_no_dangling_commits() { + let (_dir, repo) = fixture(); + let store = CobStore::new(&repo); + let key = signer(126); + let created = store + .create(&cob_home(), &Tag::Add("base".into()), &key, at(1)) + .unwrap(); + let calls = std::cell::Cell::new(0i64); + + let result = store.update_with::( + &cob_home(), + created.object, + &key, + at(1000), + |_state| { + let i = calls.get(); + calls.set(i + 1); + store + .update( + &cob_home(), + created.object, + &Tag::Add(format!("intruder{i}")), + &key, + at(100 + i), + ) + .unwrap(); + Ok(Tag::Add("mine".into())) + }, + ); + + assert!(matches!(result, Err(CobError::Contended(_)))); + assert_eq!( + calls.get(), + MAX_CAS_RETRIES as i64, + "decision ran exactly the retry budget before failing closed" + ); + + let reachable: BTreeSet = store + .graph::(created.object) + .unwrap() + .causal_order() + .into_iter() + .map(|change| change.oid()) + .collect(); + assert_eq!( + reachable.len(), + 1 + MAX_CAS_RETRIES, + "base plus one landed commit per injected intruder" + ); + assert_eq!( + loose_commits(&repo), + reachable, + "every failed attempt across whole budget cleaned up its own orphan" + ); + } + + #[test] + fn an_identical_concurrent_change_is_not_cleaned_up_as_the_live_tip() { + let (_dir, repo) = fixture(); + let store = CobStore::new(&repo); + let key = signer(125); + let nsid = Tag::type_name(); + let created = store + .create(&cob_home(), &Tag::Add("base".into()), &key, at(1)) + .unwrap(); + let dup = Tag::Add("dup".into()); + let bytes = dup.encode().unwrap(); + + let first = backend::write_change( + &cob_home(), + &repo, + &nsid, + &bytes, + &[created.tip], + Some(created.object), + &key, + at(2), + ) + .unwrap(); + let second = backend::write_change( + &cob_home(), + &repo, + &nsid, + &bytes, + &[created.tip], + Some(created.object), + &key, + at(2), + ) + .unwrap(); + assert_eq!( + first, second, + "byte-identical changes are content-addressed to one commit" + ); + + let winner = store + .append( + &cob_home(), + created.object, + &nsid, + created.tip, + &dup, + &key, + at(2), + ) + .unwrap(); + assert_eq!(winner, first); + + let loser = store.append( + &cob_home(), + created.object, + &nsid, + created.tip, + &dup, + &key, + at(2), + ); + assert!(matches!(loser, Err(CobError::StaleTip { .. }))); + + assert_eq!( + backend::resolve_tip(&repo, &nsid, created.object).unwrap(), + Some(winner.oid()), + "shared live tip survived identical-write loser's cleanup" + ); + assert!( + store.get::(created.object).is_ok(), + "live tip wasn't deleted out from under the object" + ); + } + + #[test] + fn collect_refuses_a_graph_past_its_limit() { + let (_dir, repo) = fixture(); + let key = signer(34); + let nsid = Tag::type_name(); + let payload = Tag::Add("nel".into()).encode().unwrap(); + let root = + backend::write_change(&cob_home(), &repo, &nsid, &payload, &[], None, &key, at(0)) + .unwrap(); + let object = CobId::new(root.oid()); + let tip = (1..4).fold(root, |parent, i| { + backend::write_change( + &cob_home(), + &repo, + &nsid, + &payload, + &[parent], + Some(object), + &key, + at(i), + ) + .unwrap() + }); + + assert!(matches!( + backend::collect(&repo, tip, object, 2, None), + Err(CobError::HistoryTooLong(_)) + )); + assert_eq!( + backend::collect(&repo, tip, object, 10, None) + .unwrap() + .len(), + 4 + ); + } + + #[test] + fn changes_since_returns_a_suffix_enforces_shape_and_rejects_a_non_ancestor() { + let (_dir, repo) = fixture(); + let store = CobStore::new(&repo); + let key = signer(60); + + let created = store + .create(&cob_home(), &Tag::Add("nel".into()), &key, at(1)) + .unwrap(); + let full = store.changes_since::(created.object, None).unwrap(); + assert_eq!(full.tip, created.tip); + assert_eq!( + full.changes.iter().map(|c| c.id).collect::>(), + vec![created.tip] + ); + + let second = store + .update( + &cob_home(), + created.object, + &Tag::Add("olaren".into()), + &key, + at(2), + ) + .unwrap(); + let third = store + .update( + &cob_home(), + created.object, + &Tag::Add("teq".into()), + &key, + at(3), + ) + .unwrap(); + + let delta = store + .changes_since::(created.object, Some(created.tip)) + .unwrap(); + assert_eq!(delta.tip, third); + assert_eq!( + delta.changes.iter().map(|c| c.id).collect::>(), + vec![second, third], + "delta is the appended changes in causal order instead of whole graph" + ); + + let caught_up = store + .changes_since::(created.object, Some(third)) + .unwrap(); + assert!(caught_up.changes.is_empty()); + assert_eq!(caught_up.tip, third); + + let (_diverged_dir, diverged_repo) = fixture(); + let diverged = linear_chain(&diverged_repo, &signer(71), &["nel", "olaren"]); + let stranger = ChangeId::new(knot_types::Oid::from_hex(&"a".repeat(40)).unwrap()); + assert!( + matches!( + CobStore::new(&diverged_repo).changes_since::(diverged, Some(stranger)), + Err(CobError::DivergedTip { .. }) + ), + "a since that doesn't descend from the indexed tip is refused, not re-folded onto stale state" + ); + + let (_forked_dir, forked_repo) = fixture(); + let forked = forked_tag_object(&forked_repo, &signer(70), "nel", "olaren", "teq"); + let forked_store = CobStore::new(&forked_repo); + assert!( + matches!( + forked_store.get::(forked), + Err(CobError::ForkedHistory { .. }) + ), + "materialization fails closed on forked linear history" + ); + assert!( + matches!( + forked_store.changes_since::(forked, None), + Err(CobError::ForkedHistory { .. }) + ), + "changes_since refuses the same fork instead of folding it for the index" + ); + assert!( + forked_store.changes_since::(forked, None).is_ok(), + "convergent class still folds the same graph" + ); + } + + struct LinearTags; + + impl Evaluate for LinearTags { + type State = BTreeSet; + type Change = Tag; + + const HISTORY: HistoryModel = HistoryModel::Linear; + + fn initial() -> Self::State { + BTreeSet::new() + } + + fn apply(state: Self::State, change: Self::Change, author: &ActorId) -> Self::State { + Tags::apply(state, change, author) + } + } + + #[test] + fn verify_accepts_the_owner_and_rejects_a_foreign_signer_or_mismatched_type() { + let (_dir, repo) = fixture(); + let store = CobStore::new(&repo); + let key = signer(51); + let object = linear_chain(&repo, &key, &["nel", "olaren"]); + let owner = ActorId::from_secp256k1(key.public_key().as_bytes()); + assert!(store.verify::(&cob_home(), object, &owner).is_ok()); + + let stranger = ActorId::from_secp256k1(signer(52).public_key().as_bytes()); + assert!(matches!( + store.verify::(&cob_home(), object, &stranger), + Err(CobError::UnverifiedChange { .. }) + )); + + let (_other_dir, other_repo) = fixture(); + let key = signer(53); + let foreign = TypeName::new("sh.tangled.test.other").unwrap(); + let root = tag_root(&other_repo, &key, "nel", 1); + let object = CobId::new(root.oid()); + let child = backend::write_change( + &cob_home(), + &other_repo, + &foreign, + &Tag::Add("olaren".into()).encode().unwrap(), + &[root], + Some(object), + &key, + at(2), + ) + .unwrap(); + publish(&other_repo, object, child); + let owner = ActorId::from_secp256k1(key.public_key().as_bytes()); + assert!( + matches!( + CobStore::new(&other_repo).verify::(&cob_home(), object, &owner), + Err(CobError::UnexpectedChangeType { .. }) + ), + "owner-signed change whose type doesn't match namespace is refused at import" + ); + } + + fn ops_and_perm() -> impl Strategy, Vec)> { + prop::collection::vec((any::(), 0u8..4u8), 1..8usize).prop_flat_map(|ops| { + let len = ops.len(); + (Just(ops), prop::collection::vec(any::(), len)) + }) + } + + fn permutation(keys: &[u32]) -> Vec { + let mut order: Vec = (0..keys.len()).collect(); + order.sort_by_key(|&index| keys[index]); + order + } + + fn model_state(ops: &[(bool, u8)]) -> BTreeSet { + ops.iter().fold( + BTreeSet::from(["base".to_string()]), + |mut state, (is_add, subject)| { + let name = format!("s{subject}"); + if *is_add { + state.insert(name); + } else { + state.remove(&name); + } + state + }, + ) + } + + fn build_repo(ops: &[(bool, u8)], creation_order: &[usize]) -> (TempDir, Repo, CobId) { + let (dir, repo) = fixture(); + let key = signer(500); + let nsid = Tag::type_name(); + let root = tag_root(&repo, &key, "base", 1); + let object = CobId::new(root.oid()); + let child = |index: usize| { + let (is_add, subject) = ops[index]; + let name = format!("s{subject}"); + let payload = if is_add { + Tag::Add(name) + } else { + Tag::Remove(name) + }; + backend::write_change( + &cob_home(), + &repo, + &nsid, + &payload.encode().unwrap(), + &[root], + Some(object), + &key, + at(index as i64 + 2), + ) + .unwrap() + }; + creation_order.iter().for_each(|&index| { + child(index); + }); + let parents: Vec = (0..ops.len()).map(child).collect(); + let merge = backend::write_change( + &cob_home(), + &repo, + &nsid, + &Tag::Remove("absent".into()).encode().unwrap(), + &parents, + Some(object), + &key, + at(ops.len() as i64 + 2), + ) + .unwrap(); + publish(&repo, object, merge); + (dir, repo, object) + } + + proptest! { + #![proptest_config(ProptestConfig { cases: 32, ..ProptestConfig::default() })] + + #[test] + fn prop_concurrent_changes_converge_to_the_causal_fold((ops, keys) in ops_and_perm()) { + let identity: Vec = (0..ops.len()).collect(); + let model = model_state(&ops); + + let (_canon_dir, canon_repo, canon_object) = build_repo(&ops, &identity); + let canonical = CobStore::new(&canon_repo) + .get::(canon_object) + .unwrap() + .into_state(); + + let (_perm_dir, perm_repo, perm_object) = build_repo(&ops, &permutation(&keys)); + let permuted = CobStore::new(&perm_repo) + .get::(perm_object) + .unwrap() + .into_state(); + + prop_assert_eq!(&canonical, &model); + prop_assert_eq!(&permuted, &model); + } + + #[test] + fn prop_rematerialization_is_idempotent((ops, keys) in ops_and_perm()) { + let (_dir, repo, object) = build_repo(&ops, &permutation(&keys)); + let store = CobStore::new(&repo); + let first = store.get::(object).unwrap(); + let second = store.get::(object).unwrap(); + prop_assert_eq!(first.state(), second.state()); + prop_assert_eq!(first.history().len(), second.history().len()); + prop_assert_eq!( + store.graph::(object).unwrap().causal_order(), + store.graph::(object).unwrap().causal_order() + ); + } + } +} diff --git a/knot2/crates/knot-cob/src/object.rs b/knot2/crates/knot-cob/src/object.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-cob/src/object.rs @@ -0,0 +1,102 @@ +use knot_types::{ActorId, ChangeId, CobId, TypeName}; + +use crate::change::{Change, ChangePayload}; +use crate::error::CobError; +use crate::graph::{ChangeGraph, History}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HistoryModel { + Linear, + Convergent, +} + +pub trait Evaluate { + type State; + type Change: ChangePayload; + + const HISTORY: HistoryModel; + + fn initial() -> Self::State; + fn apply(state: Self::State, change: Self::Change, author: &ActorId) -> Self::State; +} + +knot_types::scalar_newtype! { + pub struct SnapshotStride(usize); + pub struct StateSize(usize); +} + +pub trait Checkpoint: Evaluate { + const SNAPSHOT_STRIDE: SnapshotStride; + fn checkpoint_size(state: &Self::State) -> StateSize; +} + +pub(crate) fn fold_changes( + state: E::State, + changes: &[Change], + expected: &TypeName, +) -> Result { + changes.iter().try_fold(state, |state, change| { + if change.type_name != *expected { + return Err(CobError::UnexpectedChangeType { + change: change.id, + expected: expected.clone(), + found: change.type_name.clone(), + }); + } + let payload = + E::Change::decode(change.payload()).map_err(|error| CobError::UndecodableChange { + change: change.id, + reason: error.to_string(), + })?; + Ok(E::apply(state, payload, &change.author)) + }) +} + +pub(crate) fn evaluate( + graph: ChangeGraph, + expected: &TypeName, +) -> Result<(E::State, History), CobError> { + let root = ChangeId::new(graph.root().oid()); + let ordered = graph.into_ordered(); + let state = fold_changes::(E::initial(), &ordered, expected)?; + Ok((state, History::new(root, ordered))) +} + +#[derive(Debug)] +pub struct Object { + id: CobId, + type_name: TypeName, + state: S, + history: History, +} + +impl Object { + pub(crate) fn new(id: CobId, type_name: TypeName, state: S, history: History) -> Self { + Self { + id, + type_name, + state, + history, + } + } + + pub fn id(&self) -> CobId { + self.id + } + + pub fn type_name(&self) -> &TypeName { + &self.type_name + } + + pub fn state(&self) -> &S { + &self.state + } + + pub fn into_state(self) -> S { + self.state + } + + pub fn history(&self) -> &History { + &self.history + } +} diff --git a/knot2/crates/knot-cobs/fuzz/.gitignore b/knot2/crates/knot-cobs/fuzz/.gitignore new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-cobs/fuzz/.gitignore @@ -0,0 +1,4 @@ +target +corpus +artifacts +coverage diff --git a/knot2/crates/knot-cobs/fuzz/Cargo.lock b/knot2/crates/knot-cobs/fuzz/Cargo.lock new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-cobs/fuzz/Cargo.lock @@ -0,0 +1,5398 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + +[[package]] +name = "arc-swap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +dependencies = [ + "rustversion", +] + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "async-compression" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "atomic-polyfill" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" +dependencies = [ + "critical-section", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "aws-lc-rs" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + +[[package]] +name = "base-x" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" + +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + +[[package]] +name = "base256emoji" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e9430d9a245a77c92176e649af6e275f20839a48389859d1661e9a128d077c" +dependencies = [ + "const-str", + "match-lookup", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bon" +version = "3.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a602c73c7b0148ec6d12af6fd5cc7a46e2eacc8878271a999abac56eed12f561" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + +[[package]] +name = "borrow-or-share" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" + +[[package]] +name = "borsh" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3f6da4992df95bbcd9af42a6c7dcb994498fc9048230405f3b36ff7cd3f145" +dependencies = [ + "bytes", + "cfg_aliases", +] + +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "bstr" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +dependencies = [ + "serde", +] + +[[package]] +name = "bytesize" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49e78e506b9d7633710dab98996f22f95f3d0f488e8f1aa162830556ed9fc14d" + +[[package]] +name = "cbor4ii" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544cf8c89359205f4f990d0e6f3828db42df85b5dac95d09157a250eb0749c4" +dependencies = [ + "serde", +] + +[[package]] +name = "cc" +version = "1.2.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "cid" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a304f95f84d169a6f31c4d0a30d784643aaa0bbc9c1e449a2c23e963ec4971" +dependencies = [ + "multibase", + "multihash", + "serde", + "serde_bytes", + "unsigned-varint 0.8.0", +] + +[[package]] +name = "clru" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "197fd99cb113a8d5d9b6376f3aa817f32c1078f2343b714fff7d2ca44fdf67d5" +dependencies = [ + "hashbrown 0.16.1", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.18", +] + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "brotli", + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-str" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f421161cb492475f1661ddc9815a745a1c894592070661180fdec3d4872e9c3" + +[[package]] +name = "cordyceps" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "688d7fbb8092b8de775ef2536f36c8c31f2bc4006ece2e8d8ad2d17d00ce0a2a" +dependencies = [ + "loom", + "tracing", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "data-encoding-macro" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3259c913752a86488b501ed8680446a5ed2d5aeac6e596cb23ba3800768ea32c" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccc2776f0c61eca1ca32528f85548abd1a4be8fb53d1b21c013e4f18da1e7090" +dependencies = [ + "data-encoding", + "syn", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "derive_more" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "diatomic-waker" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab03c107fafeb3ee9f5925686dbb7a73bc76e3932abb0d2b365cb64b169cf04c" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "const-oid", + "crypto-common 0.1.6", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest 0.10.7", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "rand_core 0.6.4", + "serde", + "sha2 0.10.9", + "subtle", + "zeroize", +] + +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "pem-rfc7468", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "faster-hex" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7223ae2d2f179b803433d9c830478527e92b8117eab39460edae7f1614d9fb73" +dependencies = [ + "heapless 0.8.0", + "serde", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "fluent-uri" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-buffered" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4421cb78ee172b6b06080093479d3c50f058e7c81b7d577bbb8d118d551d4cd5" +dependencies = [ + "cordyceps", + "diatomic-waker", + "futures-core", + "pin-project-lite", + "spin 0.10.0", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generator" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + +[[package]] +name = "generic-array" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +dependencies = [ + "typenum", + "version_check", + "zeroize", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + +[[package]] +name = "gix" +version = "0.84.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae54ae0ebd1a5a3c3f8d95dd3b5ca6e63f4fed9bfd585e13801a97d7bde8f9ce" +dependencies = [ + "gix-actor", + "gix-archive", + "gix-attributes", + "gix-blame", + "gix-command", + "gix-commitgraph", + "gix-config", + "gix-credentials", + "gix-date", + "gix-diff", + "gix-dir", + "gix-discover", + "gix-error", + "gix-features", + "gix-filter", + "gix-fs", + "gix-glob", + "gix-hash", + "gix-hashtable", + "gix-ignore", + "gix-index", + "gix-lock", + "gix-mailmap", + "gix-negotiate", + "gix-object", + "gix-odb", + "gix-pack", + "gix-path", + "gix-pathspec", + "gix-prompt", + "gix-protocol", + "gix-ref", + "gix-refspec", + "gix-revision", + "gix-revwalk", + "gix-sec", + "gix-shallow", + "gix-status", + "gix-submodule", + "gix-tempfile", + "gix-trace", + "gix-traverse", + "gix-url", + "gix-utils", + "gix-validate", + "gix-worktree", + "gix-worktree-state", + "gix-worktree-stream", + "nonempty", + "parking_lot", + "regex", + "signal-hook", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-actor" +version = "0.41.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bc998b8f746dda8565450d08a63b792ced9165d8c27a1ed3f02799ec6a7820f" +dependencies = [ + "bstr", + "gix-date", + "gix-error", +] + +[[package]] +name = "gix-archive" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16909cacc78936ab96f6c3be08379d0a2e88bfa3a7527972d2ed75c7517ef31e" +dependencies = [ + "bstr", + "flate2", + "gix-date", + "gix-error", + "gix-object", + "gix-path", + "gix-worktree-stream", + "rawzip", + "tar", +] + +[[package]] +name = "gix-attributes" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d43f12e246d3bf7ec624c8fc15ac4a4b62b7c4c6f586cb82be6c90bf84c9d02" +dependencies = [ + "bstr", + "gix-glob", + "gix-path", + "gix-quote", + "gix-trace", + "kstring", + "smallvec", + "thiserror 2.0.18", + "unicode-bom", +] + +[[package]] +name = "gix-bitmap" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ebef0c26ad305747649e727bbcd56a7b7910754eb7cea88f6dff6f93c51283" +dependencies = [ + "gix-error", +] + +[[package]] +name = "gix-blame" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d39a0c14af94c2edaa5eefe06d5ef2cdea55316ae9a9321314288e3f55fa4c0" +dependencies = [ + "gix-commitgraph", + "gix-date", + "gix-diff", + "gix-error", + "gix-hash", + "gix-object", + "gix-revwalk", + "gix-trace", + "gix-traverse", + "gix-worktree", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-chunk" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9faee47943b638e58ddd5e275a4906ad3e4b6c8584f1d41bd18ab9032ec52afb" +dependencies = [ + "gix-error", +] + +[[package]] +name = "gix-command" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00706d4fef135ef4b01680d5218c6ee40cda8baf697b864296cbc887d19118f6" +dependencies = [ + "bstr", + "gix-path", + "gix-quote", + "gix-trace", + "shell-words", +] + +[[package]] +name = "gix-commitgraph" +version = "0.37.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f675d0df484a7f6a47e64bd6f311af489d947c0323b0564f36d14f3d7762abb" +dependencies = [ + "bstr", + "gix-chunk", + "gix-error", + "gix-hash", + "memmap2", + "nonempty", +] + +[[package]] +name = "gix-config" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2372d4b49ca28431e7d150cab9d25edc1890f0184bd57eb0e917c7799e63de" +dependencies = [ + "bstr", + "gix-config-value", + "gix-features", + "gix-glob", + "gix-path", + "gix-ref", + "gix-sec", + "smallvec", + "thiserror 2.0.18", + "unicode-bom", +] + +[[package]] +name = "gix-config-value" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed42168329552f6c2e5df09665c104199d45d84bedb53683738a49b57fe1baab" +dependencies = [ + "bitflags", + "bstr", + "gix-path", + "libc", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-credentials" +version = "0.38.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40cd22f0dd71988be12d6e78b1709de2370e1957c5f107ff31e56caeba3745d" +dependencies = [ + "bstr", + "gix-command", + "gix-config-value", + "gix-date", + "gix-path", + "gix-prompt", + "gix-sec", + "gix-trace", + "gix-url", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-date" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3ecab64a98bbac9f8e02990a9ea5e3c974a7d49b95f2bd70ad94ad22fa6b48c" +dependencies = [ + "bstr", + "gix-error", + "itoa", + "jiff", +] + +[[package]] +name = "gix-diff" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b6d9528f32d94cef2edf39a1ac01fe5a0fc44ddbb18d9e44099936047c3302b" +dependencies = [ + "bstr", + "gix-attributes", + "gix-command", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-imara-diff", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-tempfile", + "gix-trace", + "gix-traverse", + "gix-worktree", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-dir" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21bb2a53a6fd917ec499ed0bfb5b6887de7a15bd79197dcea7c987938749a9f1" +dependencies = [ + "bstr", + "gix-discover", + "gix-fs", + "gix-ignore", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-trace", + "gix-utils", + "gix-worktree", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-discover" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77bacdd12b7879d2178a80c58c2f319995e4654e1a7a23e3181e5c8a12b824f7" +dependencies = [ + "bstr", + "dunce", + "gix-fs", + "gix-path", + "gix-ref", + "gix-sec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-error" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57831e199be480af90dcd7e459abed8a174c09ec9a6e2cc8f7ca6c54598b06b" +dependencies = [ + "bstr", +] + +[[package]] +name = "gix-features" +version = "0.48.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1849ae154d38bc403185be14fa871e38e3c93ee606875d94e207fdb9fba52dbc" +dependencies = [ + "bytes", + "bytesize", + "crc32fast", + "crossbeam-channel", + "gix-path", + "gix-trace", + "gix-utils", + "libc", + "once_cell", + "parking_lot", + "prodash", + "thiserror 2.0.18", + "walkdir", + "zlib-rs", +] + +[[package]] +name = "gix-filter" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecf74b7d16f6694ce4a3049074c41be0c7987105743674f1671807bd6dce09fa" +dependencies = [ + "bstr", + "encoding_rs", + "gix-attributes", + "gix-command", + "gix-hash", + "gix-object", + "gix-packetline", + "gix-path", + "gix-quote", + "gix-trace", + "gix-utils", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-fs" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cdff46db8798e47e2f727d84b9379aac5add3dd3d9d0b07bb4d7d5d640771fe" +dependencies = [ + "bstr", + "fastrand", + "gix-features", + "gix-path", + "gix-utils", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-glob" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1fcb8ef5b16bcf874abe9b68d8abb3c0493c876d367ab824151f30a0f3f3756" +dependencies = [ + "bitflags", + "bstr", + "gix-features", + "gix-path", +] + +[[package]] +name = "gix-hash" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0926d3819c837750b4e03c7754901e73f68b8c9b690753a6372a1bed4eedce" +dependencies = [ + "faster-hex", + "gix-features", + "sha1-checked", + "sha2 0.11.0", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-hashtable" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0e30b93eea8718baf7d8153fcb938e2926175bbf18097c09f1c01b6f0be0563" +dependencies = [ + "gix-hash", + "hashbrown 0.17.1", + "parking_lot", +] + +[[package]] +name = "gix-ignore" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d491bab9bf2c9f341dc754f425c31d5d3f63aca615312167b82e1deeaca97d8d" +dependencies = [ + "bstr", + "gix-glob", + "gix-path", + "gix-trace", + "unicode-bom", +] + +[[package]] +name = "gix-imara-diff" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19753d40da53d0ec41604750eeb969097a90fb2d7f7992730d904541c04e2c19" +dependencies = [ + "bstr", + "hashbrown 0.15.5", +] + +[[package]] +name = "gix-index" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e6b28cc592dc753adb58302bb14a64e412ee591a3bec77aa4df87bff74fa80d" +dependencies = [ + "bitflags", + "bstr", + "filetime", + "fnv", + "gix-bitmap", + "gix-features", + "gix-fs", + "gix-hash", + "gix-lock", + "gix-object", + "gix-traverse", + "gix-utils", + "gix-validate", + "hashbrown 0.17.1", + "itoa", + "libc", + "memmap2", + "rustix", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-lock" +version = "23.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9dedd9e90b0d47624d2ed241d394e09294118364e87b9b7e5f1fe755f3c2c" +dependencies = [ + "gix-tempfile", + "gix-utils", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-mailmap" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "195fd20808055824531be2fd0d34136d900e5fbca3ffb0a3c07e8beeefb9c828" +dependencies = [ + "bstr", + "gix-actor", + "gix-date", + "gix-error", +] + +[[package]] +name = "gix-negotiate" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "890c936a215bae25818c076cb881cb2e54d2c66ba947ba58b8dd47cff921bf55" +dependencies = [ + "bitflags", + "gix-commitgraph", + "gix-date", + "gix-hash", + "gix-object", + "gix-revwalk", +] + +[[package]] +name = "gix-object" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5cd857e29429c7213bdef3f5aef83f8cc124774fe8ae0d27b1607d218d6d525" +dependencies = [ + "bstr", + "gix-actor", + "gix-date", + "gix-features", + "gix-hash", + "gix-hashtable", + "gix-utils", + "gix-validate", + "itoa", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-odb" +version = "0.81.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d004c32858b1556f2d7874405edb3c97dc78fc09beaa87d57bb077ee2858a7d" +dependencies = [ + "arc-swap", + "gix-features", + "gix-fs", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-pack", + "gix-path", + "gix-quote", + "memmap2", + "parking_lot", + "tempfile", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-pack" +version = "0.71.0" +dependencies = [ + "clru", + "gix-chunk", + "gix-diff", + "gix-error", + "gix-features", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-path", + "gix-tempfile", + "gix-traverse", + "parking_lot", + "smallvec", + "thiserror 2.0.18", + "uluru", +] + +[[package]] +name = "gix-packetline" +version = "0.21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b217dd0ee0c4021ecf169a4a519b1b4f80d15e3f3765f3dc466223dc0ac891d7" +dependencies = [ + "bstr", + "faster-hex", + "gix-trace", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-path" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afa6ac14cd14939ea94a496ce7460daa6511c09f5b84757e9cfc6f9c8d0f93a6" +dependencies = [ + "bstr", + "gix-trace", + "gix-validate", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-pathspec" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3050783b41ee11511e1e8fb35623df81806194f4030395f14f48ea37c2798c9f" +dependencies = [ + "bitflags", + "bstr", + "gix-attributes", + "gix-config-value", + "gix-glob", + "gix-path", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-prompt" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee604d7746080ae7e1023bf47204bcc2c5f307bfbe2306a3c90b1bfd1a2c6d8" +dependencies = [ + "gix-command", + "gix-config-value", + "parking_lot", + "rustix", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-protocol" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51dea3acb390707ab868f1f9584f18449eb95d869deffae96768e47d303595ee" +dependencies = [ + "bstr", + "gix-date", + "gix-features", + "gix-hash", + "gix-ref", + "gix-shallow", + "gix-transport", + "gix-utils", + "maybe-async", + "nonempty", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-quote" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6e541fc33cc2b783b7979040d445a0c86a2eca747c8faea4ca84230d06ae6ef" +dependencies = [ + "bstr", + "gix-error", + "gix-utils", +] + +[[package]] +name = "gix-ref" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c04f64c37eb7e6feb73c7060f8dc6f381cc5de5d53249bfd450bc48a86b2e8b" +dependencies = [ + "gix-actor", + "gix-features", + "gix-fs", + "gix-hash", + "gix-lock", + "gix-object", + "gix-path", + "gix-tempfile", + "gix-utils", + "gix-validate", + "memmap2", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-refspec" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b216ae06ec74b5f24ad0142026a997fb0a935b7410eaf9c1616fc3f0e6c5a6d3" +dependencies = [ + "bstr", + "gix-error", + "gix-glob", + "gix-hash", + "gix-revision", + "gix-validate", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-revision" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b47c88884dd3c1a19a39da19d10211fcdea2809aadc86869b6e824a1774340f" +dependencies = [ + "bitflags", + "bstr", + "gix-commitgraph", + "gix-date", + "gix-error", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-revwalk", + "gix-trace", + "nonempty", +] + +[[package]] +name = "gix-revwalk" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85f5756abffe0917827aac683b13684ed99875bc398fa1f9b8f479b0681ef9e6" +dependencies = [ + "gix-commitgraph", + "gix-date", + "gix-error", + "gix-hash", + "gix-hashtable", + "gix-object", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-sec" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab8519976e4c7e486270740a5400369f37940779b80bd1377d94cfa1125d01b3" +dependencies = [ + "bitflags", + "gix-path", + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "gix-shallow" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a292fc2fe548c5dfa575479d16b445b0ddf1dd2f56f1fec6aed386f82553cd97" +dependencies = [ + "bstr", + "gix-hash", + "gix-lock", + "nonempty", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-status" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22042e385d28a34275e029d98f4970285045be14b9073658ca897923f2ed8700" +dependencies = [ + "bstr", + "filetime", + "gix-diff", + "gix-dir", + "gix-features", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-worktree", + "portable-atomic", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-submodule" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3059890ef054066c22a94bfc6a3eaba0d806aedcd630a0bc9e5783fd88884781" +dependencies = [ + "bstr", + "gix-config", + "gix-path", + "gix-pathspec", + "gix-refspec", + "gix-url", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-tempfile" +version = "23.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27850097e1ff9515f46a0dad0f5f9c9d020e972727772dabab9450690c4adb22" +dependencies = [ + "dashmap", + "gix-fs", + "libc", + "parking_lot", + "signal-hook", + "signal-hook-registry", + "tempfile", +] + +[[package]] +name = "gix-trace" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44dc45eae785c0eb14173e0f152e6e224dcf4d45b6a6999a3aed22af541ad678" + +[[package]] +name = "gix-transport" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd0e34995b1aab0fa8dff2af8db726a0bfad3e119c89302604463264046e7ff" +dependencies = [ + "bstr", + "gix-command", + "gix-features", + "gix-packetline", + "gix-quote", + "gix-sec", + "gix-url", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-traverse" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8de590ecc86a3b2870665f2288324fa9f7f8672c7fc2d4e020fdd81cd1f7aed" +dependencies = [ + "bitflags", + "gix-commitgraph", + "gix-date", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-revwalk", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-url" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bb01ec69d55e82ccb7a19e264501ead4e6aac38463a8cebfdd81e22bb67ab2" +dependencies = [ + "bstr", + "gix-path", + "percent-encoding", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-utils" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66c50966184123caf580ffa64e28031a878597f1c7fceb8fe19566c38eb1b771" +dependencies = [ + "bstr", + "fastrand", + "unicode-normalization", +] + +[[package]] +name = "gix-validate" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bc6fc771c4063ba7cd2f47b91fb6076251c6a823b64b7fe7b8874b0fe4afae3" +dependencies = [ + "bstr", +] + +[[package]] +name = "gix-worktree" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef414ed275e8407cd5d53d301e83be19700b0dd3f859d2434417b58f454a2d1" +dependencies = [ + "bstr", + "gix-attributes", + "gix-fs", + "gix-glob", + "gix-hash", + "gix-ignore", + "gix-index", + "gix-object", + "gix-path", + "gix-validate", +] + +[[package]] +name = "gix-worktree-state" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bffae8b3ca258fdd50370cd51f06deb4c76a3b43db3868bc28dde45ffa77d69" +dependencies = [ + "bstr", + "gix-features", + "gix-filter", + "gix-fs", + "gix-index", + "gix-object", + "gix-path", + "gix-worktree", + "io-close", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-worktree-stream" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25e9ed30100c63f7590bc581c225e53f731a53e06aa79a245739c07f7dcc557" +dependencies = [ + "gix-attributes", + "gix-error", + "gix-features", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-object", + "gix-path", + "gix-traverse", + "parking_lot", +] + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "h2" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "heapless" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" +dependencies = [ + "atomic-polyfill", + "hash32 0.2.1", + "rustc_version", + "serde", + "spin 0.9.8", + "stable_deref_trait", +] + +[[package]] +name = "heapless" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32 0.3.1", + "stable_deref_trait", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "human_format" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaec953f16e5bcf6b8a3cb3aa959b17e5577dbd2693e94554c462c08be22624b" + +[[package]] +name = "hybrid-array" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "inventory" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +dependencies = [ + "rustversion", +] + +[[package]] +name = "io-close" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cadcf447f06744f8ce713d2d6239bb5bde2c357a452397a9ed90c625da390bc" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "ipld-core" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090f624976d72f0b0bb71b86d58dc16c15e069193067cb3a3a09d655246cbbda" +dependencies = [ + "cid", + "serde", + "serde_bytes", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "iroh-car" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f8cd4cb9aa083fba8b52e921764252d0b4dcb1cd6d120b809dbfe1106e81a" +dependencies = [ + "anyhow", + "cid", + "futures", + "serde", + "serde_ipld_dagcbor", + "thiserror 1.0.69", + "tokio", + "unsigned-varint 0.7.2", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jacquard-api" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5c803a3c097e3ef8aea63747b4fe3fc9e339cd18272dd0366b1d10dd90d5c3f" +dependencies = [ + "jacquard-common", + "jacquard-derive", + "jacquard-lexicon", + "miette", + "serde", + "thiserror 2.0.18", +] + +[[package]] +name = "jacquard-common" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fec8b6661e6fcfb4d8b9eb1132503be71bc3e3c6f6ff15fa5500ec62655f6da7" +dependencies = [ + "base64", + "bon", + "bytes", + "chrono", + "ciborium", + "ciborium-io", + "cid", + "ed25519-dalek", + "fluent-uri", + "futures", + "getrandom 0.2.17", + "getrandom 0.3.4", + "hashbrown 0.15.5", + "http", + "ipld-core", + "k256", + "maitake-sync", + "miette", + "multibase", + "multihash", + "n0-future", + "oxilangtag", + "p256", + "phf", + "postcard", + "rand 0.9.4", + "regex", + "regex-automata", + "regex-lite", + "reqwest 0.12.28", + "rustversion", + "serde", + "serde_bytes", + "serde_html_form", + "serde_ipld_dagcbor", + "serde_json", + "signature", + "smol_str", + "spin 0.10.0", + "thiserror 2.0.18", + "tokio", + "tokio-tungstenite-wasm", + "tokio-util", + "trait-variant", + "unicode-segmentation", +] + +[[package]] +name = "jacquard-derive" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e41b82a9a72c0d2b907a5e0734e0692615e717cda903502c6c034ae6f3ddddf7" +dependencies = [ + "heck", + "jacquard-lexicon", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jacquard-lexicon" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "160049c269e3d7ec56f130736d25d61953fc6216da660c8b5e102f6b052dd3a1" +dependencies = [ + "cid", + "dashmap", + "heck", + "inventory", + "jacquard-common", + "miette", + "multihash", + "prettyplease", + "proc-macro2", + "quote", + "serde", + "serde_ipld_dagcbor", + "serde_json", + "serde_path_to_error", + "serde_repr", + "serde_with", + "sha2 0.10.9", + "syn", + "thiserror 2.0.18", + "unicode-segmentation", +] + +[[package]] +name = "jacquard-repo" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98986367bb78dadaa0f2f07196bab357786c0e3670d8311b350585b91f84d6eb" +dependencies = [ + "bytes", + "cid", + "ed25519-dalek", + "iroh-car", + "jacquard-api", + "jacquard-common", + "jacquard-derive", + "k256", + "miette", + "multihash", + "n0-future", + "p256", + "serde", + "serde_bytes", + "serde_ipld_dagcbor", + "sha2 0.10.9", + "smol_str", + "thiserror 2.0.18", + "tokio", + "trait-variant", +] + +[[package]] +name = "jiff" +version = "0.2.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4603d3033e49e2b0e31229fcab20a5d40089c607d975cd9c80551dc69eed9102" +dependencies = [ + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link", +] + +[[package]] +name = "jiff-static" +version = "0.2.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "782d32378dddf207193ac91cefb848ad41abb58195c95168e1291227a0832b47" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c900ef84826f1338a557697dc8fc601df9ca9af4ac137c7fb61d4c6f2dfd3076" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "k256" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" +dependencies = [ + "cfg-if", + "ecdsa", + "elliptic-curve", + "once_cell", + "sha2 0.10.9", + "signature", +] + +[[package]] +name = "knot-cob" +version = "0.1.0" +dependencies = [ + "gix", + "gix-hash", + "k256", + "knot-git", + "knot-resource", + "knot-runtime", + "knot-types", + "serde", + "serde_ipld_dagcbor", + "thiserror 2.0.18", + "tracing", +] + +[[package]] +name = "knot-cobs" +version = "0.1.0" +dependencies = [ + "knot-cob", + "knot-runtime", + "knot-types", + "serde", + "thiserror 2.0.18", +] + +[[package]] +name = "knot-cobs-fuzz" +version = "0.0.0" +dependencies = [ + "knot-cobs", + "libfuzzer-sys", +] + +[[package]] +name = "knot-git" +version = "0.1.0" +dependencies = [ + "base64", + "dashmap", + "flate2", + "gix", + "gix-archive", + "gix-bitmap", + "gix-hash", + "gix-pack", + "knot-resource", + "knot-types", + "moka", + "thiserror 2.0.18", + "walkdir", +] + +[[package]] +name = "knot-resource" +version = "0.1.0" +dependencies = [ + "rustix", +] + +[[package]] +name = "knot-runtime" +version = "0.1.0" +dependencies = [ + "bytes", + "futures", + "getrandom 0.4.3", + "http", + "k256", + "reqwest 0.13.1", + "thiserror 2.0.18", + "tokio", + "url", +] + +[[package]] +name = "knot-types" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "chrono", + "cid", + "gix-hash", + "http", + "jacquard-common", + "jacquard-derive", + "jacquard-lexicon", + "jacquard-repo", + "miette", + "serde", + "serde_json", + "thiserror 2.0.18", + "walkdir", +] + +[[package]] +name = "kstring" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "558bf9508a558512042d3095138b1f7b8fe90c5467d94f9f1da28b3731c5dbd1" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" + +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "maitake-sync" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6816ab14147f80234c675b80ed6dc4f440d8a1cefc158e766067aedb84c0bcd5" +dependencies = [ + "cordyceps", + "loom", + "mycelium-bitfield", + "pin-project", + "portable-atomic", +] + +[[package]] +name = "match-lookup" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "757aee279b8bdbb9f9e676796fd459e4207a1f986e87886700abf589f5abf771" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "maybe-async" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "746873a384ad60adc5db74471dfaba74bd278afbdcfd81db93fafcdfc8b5ca0c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +dependencies = [ + "libc", +] + +[[package]] +name = "miette" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" +dependencies = [ + "cfg-if", + "miette-derive", + "unicode-width", +] + +[[package]] +name = "miette-derive" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "moka" +version = "0.12.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "event-listener", + "futures-util", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "multibase" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8694bb4835f452b0e3bb06dbebb1d6fc5385b6ca1caf2e55fd165c042390ec77" +dependencies = [ + "base-x", + "base256emoji", + "data-encoding", + "data-encoding-macro", +] + +[[package]] +name = "multihash" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "577c63b00ad74d57e8c9aa870b5fccebf2fd64a308a5aee9f1bb88e4aea19447" +dependencies = [ + "serde", + "unsigned-varint 0.8.0", +] + +[[package]] +name = "mycelium-bitfield" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24e0cc5e2c585acbd15c5ce911dff71e1f4d5313f43345873311c4f5efd741cc" + +[[package]] +name = "n0-future" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb0e5d99e681ab3c938842b96fcb41bf8a7bb4bfdb11ccbd653a7e83e06c794" +dependencies = [ + "cfg_aliases", + "derive_more", + "futures-buffered", + "futures-lite", + "futures-util", + "js-sys", + "pin-project", + "send_wrapper", + "tokio", + "tokio-util", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "nonempty" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9737e026353e5cd0736f98eddae28665118eb6f6600902a7f50db585621fecb6" + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.60.2", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "oxilangtag" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d3b4eb570abd4a1dcb062c31fd37b832264d9dc7292c3e69acfe926c87b063f" +dependencies = [ + "serde", +] + +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2 0.10.9", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand 0.8.6", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "heapless 0.7.17", + "serde", +] + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prodash" +version = "31.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "962200e2d7d551451297d9fdce85138374019ada198e30ea9ede38034e27604c" +dependencies = [ + "bytesize", + "human_format", + "parking_lot", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.4", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rawzip" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d9575f44c8cf85bc843ad666dcdf20d05a7753772bef56eb2a5140282b32150" + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "regex" +version = "1.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "mime", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "reqwest" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04e9018c9d814e5f30cc16a0f03271aeab3571e609612d9fe78c1aa8d11c2f62" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "aws-lc-rs", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "send_wrapper" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_html_form" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acf96b1d9364968fce46ebb548f1c0e1d7eceae27bdff73865d42e6c7369d94" +dependencies = [ + "form_urlencoded", + "indexmap", + "itoa", + "serde_core", +] + +[[package]] +name = "serde_ipld_dagcbor" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46182f4f08349a02b45c998ba3215d3f9de826246ba02bb9dddfe9a2a2100778" +dependencies = [ + "cbor4ii", + "ipld-core", + "scopeguard", + "serde", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" +dependencies = [ + "base64", + "bs58", + "chrono", + "hex", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha1-checked" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89f599ac0c323ebb1c6082821a54962b839832b03984598375bff3975b804423" +dependencies = [ + "digest 0.10.7", + "sha1", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shell-words" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a0c28ca5908dbdbcd52e6fdaa00358ab88637f8ab33e1f188dd510eb44b53d" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "smol_str" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa7368fcf4852a4c2dd92df0cace6a71f2091ca0a23391ce7f3a31833f1523" +dependencies = [ + "borsh", + "serde_core", +] + +[[package]] +name = "socket2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.52.0", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711a53c2d47bbd818258c498c8dbfe186a2526c631495cfe7e078567f86b8469" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" +dependencies = [ + "futures-util", + "log", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tungstenite", +] + +[[package]] +name = "tokio-tungstenite-wasm" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e21a5c399399c3db9f08d8297ac12b500e86bca82e930253fdc62eaf9c0de6ae" +dependencies = [ + "futures-channel", + "futures-util", + "http", + "httparse", + "js-sys", + "rustls", + "thiserror 1.0.69", + "tokio", + "tokio-tungstenite", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "async-compression", + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tokio", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "trait-variant" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.8.6", + "rustls", + "rustls-pki-types", + "sha1", + "thiserror 1.0.69", + "utf-8", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "uluru" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c8a2469e56e6e5095c82ccd3afb98dad95f7af7929aab6d8ba8d6e0f73657da" +dependencies = [ + "arrayvec", +] + +[[package]] +name = "unicode-bom" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eec5d1121208364f6793f7d2e222bf75a915c19557537745b195b253dd64217" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "unsigned-varint" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" + +[[package]] +name = "unsigned-varint" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", + "serde_derive", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "webpki-roots" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zlib-rs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/knot2/crates/knot-cobs/fuzz/Cargo.toml b/knot2/crates/knot-cobs/fuzz/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-cobs/fuzz/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "knot-cobs-fuzz" +version = "0.0.0" +publish = false +edition = "2024" + +[package.metadata] +cargo-fuzz = true + +[workspace] + +[dependencies] +libfuzzer-sys = "0.4" + +[dependencies.knot-cobs] +path = ".." + +[[bin]] +name = "cob_change" +path = "fuzz_targets/cob_change.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "cob_ref" +path = "fuzz_targets/cob_ref.rs" +test = false +doc = false +bench = false + +[patch.crates-io] +gix-pack = { path = "../../../third_party/gix-pack" } diff --git a/knot2/crates/knot-cobs/src/blocklist.rs b/knot2/crates/knot-cobs/src/blocklist.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-cobs/src/blocklist.rs @@ -0,0 +1,20 @@ +use crate::grant::grant_set_cob; + +grant_set_cob! { + change = BlocklistChange, + cob = BlocklistCob, + state = Blocklist, + type_name = "sh.tangled.knot.block", + add = block_account, + remove = unblock_account, +} + +#[cfg(test)] +mod tests { + use knot_cob::ChangePayload; + + #[test] + fn type_name_is_stable() { + assert_eq!(super::BlocklistChange::TYPE, "sh.tangled.knot.block"); + } +} diff --git a/knot2/crates/knot-cobs/src/collaborators.rs b/knot2/crates/knot-cobs/src/collaborators.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-cobs/src/collaborators.rs @@ -0,0 +1,23 @@ +use crate::grant::grant_set_cob; + +grant_set_cob! { + change = CollaboratorsChange, + cob = CollaboratorsCob, + state = Collaborators, + type_name = "sh.tangled.repo.collaborator", + add = add_collaborator, + remove = remove_collaborator, +} + +#[cfg(test)] +mod tests { + use knot_cob::ChangePayload; + + #[test] + fn type_name_is_stable() { + assert_eq!( + super::CollaboratorsChange::TYPE, + "sh.tangled.repo.collaborator" + ); + } +} diff --git a/knot2/crates/knot-cobs/src/grant.rs b/knot2/crates/knot-cobs/src/grant.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-cobs/src/grant.rs @@ -0,0 +1,214 @@ +use std::collections::BTreeMap; + +use knot_types::{AccountDid, UnixSeconds}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Grant { + pub subject: AccountDid, + pub added_by: AccountDid, + pub created_at: UnixSeconds, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Removal { + pub subject: AccountDid, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Entry { + pub added_by: AccountDid, + pub created_at: UnixSeconds, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Roster { + entries: BTreeMap, +} + +impl Roster { + pub fn empty() -> Self { + Self { + entries: BTreeMap::new(), + } + } + + pub fn admit(mut self, grant: Grant) -> Self { + self.entries.entry(grant.subject).or_insert(Entry { + added_by: grant.added_by, + created_at: grant.created_at, + }); + self + } + + pub fn revoke(mut self, removal: Removal) -> Self { + self.entries.remove(&removal.subject); + self + } + + pub fn get(&self, subject: &AccountDid) -> Option<&Entry> { + self.entries.get(subject) + } + + pub fn contains(&self, subject: &AccountDid) -> bool { + self.entries.contains_key(subject) + } + + pub fn len(&self) -> usize { + self.entries.len() + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + pub fn entries(&self) -> impl Iterator { + self.entries.iter() + } +} + +pub trait GrantChange { + fn subject(&self) -> &AccountDid; + fn adds(&self) -> bool; + fn as_grant(&self) -> Option<&Grant>; +} + +macro_rules! grant_set_cob { + ( + change = $change:ident, + cob = $cob:ident, + state = $state:ident, + type_name = $type_name:literal, + add = $add:ident, + remove = $remove:ident $(,)? + ) => { + #[derive(Debug, Clone, PartialEq, Eq, ::serde::Serialize, ::serde::Deserialize)] + #[serde(tag = "op", content = "data", rename_all = "snake_case")] + pub enum $change { + Add($crate::grant::Grant), + Remove($crate::grant::Removal), + } + + impl ::knot_cob::ChangePayload for $change { + const TYPE: &'static str = $type_name; + } + + impl $crate::grant::GrantChange for $change { + fn subject(&self) -> &::knot_types::AccountDid { + match self { + $change::Add(grant) => &grant.subject, + $change::Remove(removal) => &removal.subject, + } + } + + fn adds(&self) -> bool { + ::core::matches!(self, $change::Add(_)) + } + + fn as_grant(&self) -> ::core::option::Option<&$crate::grant::Grant> { + match self { + $change::Add(grant) => ::core::option::Option::Some(grant), + $change::Remove(_) => ::core::option::Option::None, + } + } + } + + pub type $state = $crate::grant::Roster; + + pub struct $cob; + + impl ::knot_cob::Evaluate for $cob { + type State = $state; + type Change = $change; + + const HISTORY: ::knot_cob::HistoryModel = ::knot_cob::HistoryModel::Linear; + + fn initial() -> Self::State { + $crate::grant::Roster::empty() + } + + fn apply( + state: Self::State, + change: Self::Change, + _author: &::knot_types::ActorId, + ) -> Self::State { + match change { + $change::Add(grant) => state.admit(grant), + $change::Remove(removal) => state.revoke(removal), + } + } + } + + impl ::knot_cob::Checkpoint for $cob { + const SNAPSHOT_STRIDE: ::knot_cob::SnapshotStride = + ::knot_cob::SnapshotStride::new(256); + fn checkpoint_size(state: &Self::State) -> ::knot_cob::StateSize { + ::knot_cob::StateSize::new(state.len()) + } + } + + pub fn $add( + store: &::knot_cob::CobStore, + home: &::knot_cob::CobHome, + object: ::knot_cob::CobId, + grant: $crate::grant::Grant, + signer: &dyn ::knot_runtime::Signer, + timestamp: ::knot_types::UnixSeconds, + ) -> ::core::result::Result<::knot_cob::ChangeId, ::knot_cob::CobError> { + store.update_with_checkpointed::<$cob, ::knot_cob::CobError>( + home, + object, + signer, + timestamp, + |_state| ::core::result::Result::Ok($change::Add(grant.clone())), + ) + } + + pub fn $remove( + store: &::knot_cob::CobStore, + home: &::knot_cob::CobHome, + object: ::knot_cob::CobId, + removal: $crate::grant::Removal, + signer: &dyn ::knot_runtime::Signer, + timestamp: ::knot_types::UnixSeconds, + ) -> ::core::result::Result<::knot_cob::ChangeId, ::knot_cob::CobError> { + store.update_with_checkpointed::<$cob, ::knot_cob::CobError>( + home, + object, + signer, + timestamp, + |_state| ::core::result::Result::Ok($change::Remove(removal.clone())), + ) + } + }; +} + +pub(crate) use grant_set_cob; + +#[cfg(test)] +mod tests { + use super::*; + + fn did(suffix: &str) -> AccountDid { + AccountDid::new(format!("did:plc:{suffix}")).unwrap() + } + + fn grant(subject: &str, added_by: &str, at: i64) -> Grant { + Grant { + subject: did(subject), + added_by: did(added_by), + created_at: UnixSeconds::new(at), + } + } + + #[test] + fn admit_keeps_the_first_provenance() { + let roster = Roster::empty() + .admit(grant("nel", "olaren", 1)) + .admit(grant("nel", "teq", 5)); + let entry = roster.get(&did("nel")).unwrap(); + assert_eq!(entry.added_by, did("olaren")); + assert_eq!(entry.created_at, UnixSeconds::new(1)); + assert_eq!(roster.len(), 1); + } +} diff --git a/knot2/crates/knot-cobs/src/import.rs b/knot2/crates/knot-cobs/src/import.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-cobs/src/import.rs @@ -0,0 +1,55 @@ +use knot_cob::{ + ActorId, ChangePayload, CobError, CobHome, CobId, CobStore, TypeName, parse_cob_ref, +}; +use knot_types::RefName; + +use crate::collaborators::{CollaboratorsChange, CollaboratorsCob}; +use crate::members::{MembersChange, MembersCob}; +use crate::registry::{RegistryChange, RepoRegistryCob}; + +#[derive(Debug, thiserror::Error)] +pub enum ImportError { + #[error("'{0}' isn't refs/cobs// ref")] + NotCobRef(String), + #[error("no collaborative object type is registered for namespace '{0}'")] + UnknownType(TypeName), + #[error(transparent)] + Cob(#[from] CobError), +} + +type Verifier = fn(&CobStore<'_>, &CobHome, CobId, &ActorId) -> Result<(), CobError>; + +fn verifier_for(type_name: &TypeName) -> Option { + [ + (MembersChange::type_name(), { + |store: &CobStore<'_>, home, object, owner| { + store.verify::(home, object, owner) + } + } as Verifier), + (CollaboratorsChange::type_name(), { + |store: &CobStore<'_>, home, object, owner| { + store.verify::(home, object, owner) + } + } as Verifier), + (RegistryChange::type_name(), { + |store: &CobStore<'_>, home, object, owner| { + store.verify::(home, object, owner) + } + } as Verifier), + ] + .into_iter() + .find_map(|(name, verifier)| (name == *type_name).then_some(verifier)) +} + +pub fn verify_cob_ref( + store: &CobStore, + home: &CobHome, + refname: &RefName, + owner: &ActorId, +) -> Result { + let (type_name, object) = parse_cob_ref(refname.as_str()) + .ok_or_else(|| ImportError::NotCobRef(refname.as_str().to_string()))?; + let verifier = verifier_for(&type_name).ok_or(ImportError::UnknownType(type_name))?; + verifier(store, home, object, owner)?; + Ok(object) +} diff --git a/knot2/crates/knot-cobs/src/lib.rs b/knot2/crates/knot-cobs/src/lib.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-cobs/src/lib.rs @@ -0,0 +1,36 @@ +mod blocklist; +mod collaborators; +mod grant; +mod import; +mod members; +mod registry; + +pub use blocklist::{Blocklist, BlocklistChange, BlocklistCob, block_account, unblock_account}; +pub use collaborators::{ + Collaborators, CollaboratorsChange, CollaboratorsCob, add_collaborator, remove_collaborator, +}; +pub use grant::{Entry, Grant, GrantChange, Removal, Roster}; +pub use import::{ImportError, verify_cob_ref}; +pub use members::{Members, MembersChange, MembersCob, add_member, remove_member}; +pub use registry::{ + Registration, Registry, RegistryChange, RegistryError, Rename, RepoRecord, RepoRef, + RepoRegistryCob, deregister_repo, register_repo, rename_repo, +}; + +#[doc(hidden)] +pub mod fuzz { + use knot_cob::ChangePayload; + + use crate::{BlocklistChange, CollaboratorsChange, MembersChange, RegistryChange}; + + pub fn change_decode(data: &[u8]) { + let _ = RegistryChange::decode(data); + let _ = MembersChange::decode(data); + let _ = BlocklistChange::decode(data); + let _ = CollaboratorsChange::decode(data); + } + + pub fn ref_parse(data: &[u8]) { + let _ = knot_cob::parse_cob_ref(&String::from_utf8_lossy(data)); + } +} diff --git a/knot2/crates/knot-cobs/src/members.rs b/knot2/crates/knot-cobs/src/members.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-cobs/src/members.rs @@ -0,0 +1,62 @@ +use crate::grant::grant_set_cob; + +grant_set_cob! { + change = MembersChange, + cob = MembersCob, + state = Members, + type_name = "sh.tangled.knot.member", + add = add_member, + remove = remove_member, +} + +#[cfg(test)] +mod tests { + use knot_cob::{ChangePayload, Evaluate}; + use knot_types::{AccountDid, ActorId, UnixSeconds}; + + use super::*; + use crate::grant::{Grant, Removal}; + + fn did(suffix: &str) -> AccountDid { + AccountDid::new(format!("did:plc:{suffix}")).unwrap() + } + + fn grant(subject: &str, added_by: &str, at: i64) -> Grant { + Grant { + subject: did(subject), + added_by: did(added_by), + created_at: UnixSeconds::new(at), + } + } + + fn fold(changes: Vec) -> Members { + let author = ActorId::from_secp256k1(&[0x02; 33]); + changes + .into_iter() + .fold(MembersCob::initial(), |state, change| { + MembersCob::apply(state, change, &author) + }) + } + + #[test] + fn members_fold_add_then_remove() { + let state = fold(vec![ + MembersChange::Add(grant("nel", "nel", 1)), + MembersChange::Add(grant("olaren", "nel", 2)), + MembersChange::Remove(Removal { + subject: did("nel"), + }), + ]); + assert!(state.contains(&did("olaren"))); + assert!(!state.contains(&did("nel"))); + assert_eq!(state.len(), 1); + } + + #[test] + fn change_payload_roundtrips_through_dag_cbor() { + assert_eq!(MembersChange::TYPE, "sh.tangled.knot.member"); + let change = MembersChange::Add(grant("nel", "olaren", 9)); + let bytes = change.encode().unwrap(); + assert_eq!(MembersChange::decode(&bytes).unwrap(), change); + } +} diff --git a/knot2/crates/knot-cobs/src/registry.rs b/knot2/crates/knot-cobs/src/registry.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-cobs/src/registry.rs @@ -0,0 +1,552 @@ +use std::collections::BTreeMap; + +use knot_cob::{ + ChangeId, ChangePayload, Checkpoint, CobError, CobHome, CobId, CobStore, Evaluate, + HistoryModel, SnapshotStride, StateSize, +}; +use knot_runtime::Signer; +use knot_types::{ActorId, OwnerDid, RepoDid, RepoName, RepoRkey, UnixSeconds}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Registration { + pub owner: OwnerDid, + pub rkey: RepoRkey, + pub name: RepoName, + pub repo: RepoDid, + pub created_at: UnixSeconds, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Rename { + pub owner: OwnerDid, + pub rkey: RepoRkey, + pub name: RepoName, + pub repo: RepoDid, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RepoRef { + pub owner: OwnerDid, + pub rkey: RepoRkey, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "op", content = "data", rename_all = "snake_case")] +pub enum RegistryChange { + Register(Registration), + Rename(Rename), + Deregister(RepoRef), +} + +impl ChangePayload for RegistryChange { + const TYPE: &'static str = "sh.tangled.knot.repoRegistry"; +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RepoRecord { + pub owner: OwnerDid, + pub rkey: RepoRkey, + pub name: RepoName, + pub created_at: UnixSeconds, +} + +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct Registry { + records: BTreeMap, + aliases: BTreeMap>, +} + +impl Registry { + pub fn resolve(&self, owner: &OwnerDid, rkey: &RepoRkey) -> Option<&RepoDid> { + self.aliases.get(owner)?.get(rkey) + } + + pub fn record_of(&self, repo: &RepoDid) -> Option<&RepoRecord> { + self.records.get(repo) + } + + pub fn owner_of(&self, repo: &RepoDid) -> Option { + self.records.get(repo).map(|record| record.owner.clone()) + } + + pub fn len(&self) -> usize { + self.records.len() + } + + pub fn is_empty(&self) -> bool { + self.records.is_empty() + } + + pub fn records(&self) -> impl Iterator { + self.records.iter() + } + + pub fn aliases(&self) -> impl Iterator { + self.aliases + .iter() + .flat_map(|(owner, names)| names.iter().map(move |(rkey, repo)| (owner, rkey, repo))) + } + + fn canonical_holder(&self, owner: &OwnerDid, rkey: &RepoRkey) -> Option<&RepoDid> { + let holder = self.resolve(owner, rkey)?; + self.records + .get(holder) + .filter(|record| record.rkey == *rkey) + .map(|_| holder) + } + + fn register(mut self, registration: Registration) -> Self { + self = self.drop_repo(®istration.repo); + self = self.steal_alias(®istration.owner, ®istration.rkey, ®istration.repo); + self.aliases + .entry(registration.owner.clone()) + .or_default() + .insert(registration.rkey.clone(), registration.repo.clone()); + self.records.insert( + registration.repo, + RepoRecord { + owner: registration.owner, + rkey: registration.rkey, + name: registration.name, + created_at: registration.created_at, + }, + ); + self + } + + fn rename(mut self, rename: Rename) -> Self { + match self.records.get(&rename.repo) { + Some(record) if record.owner == rename.owner => {} + _ => return self, + } + self = self.steal_alias(&rename.owner, &rename.rkey, &rename.repo); + self.aliases + .entry(rename.owner.clone()) + .or_default() + .insert(rename.rkey.clone(), rename.repo.clone()); + if let Some(record) = self.records.get_mut(&rename.repo) { + record.rkey = rename.rkey; + record.name = rename.name; + } + self + } + + fn deregister(self, target: RepoRef) -> Self { + match self.resolve(&target.owner, &target.rkey).cloned() { + Some(repo) => self.drop_repo(&repo), + None => self, + } + } + + fn steal_alias(mut self, owner: &OwnerDid, rkey: &RepoRkey, target: &RepoDid) -> Self { + match self.resolve(owner, rkey).cloned() { + Some(holder) if holder != *target => { + let canonical = self + .records + .get(&holder) + .is_some_and(|record| record.rkey == *rkey); + if canonical { + self.drop_repo(&holder) + } else { + if let Some(names) = self.aliases.get_mut(owner) { + names.remove(rkey); + } + self.prune_empty_owners() + } + } + _ => self, + } + } + + fn drop_repo(mut self, repo: &RepoDid) -> Self { + self.records.remove(repo); + self.aliases + .values_mut() + .for_each(|names| names.retain(|_, holder| holder != repo)); + self.prune_empty_owners() + } + + fn prune_empty_owners(mut self) -> Self { + self.aliases.retain(|_, names| !names.is_empty()); + self + } +} + +pub struct RepoRegistryCob; + +impl Evaluate for RepoRegistryCob { + type State = Registry; + type Change = RegistryChange; + + const HISTORY: HistoryModel = HistoryModel::Linear; + + fn initial() -> Self::State { + Registry::default() + } + + fn apply(state: Self::State, change: Self::Change, _author: &ActorId) -> Self::State { + match change { + RegistryChange::Register(registration) => state.register(registration), + RegistryChange::Rename(rename) => state.rename(rename), + RegistryChange::Deregister(target) => state.deregister(target), + } + } +} + +impl Checkpoint for RepoRegistryCob { + const SNAPSHOT_STRIDE: SnapshotStride = SnapshotStride::new(256); + fn checkpoint_size(state: &Self::State) -> StateSize { + StateSize::new(state.len()) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum RegistryError { + #[error(transparent)] + Cob(#[from] CobError), + #[error("no repo is registered at {owner}/{rkey}")] + NotRegistered { owner: OwnerDid, rkey: RepoRkey }, + #[error("record key {owner}/{rkey} resolves to {found}, expected {expected}")] + RepoMismatch { + owner: OwnerDid, + rkey: RepoRkey, + expected: RepoDid, + found: RepoDid, + }, + #[error("repo {repo} is already registered as {owner}/{rkey}")] + AlreadyRegistered { + repo: RepoDid, + owner: OwnerDid, + rkey: RepoRkey, + }, + #[error("record key {owner}/{rkey} is canonical key of {existing}")] + RkeyTaken { + owner: OwnerDid, + rkey: RepoRkey, + existing: RepoDid, + }, + #[error("repo {repo} isn't hosted on this knot")] + NotHosted { repo: RepoDid }, + #[error("repo {repo} is no longer registered to {expected}")] + OwnerMoved { repo: RepoDid, expected: OwnerDid }, +} + +pub fn register_repo( + store: &CobStore, + home: &CobHome, + object: CobId, + registration: Registration, + signer: &dyn Signer, + timestamp: UnixSeconds, +) -> Result, RegistryError> { + store.update_maybe_checkpointed::( + home, + object, + signer, + timestamp, + |registry| { + if let Some(holder) = registry.canonical_holder(®istration.owner, ®istration.rkey) + && holder != ®istration.repo + { + return Err(RegistryError::RkeyTaken { + owner: registration.owner.clone(), + rkey: registration.rkey.clone(), + existing: holder.clone(), + }); + } + match registry.record_of(®istration.repo) { + Some(record) + if record.owner != registration.owner || record.rkey != registration.rkey => + { + Err(RegistryError::AlreadyRegistered { + repo: registration.repo.clone(), + owner: record.owner.clone(), + rkey: record.rkey.clone(), + }) + } + Some(_) => Ok(None), + None => Ok(Some(RegistryChange::Register(registration.clone()))), + } + }, + ) +} + +pub fn rename_repo( + store: &CobStore, + home: &CobHome, + object: CobId, + rename: Rename, + signer: &dyn Signer, + timestamp: UnixSeconds, +) -> Result, RegistryError> { + store.update_maybe_checkpointed::( + home, + object, + signer, + timestamp, + |registry| { + let record = + registry + .record_of(&rename.repo) + .ok_or_else(|| RegistryError::NotHosted { + repo: rename.repo.clone(), + })?; + if record.owner != rename.owner { + return Err(RegistryError::OwnerMoved { + repo: rename.repo.clone(), + expected: rename.owner.clone(), + }); + } + if record.rkey == rename.rkey && record.name == rename.name { + return Ok(None); + } + if let Some(holder) = registry.canonical_holder(&rename.owner, &rename.rkey) + && holder != &rename.repo + { + return Err(RegistryError::RkeyTaken { + owner: rename.owner.clone(), + rkey: rename.rkey.clone(), + existing: holder.clone(), + }); + } + Ok(Some(RegistryChange::Rename(rename.clone()))) + }, + ) +} + +pub fn deregister_repo( + store: &CobStore, + home: &CobHome, + object: CobId, + target: RepoRef, + expected: RepoDid, + signer: &dyn Signer, + timestamp: UnixSeconds, +) -> Result { + store.update_with_checkpointed::( + home, + object, + signer, + timestamp, + |registry| match registry.resolve(&target.owner, &target.rkey) { + None => Err(RegistryError::NotRegistered { + owner: target.owner.clone(), + rkey: target.rkey.clone(), + }), + Some(found) if found != &expected => Err(RegistryError::RepoMismatch { + owner: target.owner.clone(), + rkey: target.rkey.clone(), + expected: expected.clone(), + found: found.clone(), + }), + Some(_) => Ok(RegistryChange::Deregister(target.clone())), + }, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn owner(suffix: &str) -> OwnerDid { + OwnerDid::new(format!("did:plc:{suffix}")).unwrap() + } + + fn repo(suffix: &str) -> RepoDid { + RepoDid::new(format!("did:plc:{suffix}")).unwrap() + } + + fn rkey(value: &str) -> RepoRkey { + RepoRkey::new(value).unwrap() + } + + fn name(value: &str) -> RepoName { + RepoName::new(value).unwrap() + } + + fn register(owner_id: &str, key: &str, repo_id: &str, at: i64) -> RegistryChange { + RegistryChange::Register(Registration { + owner: owner(owner_id), + rkey: rkey(key), + name: name(key), + repo: repo(repo_id), + created_at: UnixSeconds::new(at), + }) + } + + fn rename(owner_id: &str, key: &str, repo_id: &str) -> RegistryChange { + RegistryChange::Rename(Rename { + owner: owner(owner_id), + rkey: rkey(key), + name: name(key), + repo: repo(repo_id), + }) + } + + fn deregister(owner_id: &str, key: &str) -> RegistryChange { + RegistryChange::Deregister(RepoRef { + owner: owner(owner_id), + rkey: rkey(key), + }) + } + + fn fold(changes: Vec) -> Registry { + let author = ActorId::from_secp256k1(&[0x02; 33]); + changes + .into_iter() + .fold(RepoRegistryCob::initial(), |state, change| { + RepoRegistryCob::apply(state, change, &author) + }) + } + + #[test] + fn register_maps_owner_and_rkey_to_a_repo() { + let state = fold(vec![register("nel", "anemone", "squid", 5)]); + assert_eq!( + state.resolve(&owner("nel"), &rkey("anemone")), + Some(&repo("squid")) + ); + let record = state.record_of(&repo("squid")).unwrap(); + assert_eq!(record.owner, owner("nel")); + assert_eq!(record.rkey, rkey("anemone")); + assert_eq!(record.name, name("anemone")); + assert_eq!(record.created_at, UnixSeconds::new(5)); + } + + #[test] + fn re_register_replaces_the_repo_under_an_rkey() { + let state = fold(vec![ + register("nel", "anemone", "squid", 1), + register("nel", "anemone", "limpet", 2), + ]); + assert_eq!( + state.resolve(&owner("nel"), &rkey("anemone")), + Some(&repo("limpet")) + ); + assert!( + state.record_of(&repo("squid")).is_none(), + "repo whose canonical rkey is taken by later register is dropped wholesale" + ); + assert_eq!(state.len(), 1); + } + + #[test] + fn rename_retains_the_prior_rkey_as_an_alias() { + let state = fold(vec![ + register("nel", "anemone", "squid", 1), + rename("nel", "barnacle", "squid"), + ]); + assert_eq!( + state.resolve(&owner("nel"), &rkey("barnacle")), + Some(&repo("squid")), + "new rkey resolves" + ); + assert_eq!( + state.resolve(&owner("nel"), &rkey("anemone")), + Some(&repo("squid")), + "prior rkey keeps resolving as an alias" + ); + let record = state.record_of(&repo("squid")).unwrap(); + assert_eq!(record.rkey, rkey("barnacle")); + assert_eq!(record.name, name("barnacle")); + assert_eq!(state.len(), 1); + } + + #[test] + fn rename_of_an_unregistered_repo_is_a_no_op() { + let registered = fold(vec![register("nel", "anemone", "squid", 1)]); + let after = fold(vec![ + register("nel", "anemone", "squid", 1), + rename("nel", "barnacle", "conch"), + ]); + assert_eq!(after, registered); + } + + #[test] + fn rename_under_a_mismatched_owner_is_a_no_op() { + let registered = fold(vec![register("nel", "anemone", "squid", 1)]); + let after = fold(vec![ + register("nel", "anemone", "squid", 1), + rename("olaren", "barnacle", "squid"), + ]); + assert_eq!(after, registered); + } + + #[test] + fn deregister_by_any_alias_removes_the_repo_and_every_alias() { + let state = fold(vec![ + register("nel", "anemone", "squid", 1), + rename("nel", "barnacle", "squid"), + deregister("nel", "anemone"), + deregister("nel", "anemone"), + ]); + assert_eq!(state.resolve(&owner("nel"), &rkey("anemone")), None); + assert_eq!(state.resolve(&owner("nel"), &rkey("barnacle")), None); + assert!(state.is_empty(), "replaying a deregister folds as a no-op"); + assert_eq!(state, Registry::default()); + } + + #[test] + fn a_later_change_steals_a_stale_alias_but_keeps_the_victim_canonical() { + let state = fold(vec![ + register("nel", "anemone", "squid", 1), + rename("nel", "barnacle", "squid"), + register("nel", "anemone", "whelk", 2), + ]); + assert_eq!( + state.resolve(&owner("nel"), &rkey("anemone")), + Some(&repo("whelk")), + "later register wins stale alias" + ); + assert_eq!( + state.resolve(&owner("nel"), &rkey("barnacle")), + Some(&repo("squid")), + "victim keeps its canonical rkey" + ); + assert_eq!(state.len(), 2); + } + + #[test] + fn owner_of_resolves_through_the_record_with_later_register_precedence() { + let unique = fold(vec![ + register("nel", "anemone", "squid", 1), + register("nel", "barnacle", "whelk", 2), + ]); + assert_eq!(unique.owner_of(&repo("squid")), Some(owner("nel"))); + assert_eq!( + unique.record_of(&repo("squid")).unwrap().rkey, + rkey("anemone") + ); + assert_eq!(unique.owner_of(&repo("conch")), None); + + let moved = fold(vec![ + register("nel", "anemone", "squid", 1), + register("olaren", "fork", "squid", 2), + ]); + assert_eq!( + moved.owner_of(&repo("squid")), + Some(owner("olaren")), + "linear causal order gives later register deterministic precedence" + ); + assert_eq!( + moved.resolve(&owner("nel"), &rkey("anemone")), + None, + "re-register under a new owner drops old owner's aliases" + ); + } + + #[test] + fn change_payload_roundtrips_through_dag_cbor() { + [ + register("nel", "anemone", "squid", 5), + rename("nel", "barnacle", "squid"), + deregister("nel", "anemone"), + ] + .into_iter() + .for_each(|change| { + let bytes = change.encode().unwrap(); + assert_eq!(RegistryChange::decode(&bytes).unwrap(), change); + }); + } +} diff --git a/knot2/crates/knot-cobs/tests/cobs.rs b/knot2/crates/knot-cobs/tests/cobs.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-cobs/tests/cobs.rs @@ -0,0 +1,310 @@ +mod common; + +use common::{account, at, did, fixture, grant, home, registration, rename, reopen, rkey, signer}; +use knot_cob::CobStore; +use knot_cobs::{ + CollaboratorsChange, MembersChange, MembersCob, RegistryChange, RegistryError, Removal, + RepoRegistryCob, register_repo, rename_repo, +}; +use knot_types::{AccountDid, OwnerDid, RepoDid}; + +#[test] +fn members_roundtrip_and_reload_is_identical() { + let (_dir, repo) = fixture(); + let key = signer(1); + let store = CobStore::new(&repo); + + let created = store + .create( + &home(), + &MembersChange::Add(grant("nel", "nel", 1)), + &key, + at(1), + ) + .unwrap(); + store + .update( + &home(), + created.object, + &MembersChange::Add(grant("olaren", "nel", 2)), + &key, + at(2), + ) + .unwrap(); + store + .update( + &home(), + created.object, + &MembersChange::Add(grant("teq", "nel", 3)), + &key, + at(3), + ) + .unwrap(); + store + .update( + &home(), + created.object, + &MembersChange::Remove(Removal { + subject: account("teq"), + }), + &key, + at(4), + ) + .unwrap(); + + let object = store.get::(created.object).unwrap(); + let state = object.state(); + assert!(state.contains(&account("nel"))); + assert_eq!( + state.get(&account("olaren")).unwrap().added_by, + account("nel") + ); + assert!(!state.contains(&account("teq"))); + assert_eq!(state.len(), 2); + + let listed: Vec<&AccountDid> = state.entries().map(|(subject, _)| subject).collect(); + assert_eq!(listed, vec![&account("nel"), &account("olaren")]); + + let reopened = reopen(repo); + let reloaded = CobStore::new(&reopened) + .get::(created.object) + .unwrap(); + assert_eq!(state, reloaded.state()); + + let (_dup_dir, dup_repo) = fixture(); + let dup_store = CobStore::new(&dup_repo); + let dup = dup_store + .create( + &home(), + &MembersChange::Add(grant("nel", "olaren", 7)), + &key, + at(1), + ) + .unwrap(); + dup_store + .update( + &home(), + dup.object, + &MembersChange::Add(grant("nel", "olaren", 7)), + &key, + at(2), + ) + .unwrap(); + let replayed = dup_store + .get::(dup.object) + .unwrap() + .into_state(); + assert_eq!(replayed.len(), 1, "replaying an identical add is a no-op"); + let entry = replayed.get(&account("nel")).unwrap(); + assert_eq!(entry.added_by, account("olaren")); + assert_eq!(entry.created_at, at(7)); + + let converge = |order: [(&str, &str, i64); 2]| { + let (_dir, repo) = fixture(); + let store = CobStore::new(&repo); + let [first, second] = order; + let created = store + .create( + &home(), + &MembersChange::Add(grant(first.0, first.1, first.2)), + &key, + at(1), + ) + .unwrap(); + store + .update( + &home(), + created.object, + &MembersChange::Add(grant(second.0, second.1, second.2)), + &key, + at(2), + ) + .unwrap(); + store + .get::(created.object) + .unwrap() + .into_state() + }; + assert_eq!( + converge([("nel", "nel", 10), ("olaren", "nel", 20)]), + converge([("olaren", "nel", 20), ("nel", "nel", 10)]), + "independent grants converge across write order" + ); +} + +#[test] +fn collaborators_live_in_a_per_repo_cob_namespace() { + let (_dir, repo) = fixture(); + let key = signer(2); + let store = CobStore::new(&repo); + store + .create( + &home(), + &CollaboratorsChange::Add(grant("lyna", "nel", 1)), + &key, + at(1), + ) + .unwrap(); + + let cob_ref = repo.references().unwrap().into_iter().find(|record| { + record + .name + .as_str() + .contains("sh.tangled.repo.collaborator") + }); + assert!(cob_ref.is_some(), "collaborators live under refs/cobs"); + assert!(repo.advertised_refs().unwrap().is_empty()); +} + +#[test] +fn registry_handler_semantics() { + let (_dir, repo) = fixture(); + let key = signer(21); + let store = CobStore::new(&repo); + let nel = did::("nel"); + + let created = store + .create( + &home(), + &RegistryChange::Register(registration("nel", "anemone", "squid", 1)), + &key, + at(1), + ) + .unwrap(); + let object = created.object; + + let clash = register_repo( + &store, + &home(), + object, + registration("nel", "anemone", "whelk", 2), + &key, + at(2), + ); + assert!( + matches!(clash, Err(RegistryError::RkeyTaken { .. })), + "register cannot claim the canonical rkey of a live repo under the same owner" + ); + assert_eq!( + register_repo( + &store, + &home(), + object, + registration("nel", "anemone", "squid", 2), + &key, + at(2), + ) + .unwrap(), + None, + "re-registering identical owner, rkey, and repo appends nothing" + ); + + let renamed = rename_repo( + &store, + &home(), + object, + rename("nel", "barnacle", "squid"), + &key, + at(2), + ) + .unwrap(); + assert!(renamed.is_some(), "real rename appends a change"); + let state = store.get::(object).unwrap().into_state(); + assert_eq!( + state.resolve(&nel, &rkey("anemone")), + Some(&did::("squid")), + "prior rkey keeps resolving as an alias" + ); + assert_eq!( + state.resolve(&nel, &rkey("barnacle")), + Some(&did::("squid")) + ); + assert_eq!( + state.record_of(&did("squid")).unwrap().rkey, + rkey("barnacle") + ); + + let redundant = register_repo( + &store, + &home(), + object, + registration("nel", "barnacle", "squid", 3), + &key, + at(3), + ) + .unwrap(); + assert_eq!( + redundant, None, + "re-register matching canonical owner and rkey appends nothing" + ); + assert_eq!( + store + .get::(object) + .unwrap() + .into_state() + .resolve(&nel, &rkey("anemone")), + Some(&did::("squid")), + "retained alias survives a redundant re-register" + ); + + store + .update( + &home(), + object, + &RegistryChange::Register(registration("nel", "kelp", "whelk", 4)), + &key, + at(4), + ) + .unwrap(); + + let taken = rename_repo( + &store, + &home(), + object, + rename("nel", "kelp", "squid"), + &key, + at(5), + ); + assert!( + matches!(taken, Err(RegistryError::RkeyTaken { .. })), + "rename cannot take canonical rkey of another live repo" + ); + + let unhosted = rename_repo( + &store, + &home(), + object, + rename("nel", "limpet", "conch"), + &key, + at(6), + ); + assert!( + matches!(unhosted, Err(RegistryError::NotHosted { .. })), + "rename of a repo with no registration is refused" + ); + + let moved = rename_repo( + &store, + &home(), + object, + rename("olaren", "uni", "squid"), + &key, + at(7), + ); + assert!( + matches!(moved, Err(RegistryError::OwnerMoved { .. })), + "rename under an owner the repo no longer belongs to is refused" + ); + + assert_eq!( + store + .get::(object) + .unwrap() + .into_state() + .record_of(&did("squid")) + .unwrap() + .rkey, + rkey("barnacle"), + "refused renames left the canonical rkey untouched" + ); +} diff --git a/knot2/crates/knot-cobs/tests/fuzz_smoke.rs b/knot2/crates/knot-cobs/tests/fuzz_smoke.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-cobs/tests/fuzz_smoke.rs @@ -0,0 +1,15 @@ +use proptest::prelude::*; + +proptest! { + #![proptest_config(ProptestConfig::with_cases(1024))] + + #[test] + fn the_cob_change_decoder_never_panics(data in proptest::collection::vec(any::(), 0..4096)) { + knot_cobs::fuzz::change_decode(&data); + } + + #[test] + fn the_cob_ref_parser_never_panics(data in proptest::collection::vec(any::(), 0..4096)) { + knot_cobs::fuzz::ref_parse(&data); + } +} diff --git a/knot2/crates/knot-cobs/tests/invariants.rs b/knot2/crates/knot-cobs/tests/invariants.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-cobs/tests/invariants.rs @@ -0,0 +1,460 @@ +mod common; + +use common::{ + account, at, build_members, cob_ref, fixture, forked_members, forked_members_object, grant, + home, members_store, owner_of, registration, registry_with, rkey, signer, write_cob_commit, +}; +use knot_cob::{ChangePayload, CobError, CobHome, CobId, CobStore}; +use knot_cobs::{ + CollaboratorsChange, ImportError, MembersChange, MembersCob, RegistryChange, RegistryError, + Removal, RepoRef, RepoRegistryCob, add_member, deregister_repo, register_repo, verify_cob_ref, +}; +use knot_git::RefUpdate; +use knot_runtime::Signer; +use knot_types::{ActorId, OwnerDid, RepoDid, TypeName}; +use serde::Serialize; + +#[test] +fn forked_acl_is_rejected_not_merged() { + let forked = forked_members( + 1, + (MembersChange::Add(grant("seed", "seed", 1)), 1), + ( + MembersChange::Remove(Removal { + subject: account("nel"), + }), + 2, + ), + (MembersChange::Add(grant("nel", "olaren", 3)), 3), + (MembersChange::Add(grant("teq", "teq", 4)), 4), + ); + assert!( + matches!(forked, Err(CobError::ForkedHistory { .. })), + "forked ACL is refused regardless of which branch a merge would favor" + ); +} + +#[test] +fn linear_member_semantics() { + let readd = build_members( + 2, + &[ + (MembersChange::Add(grant("nel", "olaren", 1)), 1), + ( + MembersChange::Remove(Removal { + subject: account("nel"), + }), + 2, + ), + (MembersChange::Add(grant("nel", "teq", 3)), 3), + ], + ); + assert!( + readd.contains(&account("nel")), + "linear re-add after a remove is a legitimate decision and takes effect" + ); + + let stale_remove = build_members( + 20, + &[ + (MembersChange::Add(grant("nel", "olaren", 5)), 5), + ( + MembersChange::Remove(Removal { + subject: account("nel"), + }), + 2, + ), + ], + ); + assert!( + !stale_remove.contains(&account("nel")), + "in a linear chain Remove is Add's child, so it applies last even with an older timestamp" + ); + + let signed_by_one = build_members(1, &[(MembersChange::Add(grant("nel", "olaren", 9)), 1)]); + assert_eq!( + signed_by_one.get(&account("nel")).unwrap().added_by, + account("olaren"), + "added_by is whatever the payload claims, unrelated to who signed" + ); + let signed_by_another = + build_members(99, &[(MembersChange::Add(grant("nel", "olaren", 9)), 1)]); + assert_eq!( + signed_by_one, signed_by_another, + "a different signing key over an identical payload yields identical state" + ); + + let once = build_members( + 10, + &[ + (MembersChange::Add(grant("nel", "nel", 1)), 1), + ( + MembersChange::Remove(Removal { + subject: account("nel"), + }), + 2, + ), + ], + ); + let twice = build_members( + 10, + &[ + (MembersChange::Add(grant("nel", "nel", 1)), 1), + ( + MembersChange::Remove(Removal { + subject: account("nel"), + }), + 2, + ), + ( + MembersChange::Remove(Removal { + subject: account("nel"), + }), + 3, + ), + ], + ); + assert_eq!(once, twice, "replaying a remove is idempotent"); + assert!(once.is_empty()); + + let created_at = build_members( + 13, + &[ + (MembersChange::Add(grant("nel", "olaren", 100)), 1), + ( + MembersChange::Remove(Removal { + subject: account("nel"), + }), + 2, + ), + (MembersChange::Add(grant("nel", "teq", 50)), 3), + ], + ); + let entry = created_at.get(&account("nel")).unwrap(); + assert_eq!(entry.added_by, account("teq"), "last linear Add wins"); + assert_eq!( + entry.created_at, + at(50), + "the later Add's created_at takes effect even though it is older than an earlier entry's" + ); +} + +#[test] +fn verify_rejects_a_change_with_a_forged_signature() { + let (_dir, repo) = fixture(); + let nsid = MembersChange::type_name(); + let owner = owner_of(32); + let payload = MembersChange::Add(grant("nel", "nel", 1)).encode().unwrap(); + let root = write_cob_commit(&repo, &nsid, &payload, &[], &owner, 1); + let object = CobId::new(root); + repo.update_ref(&RefUpdate::Create { + name: cob_ref(&nsid, object), + new: root, + }) + .unwrap(); + + let store = CobStore::new(&repo); + assert!( + store.get::(object).is_ok(), + "read path materializes without checking signatures, by design" + ); + assert!( + matches!( + store.verify::(&home(), object, &owner), + Err(CobError::UnverifiedChange { .. }) + ), + "import verification catches forged signature the read path trusts" + ); +} + +#[derive(Serialize)] +struct WireRegister<'a> { + op: &'a str, + data: WireRegistration<'a>, +} + +#[derive(Serialize)] +struct WireRegistration<'a> { + owner: &'a str, + rkey: &'a str, + name: &'a str, + repo: &'a str, + created_at: i64, +} + +fn encode_register(rkey: &str, name: &str) -> Vec { + serde_ipld_dagcbor::to_vec(&WireRegister { + op: "register", + data: WireRegistration { + owner: "did:plc:nel", + rkey, + name, + repo: "did:plc:squid", + created_at: 1, + }, + }) + .unwrap() +} + +#[test] +fn malformed_repo_name_or_rkey_is_rejected_at_decode() { + assert!( + RegistryChange::decode(&encode_register("anemone", "anemone")).is_ok(), + "control: well-formed wire payload decodes" + ); + assert!( + RegistryChange::decode(&encode_register("anemone", "../../etc/passwd")).is_err(), + "traversal repo name fails newtype validation during decode, never reaching a ref" + ); + assert!( + RegistryChange::decode(&encode_register("anemone", "refs/heads/main")).is_err(), + "name with path separators is rejected at decode" + ); + assert!( + RegistryChange::decode(&encode_register("not a record key", "anemone")).is_err(), + "rkey outside record-key grammar is rejected at decode" + ); + assert!( + RegistryChange::decode(&encode_register("..", "anemone")).is_err(), + "reserved '..' rkey is rejected at decode" + ); +} + +#[test] +fn registry_handler_guards() { + let (_dir, repo) = fixture(); + let key = signer(70); + let store = CobStore::new(&repo); + let object = registry_with(&repo, &key, "anemone", "squid"); + let nel = || OwnerDid::new("did:plc:nel").unwrap(); + let squid = || RepoDid::new("did:plc:squid").unwrap(); + + let already = register_repo( + &store, + &home(), + object, + registration("olaren", "fork", "squid", 2), + &key, + at(2), + ); + assert!( + matches!(already, Err(RegistryError::AlreadyRegistered { .. })), + "a repo DID already registered elsewhere cannot be claimed again" + ); + + let unregistered = deregister_repo( + &store, + &home(), + object, + RepoRef { + owner: nel(), + rkey: rkey("barnacle"), + }, + squid(), + &key, + at(3), + ); + assert!(matches!( + unregistered, + Err(RegistryError::NotRegistered { .. }) + )); + + let mismatch = deregister_repo( + &store, + &home(), + object, + RepoRef { + owner: nel(), + rkey: rkey("anemone"), + }, + RepoDid::new("did:plc:whelk").unwrap(), + &key, + at(4), + ); + assert!( + matches!(mismatch, Err(RegistryError::RepoMismatch { .. })), + "deregister whose expected repo doesn't match the keyed one is refused" + ); + assert_eq!( + store + .get::(object) + .unwrap() + .into_state() + .resolve(&nel(), &rkey("anemone")), + Some(&squid()), + "a refused deregister left the registration intact" + ); + + register_repo( + &store, + &home(), + object, + registration("nel", "barnacle", "whelk", 5), + &key, + at(5), + ) + .unwrap(); + assert_eq!( + store + .get::(object) + .unwrap() + .into_state() + .owner_of(&RepoDid::new("did:plc:whelk").unwrap()), + Some(nel()), + "a fresh repo DID lands" + ); + + deregister_repo( + &store, + &home(), + object, + RepoRef { + owner: nel(), + rkey: rkey("anemone"), + }, + squid(), + &key, + at(6), + ) + .unwrap(); + let after_deregister = store.get::(object).unwrap().into_state(); + assert!( + after_deregister.resolve(&nel(), &rkey("anemone")).is_none(), + "a matching deregister removes the keyed repo" + ); + assert_eq!( + after_deregister.resolve(&nel(), &rkey("barnacle")), + Some(&RepoDid::new("did:plc:whelk").unwrap()), + "deregistering one repo leaves its sibling resolving" + ); +} + +#[test] +fn add_member_handler_lands_a_grant() { + let (_dir, repo) = fixture(); + let key = signer(81); + let store = CobStore::new(&repo); + let created = store + .create( + &home(), + &MembersChange::Add(grant("nel", "nel", 1)), + &key, + at(1), + ) + .unwrap(); + + add_member( + &store, + &home(), + created.object, + grant("olaren", "nel", 2), + &key, + at(2), + ) + .unwrap(); + + let members = store + .get::(created.object) + .unwrap() + .into_state(); + assert!(members.contains(&account("olaren"))); +} + +#[test] +fn verify_cob_ref_boundary_cases() { + let (_dir, repo, key, object) = members_store( + 90, + &[ + (MembersChange::Add(grant("nel", "nel", 1)), 1), + (MembersChange::Add(grant("olaren", "nel", 2)), 2), + ], + ); + let store = CobStore::new(&repo); + let owner = ActorId::from_secp256k1(key.public_key().as_bytes()); + let refname = cob_ref(&MembersChange::type_name(), object); + + assert!( + store.verify::(&home(), object, &owner).is_ok(), + "every change is validly signed by the owning key" + ); + assert!( + matches!( + store.verify::(&home(), object, &owner_of(31)), + Err(CobError::UnverifiedChange { .. }) + ), + "a change not authored by the claimed owner is refused at import" + ); + + assert_eq!( + verify_cob_ref(&store, &home(), &refname, &owner).unwrap(), + object, + "a genuine object verifies through the namespace dispatcher" + ); + assert!(matches!( + verify_cob_ref(&store, &home(), &refname, &owner_of(91)), + Err(ImportError::Cob(CobError::UnverifiedChange { .. })) + )); + + let elsewhere = CobHome::from(&RepoDid::new("did:plc:limpet").unwrap()); + assert!( + matches!( + verify_cob_ref(&store, &elsewhere, &refname, &owner), + Err(ImportError::Cob(CobError::UnverifiedChange { .. })) + ), + "an object pushed under a different repo home is refused at import" + ); + + assert!(matches!( + verify_cob_ref( + &store, + &home(), + &knot_types::RefName::new("refs/heads/main").unwrap(), + &owner, + ), + Err(ImportError::NotCobRef(_)) + )); + + let stray = cob_ref(&TypeName::new("sh.tangled.test.unknown").unwrap(), object); + assert!(matches!( + verify_cob_ref(&store, &home(), &stray, &owner), + Err(ImportError::UnknownType(_)) + )); + + let collaborators = store + .create( + &home(), + &CollaboratorsChange::Add(grant("lyna", "nel", 1)), + &key, + at(1), + ) + .unwrap() + .object; + let collab_ref = cob_ref(&CollaboratorsChange::type_name(), collaborators); + assert_eq!( + verify_cob_ref(&store, &home(), &collab_ref, &owner).unwrap(), + collaborators, + "the dispatcher routes a second namespace to its own resolver, not a hardcoded type" + ); + + let (_forked_dir, forked_repo, forked) = forked_members_object( + 95, + (MembersChange::Add(grant("seed", "seed", 1)), 1), + (MembersChange::Add(grant("nel", "olaren", 2)), 2), + ( + MembersChange::Remove(Removal { + subject: account("nel"), + }), + 3, + ), + (MembersChange::Add(grant("teq", "teq", 4)), 4), + ); + let forked_store = CobStore::new(&forked_repo); + let forked_ref = cob_ref(&MembersChange::type_name(), forked); + assert!( + matches!( + verify_cob_ref(&forked_store, &home(), &forked_ref, &owner_of(95)), + Err(ImportError::Cob(CobError::ForkedHistory { .. })) + ), + "forked linear history is refused at import alongside the signature check" + ); +} diff --git a/knot2/crates/knot-config/src/lib.rs b/knot2/crates/knot-config/src/lib.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-config/src/lib.rs @@ -0,0 +1,1658 @@ +use std::fmt; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::sync::OnceLock; +use std::time::Duration; + +use base64::Engine; +use confique::Config; +use knot_runtime::HttpLimits; +use knot_types::{AccountDid, AdmissionPolicy, AppviewEndpoint}; +use url::Url; + +#[derive(Debug, Config)] +pub struct KnotConfig { + #[config(nested)] + pub server: ServerConfig, + #[config(nested)] + pub tls: TlsConfig, + #[config(nested)] + pub acl: AclConfig, + #[config(nested)] + pub repo: RepoConfig, + #[config(nested)] + pub git: GitConfig, + #[config(nested)] + pub secrets: SecretsConfig, + #[config(nested)] + pub http: HttpConfig, + #[config(nested)] + pub atproto: AtprotoConfig, + #[config(nested)] + pub xrpc: XrpcConfig, + #[config(nested)] + pub maintenance: MaintenanceConfig, + #[config(nested)] + pub pack_cache: PackCacheConfig, + #[config(nested)] + pub pack: PackConfig, + #[config(nested)] + pub lfs: LfsConfig, + #[config(nested)] + pub resources: ResourcesConfig, + #[config(nested)] + pub homepage: HomepageConfig, + + #[config(nested)] + pub ci: CiConfig, + #[config(nested)] + pub messages: knot_messages::MessagesConfig, +} + +#[derive(Debug, Config)] +pub struct AclConfig { + #[config(env = "KNOT_ADMISSION", default = "closed")] + pub admission: AdmissionPolicy, +} + +#[derive(Debug, Config)] +pub struct TlsConfig { + #[config(env = "KNOT_TLS_CERT_PATH")] + pub cert_path: Option, + + #[config(env = "KNOT_TLS_KEY_PATH")] + pub key_path: Option, + + #[config(env = "KNOT_TLS_HTTP3", default = true)] + pub http3: bool, + + #[config(env = "KNOT_TLS_ACME_ENABLED", default = false)] + pub acme_enabled: bool, + + #[config(env = "KNOT_TLS_ACME_CACHE_DIR")] + pub acme_cache_dir: Option, + + #[config(env = "KNOT_TLS_ACME_CONTACT")] + pub acme_contact: Option, + + #[config(env = "KNOT_TLS_ACME_STAGING", default = false)] + pub acme_staging: bool, + + #[config(env = "KNOT_TLS_MTLS_ENABLED", default = false)] + pub mtls_enabled: bool, + + #[config(env = "KNOT_TLS_MTLS_CLIENT_CA_PATH")] + pub mtls_client_ca_path: Option, + + #[config(env = "KNOT_TLS_MTLS_ADMIN_SPKI_PIN")] + pub mtls_admin_spki_pin: Option, +} + +#[derive(Debug, Config)] +pub struct ServerConfig { + #[config(env = "KNOT_HOSTNAME")] + pub hostname: String, + + #[config(env = "KNOT_ADMINS", parse_env = parse_admins)] + pub admins: Vec, + + #[config(env = "KNOT_LISTEN_ADDR", default = "[::]:5555")] + pub listen_addr: SocketAddr, + + #[config(env = "KNOT_LISTEN_HEADER_TIMEOUT_MS", default = 10_000)] + pub listen_header_timeout_ms: u64, + + #[config(env = "KNOT_LISTEN_IDLE_TIMEOUT_MS", default = 60_000)] + pub listen_idle_timeout_ms: u64, + + #[config(env = "KNOT_LISTEN_MAX_CONNECTIONS", default = 1_024)] + pub listen_max_connections: u32, + + #[config(env = "KNOT_LISTEN_RATE_LIMIT_PER_SECOND", default = 50)] + pub listen_rate_limit_per_second: u32, + + #[config(env = "KNOT_LISTEN_RATE_LIMIT_BURST", default = 200)] + pub listen_rate_limit_burst: u32, + + #[config(env = "KNOT_LISTEN_MAX_INFLIGHT_REQUESTS", default = 1_024)] + pub listen_max_inflight_requests: u32, + + #[config(env = "KNOT_LISTEN_REQUEST_TIMEOUT_MS", default = 60_000)] + pub listen_request_timeout_ms: u64, + + #[config(env = "KNOT_LISTEN_BODY_TIMEOUT_MS", default = 30_000)] + pub listen_body_timeout_ms: u64, + + #[config(env = "KNOT_LISTEN_WRITE_REQUEST_TIMEOUT_MS", default = 1_800_000)] + pub listen_write_request_timeout_ms: u64, + + #[config(env = "KNOT_INTERNAL_LISTEN_ADDR", default = "[::1]:5444")] + pub internal_listen_addr: SocketAddr, + + #[config(env = "KNOT_SSH_LISTEN_ADDR", default = "[::]:2222")] + pub ssh_listen_addr: SocketAddr, + + #[config(env = "KNOT_SSH_HOST_KEY_FILE")] + pub ssh_host_key_file: PathBuf, + + #[config(env = "KNOT_SSH_MAX_PACK_BYTES", default = 8_589_934_592u64)] + pub ssh_max_pack_bytes: u64, + + #[config(env = "KNOT_APPVIEW_ENDPOINT", default = "https://tangled.org")] + pub appview_endpoint: AppviewEndpoint, +} + +#[derive(Debug, Config)] +pub struct RepoConfig { + #[config(env = "KNOT_SCAN_PATH")] + pub scan_path: PathBuf, + + #[config(env = "KNOT_DEFAULT_BRANCH", default = "main")] + pub default_branch: String, +} + +#[derive(Debug, Config)] +pub struct CiConfig { + #[config(env = "KNOT_CI_LOGS_ADDR")] + pub logs_addr: Option, +} + +#[derive(Debug, Config)] +pub struct HomepageConfig { + #[config(env = "KNOT_HOMEPAGE_ENABLED", default = true)] + pub enabled: bool, + + #[config(env = "KNOT_HOMEPAGE_PATH")] + pub path: Option, +} + +#[derive(Debug)] +pub enum HomepageSource { + Disabled, + Default, + File(PathBuf), +} + +impl HomepageConfig { + pub fn source(&self) -> HomepageSource { + match (self.enabled, self.path.as_ref()) { + (false, _) => HomepageSource::Disabled, + (true, None) => HomepageSource::Default, + (true, Some(path)) => HomepageSource::File(path.clone()), + } + } +} + +#[derive(Debug, Config)] +pub struct GitConfig { + /// Committer identity stamped on merge commits the knot creates. + #[config(env = "KNOT_GIT_USER_NAME", default = "Tangled")] + pub user_name: String, + + #[config(env = "KNOT_GIT_USER_EMAIL", default = "noreply@tangled.sh")] + pub user_email: String, + + #[config(env = "KNOT_GIT_OBJECT_FORMAT", default = "sha256")] + pub object_format: String, +} + +#[derive(Debug, Config)] +pub struct SecretsConfig { + #[config(env = "KNOT_SEALED_KEY_FILE")] + pub sealed_key_file: PathBuf, + + #[config(env = "KNOT_MASTER_KEY_ENV")] + pub master_key_env: String, +} + +#[derive(Debug, Config)] +pub struct HttpConfig { + #[config(env = "KNOT_HTTP_CONNECT_TIMEOUT_MS", default = 5_000)] + pub connect_timeout_ms: u64, + + #[config(env = "KNOT_HTTP_READ_TIMEOUT_MS", default = 30_000)] + pub read_timeout_ms: u64, + + #[config(env = "KNOT_HTTP_REQUEST_TIMEOUT_MS", default = 60_000)] + pub request_timeout_ms: u64, + + #[config(env = "KNOT_HTTP_MAX_RESPONSE_BYTES", default = 16_777_216)] + pub max_response_bytes: u64, +} + +#[derive(Debug, Config)] +pub struct AtprotoConfig { + #[config(env = "KNOT_PLC_DIRECTORY")] + pub plc_directory: Url, +} + +#[derive(Debug, Config)] +pub struct XrpcConfig { + #[config(env = "KNOT_XRPC_MAX_BODY_BYTES", default = 65_536)] + pub max_body_bytes: u64, + + #[config(env = "KNOT_XRPC_MAX_RESPONSE_BYTES", default = 5_242_880)] + pub max_response_bytes: u64, + + #[config(env = "KNOT_XRPC_MAX_ARCHIVE_BYTES", default = 1_073_741_824)] + pub max_archive_bytes: u64, + + #[config(env = "KNOT_XRPC_TREE_LAST_COMMIT_BUDGET_MS", default = 300)] + pub tree_last_commit_budget_ms: u64, + + #[config(env = "KNOT_XRPC_BLOB_LAST_COMMIT_BUDGET_MS", default = 2_000)] + pub blob_last_commit_budget_ms: u64, + + #[config(env = "KNOT_XRPC_LANGUAGES_BUDGET_MS", default = 1_000)] + pub languages_budget_ms: u64, + + #[config(env = "KNOT_XRPC_LANGUAGES_PUSH_BUDGET_MS", default = 2_000)] + pub languages_push_budget_ms: u64, + + /// Body limit for the merge and mergeCheck procedures, whose patch payloads + /// routinely exceed the general XRPC body limit. + #[config(env = "KNOT_XRPC_MAX_PATCH_BYTES", default = 16_777_216)] + pub max_patch_bytes: u64, + + /// Limit on the total decompressed size of a patch the merge procedures parse, + /// bounding binary-delta inflation and hunk expansion apart from the + /// compressed body limit above. + #[config(env = "KNOT_XRPC_MAX_PATCH_DECOMPRESSED_BYTES", default = 134_217_728)] + pub max_patch_decompressed_bytes: u64, + + #[config(env = "KNOT_XRPC_PREAUTH_BURST", default = 20)] + pub preauth_burst: u32, + + #[config(env = "KNOT_XRPC_PREAUTH_REFILL_MS", default = 100)] + pub preauth_refill_ms: u64, + + #[config(env = "KNOT_XRPC_PER_PEER_INFLIGHT", default = 8)] + pub per_peer_inflight: u32, + + #[config(env = "KNOT_XRPC_GLOBAL_INFLIGHT", default = 64)] + pub global_inflight: u32, + + #[config(env = "KNOT_XRPC_MAX_PENDING_RESERVATIONS", default = 256)] + pub max_pending_reservations: u32, + + /// Per-account limit on reserved repository keys awaiting creation, so one + /// account cannot consume the whole pending-reservation budget. + #[config(env = "KNOT_XRPC_PER_ACTOR_RESERVATIONS", default = 32)] + pub per_actor_reservations: u32, + + /// How long a reserved repository key is held before it lapses and its + /// sealed key is reclaimed, in seconds. + #[config(env = "KNOT_XRPC_RESERVATION_TTL_SECS", default = 3600)] + pub reservation_ttl_secs: u64, + + #[config(env = "KNOT_XRPC_FORK_MAX_PACK_BYTES", default = 1_073_741_824)] + pub fork_max_pack_bytes: u64, + + #[config(env = "KNOT_XRPC_FORK_FETCH_TIMEOUT_MS", default = 600_000)] + pub fork_fetch_timeout_ms: u64, + + /// When the knot runs behind a trusted reverse proxy that terminates TLS, + /// set this to the header the proxy appends the client address to, for + /// example x-forwarded-for. The rightmost entry is used. Leave unset when + /// the knot is directly exposed so the socket peer address is used. Only set + /// this when a trusted proxy overwrites or appends the header, since a client + /// can forge it otherwise. + #[config(env = "KNOT_XRPC_TRUSTED_PROXY_HEADER")] + pub trusted_proxy_header: Option, + + #[config(env = "KNOT_XRPC_EVENTS_REPLAY_BUFFER", default = 4096)] + pub events_replay_buffer: u32, + + #[config(env = "KNOT_XRPC_EVENTS_REPLAY_BYTES", default = 67_108_864)] + pub events_replay_bytes: u64, + + #[config(env = "KNOT_XRPC_EVENTS_MAX_SUBSCRIBERS", default = 256)] + pub events_max_subscribers: u32, + + #[config(env = "KNOT_XRPC_EVENTS_MAX_PER_PEER", default = 8)] + pub events_max_per_peer: u32, +} + +#[derive(Debug, Config)] +pub struct MaintenanceConfig { + #[config(env = "KNOT_MAINTENANCE_ENABLED", default = true)] + pub enabled: bool, + + #[config(env = "KNOT_MAINTENANCE_COMMIT_GRAPH", default = true)] + pub commit_graph: bool, + + #[config(env = "KNOT_MAINTENANCE_MULTI_PACK_INDEX", default = true)] + pub multi_pack_index: bool, + + #[config(env = "KNOT_MAINTENANCE_BITMAP", default = true)] + pub bitmap: bool, + + #[config(env = "KNOT_MAINTENANCE_INTERVAL_SECS", default = 21_600)] + pub interval_secs: u64, + + #[config(env = "KNOT_MAINTENANCE_REPACK_MAX_OBJECTS", default = 16_000_000)] + pub repack_max_objects: u64, + + #[config(env = "KNOT_MAINTENANCE_REPACK_GEOMETRIC_FACTOR", default = 2)] + pub repack_geometric_factor: u64, + + #[config(env = "KNOT_MAINTENANCE_PRUNE_GRACE_SECS", default = 1_209_600)] + pub prune_grace_secs: u64, + + #[config(env = "KNOT_MAINTENANCE_REFLOG_EXPIRE_SECS", default = 7_776_000)] + pub reflog_expire_secs: u64, + + #[config(env = "KNOT_MAINTENANCE_LARGE_PUSH_BYTES", default = 52_428_800)] + pub large_push_bytes: u64, +} + +#[derive(Debug, Config)] +pub struct PackCacheConfig { + #[config(env = "KNOT_PACK_CACHE_ENABLED", default = true)] + pub enabled: bool, + + #[config(env = "KNOT_PACK_CACHE_TTL_SECS", default = 60)] + pub ttl_secs: u64, + + #[config(env = "KNOT_PACK_CACHE_MAX_ENTRY_BYTES", default = 67_108_864)] + pub max_entry_bytes: u64, + + #[config(env = "KNOT_PACK_CACHE_MAX_TOTAL_BYTES", default = 2_147_483_648u64)] + pub max_total_bytes: u64, +} + +#[derive(Debug, Config)] +pub struct PackConfig { + #[config(env = "KNOT_PACK_MAX_OBJECTS", default = 16_000_000)] + pub max_objects: u32, + + #[config(env = "KNOT_PACK_MAX_TOTAL_BYTES", default = 68_719_476_736u64)] + pub max_total_bytes: u64, + + #[config(env = "KNOT_PACK_SELECTION_MAX_OBJECTS", default = 16_000_000)] + pub selection_max_objects: u32, + + #[config(env = "KNOT_PACK_SELECTION_TIME_BUDGET_SECS", default = 600)] + pub selection_time_budget_secs: u64, +} + +#[derive(Debug, Config)] +pub struct LfsConfig { + #[config(env = "KNOT_LFS_STORE_PATH")] + pub store_path: Option, + + #[config(env = "KNOT_LFS_MAX_OBJECT_BYTES", default = 5_368_709_120u64)] + pub max_object_bytes: u64, + + #[config(env = "KNOT_LFS_FREE_SPACE_FLOOR_BYTES", default = 1_073_741_824u64)] + pub free_space_floor_bytes: u64, + + #[config(env = "KNOT_LFS_GC_GRACE_SECS", default = 1_209_600)] + pub gc_grace_secs: u64, + + #[config(env = "KNOT_LFS_GC_INTERVAL_SECS", default = 21_600)] + pub gc_interval_secs: u64, + + #[config(env = "KNOT_LFS_MAX_SSH_TRANSFERS", default = 16)] + pub max_ssh_transfers: u32, + + #[config(env = "KNOT_LFS_MAX_HTTP_DOWNLOADS", default = 64)] + pub max_http_downloads: u32, +} + +#[derive(Debug, Config)] +pub struct ResourcesConfig { + #[config(env = "KNOT_MAX_THREADS", default = 0)] + pub max_threads: u32, + + #[config(env = "KNOT_MAX_MEMORY_BYTES", default = 0)] + pub max_memory_bytes: u64, +} + +fn parse_admins(raw: &str) -> Result, knot_types::ParseError> { + raw.split(',') + .map(str::trim) + .filter(|item| !item.is_empty()) + .map(AccountDid::new) + .collect() +} + +impl KnotConfig { + pub fn object_format(&self) -> Option { + knot_types::ObjectFormat::from_capability(&self.git.object_format) + } + + pub fn tls_enabled(&self) -> bool { + self.static_cert_enabled() || self.tls.acme_enabled + } + + pub fn static_cert_enabled(&self) -> bool { + self.tls.cert_path.is_some() && self.tls.key_path.is_some() + } + + pub fn http_limits(&self) -> HttpLimits { + HttpLimits { + connect_timeout: Duration::from_millis(self.http.connect_timeout_ms), + read_timeout: Duration::from_millis(self.http.read_timeout_ms), + request_timeout: Duration::from_millis(self.http.request_timeout_ms), + max_response_bytes: self.http.max_response_bytes, + block_private_addresses: true, + } + } + + pub fn fork_http_limits(&self) -> HttpLimits { + HttpLimits { + connect_timeout: Duration::from_millis(self.http.connect_timeout_ms), + read_timeout: Duration::from_millis(self.http.read_timeout_ms), + request_timeout: Duration::from_millis(self.xrpc.fork_fetch_timeout_ms), + max_response_bytes: self + .xrpc + .fork_max_pack_bytes + .saturating_add(self.xrpc.fork_max_pack_bytes / 64) + .saturating_add(1_048_576), + block_private_addresses: true, + } + } + + pub fn validate(&self) -> Result<(), ConfigError> { + let errors: Vec = [ + check( + !self.server.hostname.is_empty(), + "server.hostname mustn't be empty", + ), + knot_messages::Catalog::parse(&self.messages) + .err() + .map(|error| error.to_string()), + knot_types::KnotHostname::new(self.server.hostname.clone()) + .err() + .map(|_| "server.hostname isn't a valid bare hostname".to_string()), + check( + !self.server.admins.is_empty(), + "server.admins must list at least one DID", + ), + check( + self.repo.scan_path.is_absolute(), + "repo.scan_path must be absolute path", + ), + check( + self.secrets.sealed_key_file.is_absolute(), + "secrets.sealed_key_file must be absolute path", + ), + check( + self.server.ssh_host_key_file.is_absolute(), + "server.ssh_host_key_file must be absolute path", + ), + check( + self.tls.cert_path.is_some() == self.tls.key_path.is_some(), + "tls.cert_path and tls.key_path must both be set or both unset", + ), + self.tls + .cert_path + .as_ref() + .filter(|path| !path.is_absolute()) + .map(|_| "tls.cert_path must be absolute path".to_string()), + self.tls + .key_path + .as_ref() + .filter(|path| !path.is_absolute()) + .map(|_| "tls.key_path must be absolute path".to_string()), + check( + !(self.tls.acme_enabled && self.static_cert_enabled()), + "tls.acme_enabled cannot combine with a static tls.cert_path and tls.key_path", + ), + check( + !self.tls.acme_enabled || self.tls.acme_cache_dir.is_some(), + "tls.acme_cache_dir is required when tls.acme_enabled is set", + ), + self.tls + .acme_cache_dir + .as_ref() + .filter(|path| !path.is_absolute()) + .map(|_| "tls.acme_cache_dir must be absolute path".to_string()), + check( + !self.tls.acme_enabled + || self + .tls + .acme_contact + .as_deref() + .is_some_and(is_contact_email), + "tls.acme_contact must be a contact email when tls.acme_enabled is set", + ), + check( + !self.tls.mtls_enabled || self.tls_enabled(), + "tls.mtls_enabled requires a server certificate via static paths or ACME", + ), + check( + !self.tls.mtls_enabled || self.tls.mtls_client_ca_path.is_some(), + "tls.mtls_client_ca_path is required when tls.mtls_enabled is set", + ), + self.tls + .mtls_client_ca_path + .as_ref() + .filter(|path| !path.is_absolute()) + .map(|_| "tls.mtls_client_ca_path must be absolute path".to_string()), + check( + !self.tls.mtls_enabled + || self + .tls + .mtls_admin_spki_pin + .as_deref() + .is_some_and(is_spki_pin), + "tls.mtls_admin_spki_pin must be a base64 SHA-256 pin when tls.mtls_enabled is set", + ), + check( + is_env_var_name(&self.secrets.master_key_env), + "secrets.master_key_env must be valid environment variable name", + ), + check( + self.server.ssh_max_pack_bytes > 0, + "server.ssh_max_pack_bytes must be greater than zero", + ), + check( + self.server.listen_header_timeout_ms > 0, + "server.listen_header_timeout_ms must be greater than zero", + ), + check( + self.server.listen_idle_timeout_ms > 0, + "server.listen_idle_timeout_ms must be greater than zero", + ), + check( + self.server.listen_idle_timeout_ms >= self.server.listen_header_timeout_ms, + "server.listen_idle_timeout_ms must be at least server.listen_header_timeout_ms", + ), + check( + self.server.listen_max_connections > 0, + "server.listen_max_connections must be greater than zero", + ), + check( + self.server.listen_rate_limit_per_second > 0, + "server.listen_rate_limit_per_second must be greater than zero", + ), + check( + self.server.listen_rate_limit_burst > 0, + "server.listen_rate_limit_burst must be greater than zero", + ), + check( + self.server.listen_max_inflight_requests > 0, + "server.listen_max_inflight_requests must be greater than zero", + ), + check( + self.server.listen_request_timeout_ms > 0, + "server.listen_request_timeout_ms must be greater than zero", + ), + check( + self.server.listen_body_timeout_ms > 0, + "server.listen_body_timeout_ms must be greater than zero", + ), + check( + self.server.listen_write_request_timeout_ms > 0, + "server.listen_write_request_timeout_ms must be greater than zero", + ), + knot_types::RefName::new(format!("refs/heads/{}", self.repo.default_branch)) + .err() + .map(|_| "repo.default_branch isn't valid branch name".to_string()), + check( + self.http.connect_timeout_ms > 0, + "http.connect_timeout_ms must be greater than zero", + ), + check( + self.http.read_timeout_ms > 0, + "http.read_timeout_ms must be greater than zero", + ), + check( + self.http.request_timeout_ms > 0, + "http.request_timeout_ms must be greater than zero", + ), + check( + self.http.max_response_bytes > 0, + "http.max_response_bytes must be greater than zero", + ), + check( + self.atproto.plc_directory.scheme() == "https", + "atproto.plc_directory must be https URL", + ), + check( + self.atproto.plc_directory.host().is_some(), + "atproto.plc_directory must have host", + ), + check( + self.xrpc.max_body_bytes > 0, + "xrpc.max_body_bytes must be greater than zero", + ), + check( + self.xrpc.max_response_bytes > 0, + "xrpc.max_response_bytes must be greater than zero", + ), + check( + self.xrpc.max_archive_bytes > 0, + "xrpc.max_archive_bytes must be greater than zero", + ), + check( + self.xrpc.tree_last_commit_budget_ms > 0, + "xrpc.tree_last_commit_budget_ms must be greater than zero", + ), + check( + self.xrpc.blob_last_commit_budget_ms > 0, + "xrpc.blob_last_commit_budget_ms must be greater than zero", + ), + check( + self.xrpc.languages_budget_ms > 0, + "xrpc.languages_budget_ms must be greater than zero", + ), + check( + self.xrpc.languages_push_budget_ms > 0, + "xrpc.languages_push_budget_ms must be greater than zero", + ), + check( + self.xrpc.max_patch_bytes > 0, + "xrpc.max_patch_bytes must be greater than zero", + ), + check( + self.xrpc.max_patch_decompressed_bytes > 0, + "xrpc.max_patch_decompressed_bytes must be greater than zero", + ), + check( + self.xrpc.fork_max_pack_bytes > 0, + "xrpc.fork_max_pack_bytes must be greater than zero", + ), + check( + self.xrpc.fork_fetch_timeout_ms > 0, + "xrpc.fork_fetch_timeout_ms must be greater than zero", + ), + check( + !self.git.user_name.trim().is_empty(), + "git.user_name mustn't be empty", + ), + check( + !self.git.user_email.trim().is_empty(), + "git.user_email mustn't be empty", + ), + check( + self.object_format().is_some(), + "git.object_format must be \"sha1\" or \"sha256\"", + ), + check( + self.xrpc.preauth_burst > 0, + "xrpc.preauth_burst must be greater than zero", + ), + check( + self.xrpc.preauth_refill_ms > 0, + "xrpc.preauth_refill_ms must be greater than zero", + ), + check( + self.xrpc.per_peer_inflight > 0, + "xrpc.per_peer_inflight must be greater than zero", + ), + check( + self.xrpc.global_inflight >= self.xrpc.per_peer_inflight, + "xrpc.global_inflight must be at least xrpc.per_peer_inflight", + ), + check( + self.xrpc.per_actor_reservations > 0, + "xrpc.per_actor_reservations must be greater than zero", + ), + check( + self.xrpc.max_pending_reservations >= self.xrpc.per_actor_reservations, + "xrpc.max_pending_reservations must be at least xrpc.per_actor_reservations", + ), + check( + self.xrpc.reservation_ttl_secs > 0, + "xrpc.reservation_ttl_secs must be greater than zero", + ), + check( + self.xrpc.events_replay_buffer > 0, + "xrpc.events_replay_buffer must be greater than zero", + ), + check( + self.xrpc.events_replay_bytes > 0, + "xrpc.events_replay_bytes must be greater than zero", + ), + check( + self.xrpc.events_max_subscribers > 0, + "xrpc.events_max_subscribers must be greater than zero", + ), + check( + self.xrpc.events_max_per_peer > 0, + "xrpc.events_max_per_peer must be greater than zero", + ), + check( + self.xrpc.events_max_subscribers >= self.xrpc.events_max_per_peer, + "xrpc.events_max_subscribers must be at least xrpc.events_max_per_peer", + ), + check( + self.maintenance.interval_secs > 0, + "maintenance.interval_secs must be greater than zero", + ), + check( + self.maintenance.repack_max_objects > 0, + "maintenance.repack_max_objects must be greater than zero", + ), + check( + self.maintenance.repack_geometric_factor >= 2, + "maintenance.repack_geometric_factor must be at least 2", + ), + check( + self.maintenance.large_push_bytes > 0, + "maintenance.large_push_bytes must be greater than zero", + ), + check( + self.pack_cache.ttl_secs > 0, + "pack_cache.ttl_secs must be greater than zero", + ), + check( + self.pack_cache.max_entry_bytes > 0, + "pack_cache.max_entry_bytes must be greater than zero", + ), + check( + self.pack_cache.max_total_bytes > 0, + "pack_cache.max_total_bytes must be greater than zero", + ), + check( + self.pack_cache.max_total_bytes >= self.pack_cache.max_entry_bytes, + "pack_cache.max_total_bytes must be at least pack_cache.max_entry_bytes", + ), + check( + self.pack.max_objects > 0, + "pack.max_objects must be greater than zero", + ), + check( + self.pack.max_total_bytes > 0, + "pack.max_total_bytes must be greater than zero", + ), + check( + self.pack.selection_max_objects > 0, + "pack.selection_max_objects must be greater than zero", + ), + check( + self.pack.selection_time_budget_secs > 0, + "pack.selection_time_budget_secs must be greater than zero", + ), + self.lfs + .store_path + .as_ref() + .filter(|path| !path.is_absolute()) + .map(|_| "lfs.store_path must be absolute path".to_string()), + self.lfs + .store_path + .as_ref() + .filter(|path| { + path.starts_with(&self.repo.scan_path) || self.repo.scan_path.starts_with(path) + }) + .map(|_| "lfs.store_path mustn't overlap repo.scan_path".to_string()), + check( + self.lfs.max_object_bytes > 0, + "lfs.max_object_bytes must be greater than zero", + ), + check( + self.lfs.gc_interval_secs > 0, + "lfs.gc_interval_secs must be greater than zero", + ), + check( + self.lfs.max_ssh_transfers > 0, + "lfs.max_ssh_transfers must be greater than zero", + ), + check( + self.lfs.max_http_downloads > 0, + "lfs.max_http_downloads must be greater than zero", + ), + self.xrpc + .trusted_proxy_header + .as_ref() + .filter(|header| !is_http_token(header)) + .map(|_| "xrpc.trusted_proxy_header isn't valid HTTP header name".to_string()), + match self.homepage.source() { + HomepageSource::File(path) if !path.is_absolute() => { + Some("homepage.path must be absolute path".to_string()) + } + _ => None, + }, + ] + .into_iter() + .flatten() + .chain(self.port_collisions()) + .collect(); + + if errors.is_empty() { + Ok(()) + } else { + Err(ConfigError { errors }) + } + } + + fn port_collisions(&self) -> Vec { + let binds = [ + ("server.listen_addr", self.server.listen_addr), + ( + "server.internal_listen_addr", + self.server.internal_listen_addr, + ), + ("server.ssh_listen_addr", self.server.ssh_listen_addr), + ]; + [(0, 1), (0, 2), (1, 2)] + .into_iter() + .filter(|&(a, b)| binds[a].1.port() == binds[b].1.port()) + .map(|(a, b)| { + format!( + "{} and {} cannot bind same port {}", + binds[a].0, + binds[b].0, + binds[a].1.port() + ) + }) + .collect() + } +} + +fn check(ok: bool, message: &str) -> Option { + (!ok).then(|| message.to_string()) +} + +fn is_http_token(value: &str) -> bool { + !value.is_empty() + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"!#$%&'*+-.^_`|~".contains(&byte)) +} + +fn is_contact_email(value: &str) -> bool { + let mut parts = value.splitn(2, '@'); + matches!( + (parts.next(), parts.next()), + (Some(local), Some(domain)) + if !local.is_empty() + && domain.contains('.') + && !domain.starts_with('.') + && !domain.ends_with('.') + && !value.chars().any(char::is_whitespace) + ) +} + +fn is_spki_pin(value: &str) -> bool { + base64::engine::general_purpose::STANDARD + .decode(value) + .is_ok_and(|bytes| bytes.len() == 32) +} + +fn is_env_var_name(value: &str) -> bool { + !value.is_empty() + && !value.starts_with(|c: char| c.is_ascii_digit()) + && value + .chars() + .all(|c| c.is_ascii_uppercase() || c.is_ascii_digit() || c == '_') +} + +#[derive(Debug, thiserror::Error)] +pub struct ConfigError { + pub errors: Vec, +} + +impl fmt::Display for ConfigError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f, "{} configuration problem(s):", self.errors.len())?; + self.errors + .iter() + .try_for_each(|error| writeln!(f, " - {error}")) + } +} + +pub struct Validated(KnotConfig); + +impl Validated { + pub fn verify_environment(&self) -> Result<(), EnvError> { + verify_writable_dir("repo.scan_path", &self.0.repo.scan_path)?; + self.0 + .lfs + .store_path + .as_deref() + .map_or(Ok(()), |path| verify_writable_dir("lfs.store_path", path))?; + verify_homepage(self.0.homepage.source())?; + verify_master_key(&self.0.secrets.master_key_env) + } + + pub fn into_inner(self) -> KnotConfig { + self.0 + } +} + +impl std::ops::Deref for Validated { + type Target = KnotConfig; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[derive(Debug, thiserror::Error)] +pub enum EnvError { + #[error("{field} {path} isn't accessible")] + DirInaccessible { + field: &'static str, + path: PathBuf, + #[source] + source: std::io::Error, + }, + #[error("{field} {path} isn't directory")] + DirNotDir { field: &'static str, path: PathBuf }, + #[error("{field} {path} isn't writable")] + DirNotWritable { + field: &'static str, + path: PathBuf, + #[source] + source: std::io::Error, + }, + #[error("homepage.path {path} isn't accessible")] + HomepageInaccessible { + path: PathBuf, + #[source] + source: std::io::Error, + }, + #[error("homepage.path {path} isn't a regular file")] + HomepageNotFile { path: PathBuf }, + #[error("master key env var {name} isn't set")] + MasterKeyUnset { name: String }, + #[error("master key env var {name} is empty")] + MasterKeyEmpty { name: String }, + #[error("master key env var {name} isn't valid base64")] + MasterKeyNotBase64 { + name: String, + #[source] + source: base64::DecodeError, + }, + #[error( + "master key env var {name} decodes to only {len} of minimum {MASTER_KEY_MIN_BYTES} bytes" + )] + MasterKeyTooShort { name: String, len: usize }, +} + +const MASTER_KEY_MIN_BYTES: usize = 32; + +fn verify_writable_dir(field: &'static str, path: &Path) -> Result<(), EnvError> { + let metadata = std::fs::metadata(path).map_err(|source| EnvError::DirInaccessible { + field, + path: path.to_path_buf(), + source, + })?; + if !metadata.is_dir() { + return Err(EnvError::DirNotDir { + field, + path: path.to_path_buf(), + }); + } + tempfile::Builder::new() + .prefix(".knot-write-probe") + .tempfile_in(path) + .map(drop) + .map_err(|source| EnvError::DirNotWritable { + field, + path: path.to_path_buf(), + source, + }) +} + +fn verify_homepage(source: HomepageSource) -> Result<(), EnvError> { + let HomepageSource::File(path) = source else { + return Ok(()); + }; + let file = std::fs::File::open(&path).map_err(|source| EnvError::HomepageInaccessible { + path: path.clone(), + source, + })?; + let is_file = file + .metadata() + .map_err(|source| EnvError::HomepageInaccessible { + path: path.clone(), + source, + })? + .is_file(); + is_file + .then_some(()) + .ok_or(EnvError::HomepageNotFile { path }) +} + +fn verify_master_key(name: &str) -> Result<(), EnvError> { + let value = std::env::var(name).ok(); + validate_master_key(name, value.as_deref()) +} + +fn validate_master_key(name: &str, value: Option<&str>) -> Result<(), EnvError> { + let raw = value.ok_or_else(|| EnvError::MasterKeyUnset { + name: name.to_string(), + })?; + let trimmed = raw.trim(); + if trimmed.is_empty() { + return Err(EnvError::MasterKeyEmpty { + name: name.to_string(), + }); + } + let decoded = base64::engine::general_purpose::STANDARD + .decode(trimmed) + .map_err(|source| EnvError::MasterKeyNotBase64 { + name: name.to_string(), + source, + })?; + (decoded.len() >= MASTER_KEY_MIN_BYTES) + .then_some(()) + .ok_or(EnvError::MasterKeyTooShort { + name: name.to_string(), + len: decoded.len(), + }) +} + +static CONFIG: OnceLock = OnceLock::new(); + +#[derive(Debug, thiserror::Error)] +pub enum LoadError { + #[error("config file not found: {0}")] + Missing(PathBuf), + #[error(transparent)] + Confique(#[from] confique::Error), + #[error(transparent)] + Invalid(#[from] ConfigError), +} + +pub fn load(path: Option<&Path>) -> Result { + if let Some(path) = path + && !path.exists() + { + return Err(LoadError::Missing(path.to_path_buf())); + } + let mut builder = KnotConfig::builder().env(); + if let Some(path) = path { + builder = builder.file(path); + } + let config = builder.file("/etc/knot/config.toml").load()?; + config.validate()?; + Ok(Validated(config)) +} + +pub fn template() -> String { + confique::toml::template::(confique::toml::FormatOptions::default()) +} + +pub fn init(config: Validated) { + CONFIG + .set(config.into_inner()) + .expect("knot-config: configuration already initialized"); +} + +pub fn get() -> &'static KnotConfig { + CONFIG + .get() + .expect("knot-config: not initialized, call knot_config::init first") +} + +pub fn try_get() -> Option<&'static KnotConfig> { + CONFIG.get() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn example_toml_is_the_generated_template() { + assert_eq!( + template(), + include_str!("../../../example.toml"), + "regenerate example.toml from knot_config::template() after changing config" + ); + } + + fn sample() -> KnotConfig { + KnotConfig { + server: ServerConfig { + hostname: "oyster.cafe".to_string(), + admins: vec![AccountDid::new("did:plc:nel").unwrap()], + listen_addr: "[::]:5555".parse().unwrap(), + listen_header_timeout_ms: 10_000, + listen_idle_timeout_ms: 60_000, + listen_max_connections: 1_024, + listen_rate_limit_per_second: 50, + listen_rate_limit_burst: 200, + listen_max_inflight_requests: 1_024, + listen_request_timeout_ms: 60_000, + listen_body_timeout_ms: 30_000, + listen_write_request_timeout_ms: 1_800_000, + internal_listen_addr: "[::1]:5444".parse().unwrap(), + ssh_listen_addr: "[::]:2222".parse().unwrap(), + ssh_host_key_file: PathBuf::from("/var/lib/knot/ssh_host_key"), + ssh_max_pack_bytes: 8_589_934_592, + appview_endpoint: AppviewEndpoint::new("https://tangled.org").unwrap(), + }, + tls: TlsConfig { + cert_path: None, + key_path: None, + http3: true, + acme_enabled: false, + acme_cache_dir: None, + acme_contact: None, + acme_staging: false, + mtls_enabled: false, + mtls_client_ca_path: None, + mtls_admin_spki_pin: None, + }, + acl: AclConfig { + admission: AdmissionPolicy::Closed, + }, + repo: RepoConfig { + scan_path: PathBuf::from("/srv/git"), + default_branch: "main".to_string(), + }, + git: GitConfig { + user_name: "Tangled".to_string(), + user_email: "noreply@tangled.sh".to_string(), + object_format: "sha1".to_string(), + }, + secrets: SecretsConfig { + sealed_key_file: PathBuf::from("/var/lib/knot/keys.sealed"), + master_key_env: "KNOT_MASTER_KEY".to_string(), + }, + http: HttpConfig { + connect_timeout_ms: 5_000, + read_timeout_ms: 30_000, + request_timeout_ms: 60_000, + max_response_bytes: 16_777_216, + }, + atproto: AtprotoConfig { + plc_directory: Url::parse("https://plc.nel.pet/").unwrap(), + }, + xrpc: XrpcConfig { + max_body_bytes: 65_536, + max_response_bytes: 5_242_880, + max_archive_bytes: 1_073_741_824, + tree_last_commit_budget_ms: 300, + blob_last_commit_budget_ms: 2_000, + languages_budget_ms: 1_000, + languages_push_budget_ms: 2_000, + max_patch_bytes: 16_777_216, + max_patch_decompressed_bytes: 134_217_728, + preauth_burst: 20, + preauth_refill_ms: 100, + per_peer_inflight: 8, + global_inflight: 64, + max_pending_reservations: 256, + per_actor_reservations: 32, + reservation_ttl_secs: 3_600, + fork_max_pack_bytes: 1_073_741_824, + fork_fetch_timeout_ms: 600_000, + trusted_proxy_header: None, + events_replay_buffer: 4_096, + events_replay_bytes: 67_108_864, + events_max_subscribers: 256, + events_max_per_peer: 8, + }, + maintenance: MaintenanceConfig { + enabled: true, + commit_graph: true, + multi_pack_index: true, + bitmap: true, + interval_secs: 21_600, + repack_max_objects: 16_000_000, + repack_geometric_factor: 2, + prune_grace_secs: 1_209_600, + reflog_expire_secs: 7_776_000, + large_push_bytes: 52_428_800, + }, + pack_cache: PackCacheConfig { + enabled: true, + ttl_secs: 60, + max_entry_bytes: 67_108_864, + max_total_bytes: 536_870_912, + }, + pack: PackConfig { + max_objects: 16_000_000, + max_total_bytes: 68_719_476_736, + selection_max_objects: 16_000_000, + selection_time_budget_secs: 600, + }, + lfs: LfsConfig { + store_path: None, + max_object_bytes: 5_368_709_120, + free_space_floor_bytes: 1_073_741_824, + gc_grace_secs: 1_209_600, + gc_interval_secs: 21_600, + max_ssh_transfers: 16, + max_http_downloads: 64, + }, + resources: ResourcesConfig { + max_threads: 0, + max_memory_bytes: 0, + }, + messages: knot_messages::MessagesConfig::defaults(), + homepage: HomepageConfig { + enabled: true, + path: None, + }, + ci: CiConfig { + logs_addr: Some("logs.oyster.cafe:3333".to_string()), + }, + } + } + + fn apply_acme(config: &mut KnotConfig) { + config.tls.acme_enabled = true; + config.tls.acme_cache_dir = Some(PathBuf::from("/var/lib/knot/acme")); + config.tls.acme_contact = Some("nel@oyster.cafe".to_string()); + } + + fn apply_mtls(config: &mut KnotConfig) { + config.tls.cert_path = Some(PathBuf::from("/etc/knot/tls/cert.pem")); + config.tls.key_path = Some(PathBuf::from("/etc/knot/tls/key.pem")); + config.tls.mtls_enabled = true; + config.tls.mtls_client_ca_path = Some(PathBuf::from("/etc/knot/tls/admin-ca.pem")); + config.tls.mtls_admin_spki_pin = + Some(base64::engine::general_purpose::STANDARD.encode([7u8; 32])); + } + + #[test] + fn accepts_valid_config() { + type Case = (&'static str, fn(&mut KnotConfig), bool, bool); + let cases: &[Case] = &[ + ("valid_config_passes", |_| {}, false, false), + ( + "matched_absolute_tls_paths", + |config| { + config.tls.cert_path = Some(PathBuf::from("/etc/knot/tls/cert.pem")); + config.tls.key_path = Some(PathBuf::from("/etc/knot/tls/key.pem")); + }, + true, + true, + ), + ("acme_enables_tls", apply_acme, true, false), + ("mtls_with_server_cert_and_pin", apply_mtls, true, true), + ( + "an_immediate_prune_grace", + |config| config.maintenance.prune_grace_secs = 0, + false, + false, + ), + ]; + cases + .iter() + .for_each(|(label, mutate, tls_enabled, static_cert)| { + let mut config = sample(); + mutate(&mut config); + assert!(config.validate().is_ok(), "{label}"); + assert_eq!(config.tls_enabled(), *tls_enabled, "{label} tls_enabled"); + assert_eq!( + config.static_cert_enabled(), + *static_cert, + "{label} static_cert_enabled" + ); + }); + } + + #[test] + fn rejects_invalid_config() { + type Case = (&'static str, fn(&mut KnotConfig), &'static str); + let cases: &[Case] = &[ + ( + "a_cert_without_a_key", + |config| config.tls.cert_path = Some(PathBuf::from("/etc/knot/tls/cert.pem")), + "both be set or both unset", + ), + ( + "a_relative_cert_path", + |config| { + config.tls.cert_path = Some(PathBuf::from("tls/cert.pem")); + config.tls.key_path = Some(PathBuf::from("tls/key.pem")); + }, + "tls.cert_path", + ), + ( + "acme_cannot_combine_with_static", + |config| { + apply_acme(config); + config.tls.cert_path = Some(PathBuf::from("/etc/knot/tls/cert.pem")); + config.tls.key_path = Some(PathBuf::from("/etc/knot/tls/key.pem")); + }, + "cannot combine", + ), + ( + "acme_without_a_cache_dir", + |config| { + apply_acme(config); + config.tls.acme_cache_dir = None; + }, + "tls.acme_cache_dir is required", + ), + ( + "acme_without_a_valid_contact", + |config| { + apply_acme(config); + config.tls.acme_contact = Some("not-an-email".to_string()); + }, + "tls.acme_contact", + ), + ( + "mtls_without_a_server_certificate", + |config| { + apply_mtls(config); + config.tls.cert_path = None; + config.tls.key_path = None; + }, + "tls.mtls_enabled requires a server certificate", + ), + ( + "mtls_with_a_malformed_pin", + |config| { + apply_mtls(config); + config.tls.mtls_admin_spki_pin = + Some(base64::engine::general_purpose::STANDARD.encode([0u8; 16])); + }, + "tls.mtls_admin_spki_pin", + ), + ( + "empty_admin_list", + |config| config.server.admins = Vec::new(), + "admins", + ), + ( + "a_zero_maintenance_interval", + |config| config.maintenance.interval_secs = 0, + "maintenance.interval_secs", + ), + ( + "a_zero_repack_object_limit", + |config| config.maintenance.repack_max_objects = 0, + "maintenance.repack_max_objects", + ), + ( + "a_geometric_factor_below_two", + |config| config.maintenance.repack_geometric_factor = 1, + "maintenance.repack_geometric_factor", + ), + ( + "a_zero_large_push_threshold", + |config| config.maintenance.large_push_bytes = 0, + "maintenance.large_push_bytes", + ), + ( + "a_zero_pack_cache_ttl", + |config| config.pack_cache.ttl_secs = 0, + "pack_cache.ttl_secs", + ), + ( + "a_zero_pack_cache_entry_limit", + |config| config.pack_cache.max_entry_bytes = 0, + "pack_cache.max_entry_bytes", + ), + ( + "a_zero_pack_cache_total_limit", + |config| config.pack_cache.max_total_bytes = 0, + "pack_cache.max_total_bytes", + ), + ( + "a_pack_cache_total_below_one_entry", + |config| { + config.pack_cache.max_entry_bytes = 1_000; + config.pack_cache.max_total_bytes = 500; + }, + "at least pack_cache.max_entry_bytes", + ), + ( + "relative_scan_path", + |config| config.repo.scan_path = PathBuf::from("relative/git"), + "scan_path", + ), + ( + "a_relative_lfs_store_path", + |config| config.lfs.store_path = Some(PathBuf::from("relative/lfs")), + "lfs.store_path must be absolute path", + ), + ( + "an_lfs_store_inside_the_scan_path", + |config| config.lfs.store_path = Some(config.repo.scan_path.join("lfs")), + "lfs.store_path mustn't overlap repo.scan_path", + ), + ( + "a_scan_path_inside_the_lfs_store", + |config| { + config.lfs.store_path = Some(PathBuf::from("/srv/media")); + config.repo.scan_path = PathBuf::from("/srv/media/git"); + }, + "lfs.store_path mustn't overlap repo.scan_path", + ), + ( + "a_zero_lfs_object_limit", + |config| config.lfs.max_object_bytes = 0, + "lfs.max_object_bytes", + ), + ( + "a_zero_lfs_gc_interval", + |config| config.lfs.gc_interval_secs = 0, + "lfs.gc_interval_secs", + ), + ( + "a_zero_lfs_ssh_transfer_limit", + |config| config.lfs.max_ssh_transfers = 0, + "lfs.max_ssh_transfers", + ), + ( + "a_zero_lfs_http_download_limit", + |config| config.lfs.max_http_downloads = 0, + "lfs.max_http_downloads", + ), + ( + "bad_master_key_env_name", + |config| config.secrets.master_key_env = "9 bad name".to_string(), + "master_key_env", + ), + ( + "zero_http_timeout", + |config| config.http.request_timeout_ms = 0, + "request_timeout_ms", + ), + ( + "zero_tree_last_commit_budget", + |config| config.xrpc.tree_last_commit_budget_ms = 0, + "tree_last_commit_budget_ms", + ), + ( + "zero_blob_last_commit_budget", + |config| config.xrpc.blob_last_commit_budget_ms = 0, + "blob_last_commit_budget_ms", + ), + ( + "zero_languages_budget", + |config| config.xrpc.languages_budget_ms = 0, + "languages_budget_ms", + ), + ( + "zero_languages_push_budget", + |config| config.xrpc.languages_push_budget_ms = 0, + "languages_push_budget_ms", + ), + ( + "zero_events_replay_buffer", + |config| config.xrpc.events_replay_buffer = 0, + "events_replay_buffer", + ), + ( + "zero_events_replay_bytes", + |config| config.xrpc.events_replay_bytes = 0, + "events_replay_bytes", + ), + ( + "zero_events_max_subscribers", + |config| config.xrpc.events_max_subscribers = 0, + "events_max_subscribers", + ), + ( + "zero_events_max_per_peer", + |config| config.xrpc.events_max_per_peer = 0, + "events_max_per_peer", + ), + ( + "a_per_peer_limit_above_the_global_limit", + |config| { + config.xrpc.events_max_subscribers = 4; + config.xrpc.events_max_per_peer = 8; + }, + "events_max_subscribers must be at least", + ), + ( + "a_non_https_plc_directory", + |config| { + config.atproto.plc_directory = Url::parse("http://plc.nel.pet/").unwrap(); + }, + "plc_directory", + ), + ( + "an_idle_timeout_below_the_header_timeout", + |config| { + config.server.listen_header_timeout_ms = 10_000; + config.server.listen_idle_timeout_ms = 5_000; + }, + "listen_idle_timeout_ms must be at least", + ), + ( + "colliding_bind_ports", + |config| config.server.internal_listen_addr = config.server.listen_addr, + "same port", + ), + ( + "a_relative_homepage_path", + |config| config.homepage.path = Some(PathBuf::from("homepage.html")), + "homepage.path must be absolute path", + ), + ]; + cases.iter().for_each(|(label, mutate, expected)| { + let mut config = sample(); + mutate(&mut config); + let errors = config.validate().unwrap_err().errors; + assert!( + errors.iter().any(|error| error.contains(expected)), + "{label}: expected an error containing {expected}, got {errors:?}" + ); + }); + } + + #[test] + fn homepage_source_resolves_states() { + let disabled = HomepageConfig { + enabled: false, + path: Some(PathBuf::from("/etc/knot/home.html")), + }; + assert!(matches!(disabled.source(), HomepageSource::Disabled)); + + let default = HomepageConfig { + enabled: true, + path: None, + }; + assert!(matches!(default.source(), HomepageSource::Default)); + + let file = HomepageConfig { + enabled: true, + path: Some(PathBuf::from("/etc/knot/home.html")), + }; + match file.source() { + HomepageSource::File(path) => assert_eq!(path, PathBuf::from("/etc/knot/home.html")), + other => panic!("expected File, got {other:?}"), + } + } + + #[test] + fn validates_master_key() { + let short = base64::engine::general_purpose::STANDARD.encode([0u8; 16]); + let key = base64::engine::general_purpose::STANDARD.encode([7u8; 32]); + type Case<'a> = (Option<&'a str>, fn(&Result<(), EnvError>) -> bool); + let cases: Vec> = vec![ + (None, |result| { + matches!(result, Err(EnvError::MasterKeyUnset { .. })) + }), + (Some(" "), |result| { + matches!(result, Err(EnvError::MasterKeyEmpty { .. })) + }), + (Some("not base64 *** value"), |result| { + matches!(result, Err(EnvError::MasterKeyNotBase64 { .. })) + }), + (Some(short.as_str()), |result| { + matches!(result, Err(EnvError::MasterKeyTooShort { .. })) + }), + (Some(key.as_str()), |result| result.is_ok()), + ]; + cases.iter().for_each(|(input, expect)| { + assert!(expect(&validate_master_key("KNOT_MASTER_KEY", *input))); + }); + } + + #[test] + fn admins_parse_from_comma_separated_env() { + let parsed = parse_admins("did:plc:nel, did:plc:olaren").unwrap(); + assert_eq!(parsed.len(), 2); + assert!(parse_admins("not-a-did").is_err()); + } + + #[test] + fn http_limits_map_from_config() { + let limits = sample().http_limits(); + assert_eq!(limits.connect_timeout, Duration::from_millis(5_000)); + assert_eq!(limits.request_timeout, Duration::from_millis(60_000)); + assert_eq!(limits.max_response_bytes, 16_777_216); + } + + #[test] + fn missing_dir_is_rejected() { + let dir = tempfile::tempdir().unwrap(); + let absent = dir.path().join("no-such-dir"); + assert!(matches!( + verify_writable_dir("repo.scan_path", &absent), + Err(EnvError::DirInaccessible { .. }) + )); + } + + #[test] + fn file_in_place_of_dir_is_rejected() { + let file = tempfile::NamedTempFile::new().unwrap(); + assert!(matches!( + verify_writable_dir("lfs.store_path", file.path()), + Err(EnvError::DirNotDir { .. }) + )); + } + + #[test] + fn writable_dir_passes() { + let dir = tempfile::tempdir().unwrap(); + assert!(verify_writable_dir("repo.scan_path", dir.path()).is_ok()); + } + + #[test] + fn verify_homepage_accepts_absent_and_default_sources() { + assert!(verify_homepage(HomepageSource::Disabled).is_ok()); + assert!(verify_homepage(HomepageSource::Default).is_ok()); + } + + #[test] + fn verify_homepage_rejects_missing_file() { + let dir = tempfile::tempdir().unwrap(); + let absent = dir.path().join("no-such-page.html"); + assert!(matches!( + verify_homepage(HomepageSource::File(absent)), + Err(EnvError::HomepageInaccessible { .. }) + )); + } + + #[test] + fn verify_homepage_rejects_directory() { + let dir = tempfile::tempdir().unwrap(); + assert!(matches!( + verify_homepage(HomepageSource::File(dir.path().to_path_buf())), + Err(EnvError::HomepageNotFile { .. }) + )); + } + + #[test] + fn verify_homepage_accepts_readable_file() { + let file = tempfile::NamedTempFile::new().unwrap(); + assert!(verify_homepage(HomepageSource::File(file.path().to_path_buf())).is_ok()); + } + + #[test] + fn disabled_homepage_ignores_relative_path() { + let mut config = sample(); + config.homepage.enabled = false; + config.homepage.path = Some(PathBuf::from("homepage.html")); + assert!(config.validate().is_ok()); + } +} diff --git a/knot2/crates/knot-edge/fuzz/.gitignore b/knot2/crates/knot-edge/fuzz/.gitignore new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-edge/fuzz/.gitignore @@ -0,0 +1,4 @@ +target +corpus +artifacts +coverage diff --git a/knot2/crates/knot-edge/fuzz/Cargo.lock b/knot2/crates/knot-edge/fuzz/Cargo.lock new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-edge/fuzz/Cargo.lock @@ -0,0 +1,4654 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + +[[package]] +name = "arc-swap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +dependencies = [ + "rustversion", +] + +[[package]] +name = "asn1-rs" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5493c3bedbacf7fd7382c6346bbd66687d12bbaad3a89a2d2c303ee6cf20b048" +dependencies = [ + "asn1-rs-derive 0.5.1", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive 0.6.0", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-compression" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-http-codec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "096146020b08dbc4587685b0730a7ba905625af13c65f8028035cdfd69573c91" +dependencies = [ + "anyhow", + "futures", + "http", + "httparse", + "log", +] + +[[package]] +name = "async-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" +dependencies = [ + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-net" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b948000fad4873c1c9339d60f2623323a0cfd3816e5181033c6a5cb68b2accf7" +dependencies = [ + "async-io", + "blocking", + "futures-lite", +] + +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-web-client" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8caf502b44d6d4be6154ac33af012cbb5fef11e6066edcfb42834217fbaf501b" +dependencies = [ + "async-http-codec", + "async-net", + "futures", + "futures-rustls", + "http", + "lazy_static", + "log", + "rustls-pki-types", + "serde", + "thiserror 1.0.69", + "webpki-roots 0.26.11", +] + +[[package]] +name = "atomic-polyfill" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" +dependencies = [ + "critical-section", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "aws-lc-rs" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "base-x" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" + +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + +[[package]] +name = "base256emoji" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e9430d9a245a77c92176e649af6e275f20839a48389859d1661e9a128d077c" +dependencies = [ + "const-str", + "match-lookup", +] + +[[package]] +name = "base45" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240e56f4d3c453c36faacb695c535a4d5f8c7d23dac175014f32eb0a71012a03" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "blocking" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21" +dependencies = [ + "async-channel", + "async-task", + "futures-io", + "futures-lite", + "piper", +] + +[[package]] +name = "bon" +version = "3.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a602c73c7b0148ec6d12af6fd5cc7a46e2eacc8878271a999abac56eed12f561" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + +[[package]] +name = "borrow-or-share" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" + +[[package]] +name = "borsh" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3f6da4992df95bbcd9af42a6c7dcb994498fc9048230405f3b36ff7cd3f145" +dependencies = [ + "bytes", + "cfg_aliases", +] + +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +dependencies = [ + "serde", +] + +[[package]] +name = "cbor4ii" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544cf8c89359205f4f990d0e6f3828db42df85b5dac95d09157a250eb0749c4" +dependencies = [ + "serde", +] + +[[package]] +name = "cc" +version = "1.2.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "cid" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a304f95f84d169a6f31c4d0a30d784643aaa0bbc9c1e449a2c23e963ec4971" +dependencies = [ + "multibase", + "multihash", + "serde", + "serde_bytes", + "unsigned-varint 0.8.0", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.18", +] + +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "brotli", + "compression-core", + "flate2", + "memchr", + "zstd", + "zstd-safe", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "const-str" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f421161cb492475f1661ddc9815a745a1c894592070661180fdec3d4872e9c3" + +[[package]] +name = "cordyceps" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "688d7fbb8092b8de775ef2536f36c8c31f2bc4006ece2e8d8ad2d17d00ce0a2a" +dependencies = [ + "loom", + "tracing", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "data-encoding-macro" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3259c913752a86488b501ed8680446a5ed2d5aeac6e596cb23ba3800768ea32c" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccc2776f0c61eca1ca32528f85548abd1a4be8fb53d1b21c013e4f18da1e7090" +dependencies = [ + "data-encoding", + "syn", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "der-parser" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" +dependencies = [ + "asn1-rs 0.6.2", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs 0.7.2", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "derive_more" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "diatomic-waker" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab03c107fafeb3ee9f5925686dbb7a73bc76e3932abb0d2b365cb64b169cf04c" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.6", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest 0.10.7", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "rand_core 0.6.4", + "serde", + "sha2 0.10.9", + "subtle", + "zeroize", +] + +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "pem-rfc7468", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "faster-hex" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7223ae2d2f179b803433d9c830478527e92b8117eab39460edae7f1614d9fb73" +dependencies = [ + "heapless 0.8.0", + "serde", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fluent-uri" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "forwarded-header-value" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8835f84f38484cc86f110a805655697908257fb9a7af005234060891557198e9" +dependencies = [ + "nonempty", + "thiserror 1.0.69", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-buffered" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4421cb78ee172b6b06080093479d3c50f058e7c81b7d577bbb8d118d551d4cd5" +dependencies = [ + "cordyceps", + "diatomic-waker", + "futures-core", + "pin-project-lite", + "spin 0.10.0", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f2f12607f92c69b12ed746fabf9ca4f5c482cba46679c1a75b874ed7c26adb" +dependencies = [ + "futures-io", + "rustls", + "rustls-pki-types", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-timer" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generator" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + +[[package]] +name = "generic-array" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +dependencies = [ + "typenum", + "version_check", + "zeroize", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "gix-features" +version = "0.48.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1849ae154d38bc403185be14fa871e38e3c93ee606875d94e207fdb9fba52dbc" +dependencies = [ + "gix-trace", + "libc", + "prodash", +] + +[[package]] +name = "gix-hash" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0926d3819c837750b4e03c7754901e73f68b8c9b690753a6372a1bed4eedce" +dependencies = [ + "faster-hex", + "gix-features", + "sha1-checked", + "sha2 0.11.0", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-trace" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44dc45eae785c0eb14173e0f152e6e224dcf4d45b6a6999a3aed22af541ad678" + +[[package]] +name = "governor" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9efcab3c1958580ff1f25a2a41be1668f7603d849bb63af523b208a3cc1223b8" +dependencies = [ + "cfg-if", + "dashmap", + "futures-sink", + "futures-timer", + "futures-util", + "getrandom 0.3.4", + "hashbrown 0.16.1", + "nonzero_ext", + "parking_lot", + "portable-atomic", + "quanta", + "rand 0.9.4", + "smallvec", + "spinning_top", + "web-time", +] + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "h2" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h3" +version = "0.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10872b55cfb02a821b69dc7cf8dc6a71d6af25eb9a79662bec4a9d016056b3be" +dependencies = [ + "bytes", + "fastrand", + "futures-util", + "http", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "h3-quinn" +version = "0.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2e732c8d91a74731663ac8479ab505042fbf547b9a207213ab7fbcbfc4f8b4" +dependencies = [ + "bytes", + "futures", + "h3", + "quinn", + "tokio", + "tokio-util", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heapless" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" +dependencies = [ + "atomic-polyfill", + "hash32 0.2.1", + "rustc_version", + "serde", + "spin 0.9.8", + "stable_deref_trait", +] + +[[package]] +name = "heapless" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32 0.3.1", + "stable_deref_trait", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hybrid-array" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots 1.0.8", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "inventory" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +dependencies = [ + "rustversion", +] + +[[package]] +name = "ipld-core" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090f624976d72f0b0bb71b86d58dc16c15e069193067cb3a3a09d655246cbbda" +dependencies = [ + "cid", + "serde", + "serde_bytes", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "iroh-car" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f8cd4cb9aa083fba8b52e921764252d0b4dcb1cd6d120b809dbfe1106e81a" +dependencies = [ + "anyhow", + "cid", + "futures", + "serde", + "serde_ipld_dagcbor", + "thiserror 1.0.69", + "tokio", + "unsigned-varint 0.7.2", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jacquard-api" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5c803a3c097e3ef8aea63747b4fe3fc9e339cd18272dd0366b1d10dd90d5c3f" +dependencies = [ + "jacquard-common", + "jacquard-derive", + "jacquard-lexicon", + "miette", + "serde", + "thiserror 2.0.18", +] + +[[package]] +name = "jacquard-common" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fec8b6661e6fcfb4d8b9eb1132503be71bc3e3c6f6ff15fa5500ec62655f6da7" +dependencies = [ + "base64", + "bon", + "bytes", + "chrono", + "ciborium", + "ciborium-io", + "cid", + "ed25519-dalek", + "fluent-uri", + "futures", + "getrandom 0.2.17", + "getrandom 0.3.4", + "hashbrown 0.15.5", + "http", + "ipld-core", + "k256", + "maitake-sync", + "miette", + "multibase", + "multihash", + "n0-future", + "oxilangtag", + "p256", + "phf", + "postcard", + "rand 0.9.4", + "regex", + "regex-automata", + "regex-lite", + "reqwest", + "rustversion", + "serde", + "serde_bytes", + "serde_html_form", + "serde_ipld_dagcbor", + "serde_json", + "signature", + "smol_str", + "spin 0.10.0", + "thiserror 2.0.18", + "tokio", + "tokio-tungstenite-wasm", + "tokio-util", + "trait-variant", + "unicode-segmentation", +] + +[[package]] +name = "jacquard-derive" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e41b82a9a72c0d2b907a5e0734e0692615e717cda903502c6c034ae6f3ddddf7" +dependencies = [ + "heck", + "jacquard-lexicon", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jacquard-lexicon" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "160049c269e3d7ec56f130736d25d61953fc6216da660c8b5e102f6b052dd3a1" +dependencies = [ + "cid", + "dashmap", + "heck", + "inventory", + "jacquard-common", + "miette", + "multihash", + "prettyplease", + "proc-macro2", + "quote", + "serde", + "serde_ipld_dagcbor", + "serde_json", + "serde_path_to_error", + "serde_repr", + "serde_with", + "sha2 0.10.9", + "syn", + "thiserror 2.0.18", + "unicode-segmentation", +] + +[[package]] +name = "jacquard-repo" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98986367bb78dadaa0f2f07196bab357786c0e3670d8311b350585b91f84d6eb" +dependencies = [ + "bytes", + "cid", + "ed25519-dalek", + "iroh-car", + "jacquard-api", + "jacquard-common", + "jacquard-derive", + "k256", + "miette", + "multihash", + "n0-future", + "p256", + "serde", + "serde_bytes", + "serde_ipld_dagcbor", + "sha2 0.10.9", + "smol_str", + "thiserror 2.0.18", + "tokio", + "trait-variant", +] + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "k256" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" +dependencies = [ + "cfg-if", + "ecdsa", + "elliptic-curve", + "once_cell", + "sha2 0.10.9", + "signature", +] + +[[package]] +name = "knot-edge" +version = "0.1.0" +dependencies = [ + "arc-swap", + "async-trait", + "axum", + "base64", + "bytes", + "futures", + "governor", + "h3", + "h3-quinn", + "http", + "http-body", + "hyper", + "hyper-util", + "knot-types", + "quinn", + "rustls", + "rustls-acme", + "rustls-pemfile", + "sha2 0.11.0", + "thiserror 2.0.18", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http 0.7.0", + "tower_governor", + "tracing", + "x509-parser 0.18.1", +] + +[[package]] +name = "knot-edge-fuzz" +version = "0.0.0" +dependencies = [ + "knot-edge", + "libfuzzer-sys", +] + +[[package]] +name = "knot-types" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "chrono", + "cid", + "gix-hash", + "http", + "jacquard-common", + "jacquard-derive", + "jacquard-lexicon", + "jacquard-repo", + "miette", + "serde", + "serde_json", + "thiserror 2.0.18", + "walkdir", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "maitake-sync" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6816ab14147f80234c675b80ed6dc4f440d8a1cefc158e766067aedb84c0bcd5" +dependencies = [ + "cordyceps", + "loom", + "mycelium-bitfield", + "pin-project", + "portable-atomic", +] + +[[package]] +name = "match-lookup" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "757aee279b8bdbb9f9e676796fd459e4207a1f986e87886700abf589f5abf771" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "miette" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" +dependencies = [ + "cfg-if", + "miette-derive", + "unicode-width", +] + +[[package]] +name = "miette-derive" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "multibase" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e0e4a371cbf1dfd666b658ba137763edb23c45beb43cfe369b5593cd6b437b6" +dependencies = [ + "base-x", + "base256emoji", + "base45", + "data-encoding", + "data-encoding-macro", +] + +[[package]] +name = "multihash" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "577c63b00ad74d57e8c9aa870b5fccebf2fd64a308a5aee9f1bb88e4aea19447" +dependencies = [ + "serde", + "unsigned-varint 0.8.0", +] + +[[package]] +name = "mycelium-bitfield" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24e0cc5e2c585acbd15c5ce911dff71e1f4d5313f43345873311c4f5efd741cc" + +[[package]] +name = "n0-future" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb0e5d99e681ab3c938842b96fcb41bf8a7bb4bfdb11ccbd653a7e83e06c794" +dependencies = [ + "cfg_aliases", + "derive_more", + "futures-buffered", + "futures-lite", + "futures-util", + "js-sys", + "pin-project", + "send_wrapper", + "tokio", + "tokio-util", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nonempty" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9e591e719385e6ebaeb5ce5d3887f7d5676fceca6411d1925ccc95745f3d6f7" + +[[package]] +name = "nonzero_ext" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "oid-registry" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d8034d9489cdaf79228eb9f6a3b8d7bb32ba00d6645ebd48eef4077ceb5bd9" +dependencies = [ + "asn1-rs 0.6.2", +] + +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs 0.7.2", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "oxilangtag" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d3b4eb570abd4a1dcb062c31fd37b832264d9dc7292c3e69acfe926c87b063f" +dependencies = [ + "serde", +] + +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2 0.10.9", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64", + "serde_core", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand 0.8.6", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "piper" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1" +dependencies = [ + "atomic-waker", + "fastrand", + "futures-io", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "heapless 0.7.17", + "serde", +] + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prodash" +version = "31.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "962200e2d7d551451297d9fdce85138374019ada198e30ea9ede38034e27604c" +dependencies = [ + "parking_lot", +] + +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "futures-io", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.4", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags", +] + +[[package]] +name = "rcgen" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" +dependencies = [ + "aws-lc-rs", + "pem", + "rustls-pki-types", + "time", + "yasna", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "regex" +version = "1.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "mime", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http 0.6.11", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots 1.0.8", +] + +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "aws-lc-rs", + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-acme" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9c70a17ecb067d5067565a16a2e0f26a4a2ea0924f49739d558c45186facc75" +dependencies = [ + "async-io", + "async-trait", + "async-web-client", + "aws-lc-rs", + "base64", + "blocking", + "chrono", + "futures", + "futures-rustls", + "http", + "log", + "pem", + "rcgen", + "serde", + "serde_json", + "thiserror 2.0.18", + "webpki-roots 1.0.8", + "x509-parser 0.16.0", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "send_wrapper" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_html_form" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acf96b1d9364968fce46ebb548f1c0e1d7eceae27bdff73865d42e6c7369d94" +dependencies = [ + "form_urlencoded", + "indexmap", + "itoa", + "serde_core", +] + +[[package]] +name = "serde_ipld_dagcbor" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46182f4f08349a02b45c998ba3215d3f9de826246ba02bb9dddfe9a2a2100778" +dependencies = [ + "cbor4ii", + "ipld-core", + "scopeguard", + "serde", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" +dependencies = [ + "base64", + "bs58", + "chrono", + "hex", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha1-checked" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89f599ac0c323ebb1c6082821a54962b839832b03984598375bff3975b804423" +dependencies = [ + "digest 0.10.7", + "sha1", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "smol_str" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa7368fcf4852a4c2dd92df0cace6a71f2091ca0a23391ce7f3a31833f1523" +dependencies = [ + "borsh", + "serde_core", +] + +[[package]] +name = "socket2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" + +[[package]] +name = "spinning_top" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d96d2d1d716fb500937168cc09353ffdc7a012be8475ac7308e1bdf0e3923300" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711a53c2d47bbd818258c498c8dbfe186a2526c631495cfe7e078567f86b8469" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71c652a3727a9cbb9a02f707f530b618ce00d0ccd762009c8c23bd191df3c17d" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" +dependencies = [ + "futures-util", + "log", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tungstenite", +] + +[[package]] +name = "tokio-tungstenite-wasm" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e21a5c399399c3db9f08d8297ac12b500e86bca82e930253fdc62eaf9c0de6ae" +dependencies = [ + "futures-channel", + "futures-util", + "http", + "httparse", + "js-sys", + "rustls", + "thiserror 1.0.69", + "tokio", + "tokio-tungstenite", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "async-compression", + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tokio", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-http" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233" +dependencies = [ + "async-compression", + "bitflags", + "bytes", + "futures-core", + "http", + "http-body", + "percent-encoding", + "pin-project-lite", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tower_governor" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44de9b94d849d3c46e06a883d72d408c2de6403367b39df2b1c9d9e7b6736fe6" +dependencies = [ + "axum", + "forwarded-header-value", + "governor", + "http", + "pin-project", + "thiserror 2.0.18", + "tower", + "tracing", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "trait-variant" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.8.6", + "rustls", + "rustls-pki-types", + "sha1", + "thiserror 1.0.69", + "utf-8", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "unsigned-varint" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" + +[[package]] +name = "unsigned-varint" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.8", +] + +[[package]] +name = "webpki-roots" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "x509-parser" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcbc162f30700d6f3f82a24bf7cc62ffe7caea42c0b2cba8bf7f3ae50cf51f69" +dependencies = [ + "asn1-rs 0.6.2", + "data-encoding", + "der-parser 9.0.0", + "lazy_static", + "nom", + "oid-registry 0.7.1", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs 0.7.2", + "data-encoding", + "der-parser 10.0.0", + "lazy_static", + "nom", + "oid-registry 0.8.1", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] + +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/knot2/crates/knot-edge/fuzz/Cargo.toml b/knot2/crates/knot-edge/fuzz/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-edge/fuzz/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "knot-edge-fuzz" +version = "0.0.0" +publish = false +edition = "2024" + +[package.metadata] +cargo-fuzz = true + +[workspace] + +[dependencies] +libfuzzer-sys = "0.4" + +[dependencies.knot-edge] +path = ".." + +[[bin]] +name = "spki" +path = "fuzz_targets/spki.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "spki_pin" +path = "fuzz_targets/spki_pin.rs" +test = false +doc = false +bench = false diff --git a/knot2/crates/knot-edge/src/acme.rs b/knot2/crates/knot-edge/src/acme.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-edge/src/acme.rs @@ -0,0 +1,276 @@ +use std::convert::Infallible; +use std::path::PathBuf; +use std::sync::Arc; + +use async_trait::async_trait; +use futures::StreamExt; +use knot_types::KnotHostname; +use rustls::server::ResolvesServerCert; +use rustls_acme::caches::DirCache; +use rustls_acme::{AccountCache, AcmeConfig, CertCache}; +use tokio_util::sync::CancellationToken; + +#[derive(Debug, thiserror::Error)] +pub enum AcmeError { + #[error("acme cache {path}: {source}")] + Cache { + path: String, + #[source] + source: std::io::Error, + }, +} + +struct RestrictedDirCache { + dir: PathBuf, + inner: DirCache, +} + +impl RestrictedDirCache { + fn new(dir: PathBuf) -> Self { + let inner = DirCache::new(dir.clone()); + Self { dir, inner } + } +} + +#[async_trait] +impl CertCache for RestrictedDirCache { + type EC = std::io::Error; + + async fn load_cert( + &self, + domains: &[String], + directory_url: &str, + ) -> Result>, Self::EC> { + self.inner.load_cert(domains, directory_url).await + } + + async fn store_cert( + &self, + domains: &[String], + directory_url: &str, + cert: &[u8], + ) -> Result<(), Self::EC> { + self.inner.store_cert(domains, directory_url, cert).await?; + restrict_cache_dir(&self.dir).map_err(std::io::Error::other) + } +} + +#[async_trait] +impl AccountCache for RestrictedDirCache { + type EA = std::io::Error; + + async fn load_account( + &self, + contact: &[String], + directory_url: &str, + ) -> Result>, Self::EA> { + self.inner.load_account(contact, directory_url).await + } + + async fn store_account( + &self, + contact: &[String], + directory_url: &str, + account: &[u8], + ) -> Result<(), Self::EA> { + self.inner + .store_account(contact, directory_url, account) + .await?; + restrict_cache_dir(&self.dir).map_err(std::io::Error::other) + } +} + +#[derive(Debug, thiserror::Error)] +#[error("acme contact {value:?} isn't a bare email address")] +pub struct AcmeContactError { + value: String, +} + +pub struct AcmeContact(String); + +impl AcmeContact { + // `mailto()` already puts on the scheme for us, + // so someone that came with one + // already went out to the ACME account as mailto:mailto:. + // Hence no colons. + pub fn new(contact: impl Into) -> Result { + let contact = contact.into(); + let mut halves = contact.split('@'); + let well_formed = matches!((halves.next(), halves.next(), halves.next()), (Some(local), Some(domain), None) if !local.is_empty() && domain.contains('.')) + && !contact.contains(':') + && !contact.chars().any(|c| c.is_whitespace() || c.is_control()); + match well_formed { + true => Ok(Self(contact)), + false => Err(AcmeContactError { value: contact }), + } + } + + pub fn mailto(&self) -> String { + format!("mailto:{}", self.0) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AcmeCacheDir(PathBuf); + +impl AcmeCacheDir { + pub fn new(path: impl Into) -> Self { + Self(path.into()) + } + + pub fn as_path(&self) -> &std::path::Path { + &self.0 + } +} + +pub struct AcmeParams { + pub domains: Vec, + pub contact: AcmeContact, + pub cache_dir: AcmeCacheDir, + pub production: bool, +} + +pub fn start( + params: AcmeParams, + shutdown: CancellationToken, +) -> Result, AcmeError> { + std::fs::create_dir_all(params.cache_dir.as_path()).map_err(|source| AcmeError::Cache { + path: params.cache_dir.as_path().display().to_string(), + source, + })?; + restrict_cache_dir(params.cache_dir.as_path())?; + + let mut state = AcmeConfig::::new(params.domains) + .contact_push(params.contact.mailto()) + .cache(RestrictedDirCache::new(params.cache_dir.0)) + .directory_lets_encrypt(params.production) + .state(); + let resolver = state.resolver(); + + tokio::spawn(async move { + loop { + tokio::select! { + () = shutdown.cancelled() => break, + event = state.next() => match event { + Some(Ok(ok)) => tracing::info!("acme: {ok:?}"), + Some(Err(error)) => tracing::warn!("acme: {error:?}"), + None => { + tracing::warn!("acme renewal stream ended, certificates will no longer renew"); + break; + } + }, + } + } + }); + + Ok(resolver) +} + +#[cfg(unix)] +fn restrict_cache_dir(path: &std::path::Path) -> Result<(), AcmeError> { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700)).map_err(|source| { + AcmeError::Cache { + path: path.display().to_string(), + source, + } + })?; + std::fs::read_dir(path) + .map_err(|source| AcmeError::Cache { + path: path.display().to_string(), + source, + })? + .filter_map(Result::ok) + .filter(|entry| { + entry + .file_type() + .map(|kind| kind.is_file()) + .unwrap_or(false) + }) + .try_for_each(|entry| { + std::fs::set_permissions(entry.path(), std::fs::Permissions::from_mode(0o600)).map_err( + |source| AcmeError::Cache { + path: entry.path().display().to_string(), + source, + }, + ) + }) +} + +#[cfg(not(unix))] +fn restrict_cache_dir(_path: &std::path::Path) -> Result<(), AcmeError> { + Ok(()) +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use std::os::unix::fs::PermissionsExt; + + #[test] + fn acme_contact_accepts_a_bare_email_and_rejects_everything_else() { + assert_eq!( + AcmeContact::new("ops@oyster.cafe").unwrap().mailto(), + "mailto:ops@oyster.cafe" + ); + assert!(AcmeContact::new("").is_err()); + assert!(AcmeContact::new("ops").is_err()); + assert!(AcmeContact::new("ops@localhost").is_err()); + assert!(AcmeContact::new("mailto:ops@oyster.cafe").is_err()); + assert!(AcmeContact::new("ops@nel.pet@extra.dev").is_err()); + assert!(AcmeContact::new("ops @oyster.cafe").is_err()); + assert!(AcmeContact::new("@oyster.cafe").is_err()); + } + + #[test] + fn the_cache_dir_and_its_files_are_tightened_to_owner_only() { + let dir = tempfile::tempdir().unwrap(); + let key = dir.path().join("account.key"); + std::fs::write(&key, b"private material").unwrap(); + std::fs::set_permissions(&key, std::fs::Permissions::from_mode(0o644)).unwrap(); + + restrict_cache_dir(dir.path()).unwrap(); + + assert_eq!( + std::fs::metadata(dir.path()).unwrap().permissions().mode() & 0o777, + 0o700, + "the cache directory must be traversable only by its owner" + ); + assert_eq!( + std::fs::metadata(&key).unwrap().permissions().mode() & 0o777, + 0o600, + "a cached private key must be readable only by its owner" + ); + } + + #[tokio::test] + async fn a_cert_stored_after_boot_is_tightened_to_owner_only() { + let dir = tempfile::tempdir().unwrap(); + let cache = RestrictedDirCache::new(dir.path().to_path_buf()); + cache + .store_cert( + &["anemone.knot".to_string()], + "https://acme.test/directory", + b"private cert material", + ) + .await + .unwrap(); + + let modes: Vec = std::fs::read_dir(dir.path()) + .unwrap() + .filter_map(Result::ok) + .map(|entry| { + std::fs::metadata(entry.path()) + .unwrap() + .permissions() + .mode() + & 0o777 + }) + .collect(); + assert_eq!( + modes, + vec![0o600], + "a certificate written after boot must be the only entry and owner-only" + ); + } +} diff --git a/knot2/crates/knot-edge/src/altsvc.rs b/knot2/crates/knot-edge/src/altsvc.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-edge/src/altsvc.rs @@ -0,0 +1,103 @@ +use axum::Router; +use axum::body::Body; +use http::header::ALT_SVC; +use http::{HeaderValue, Request, Response, StatusCode}; + +const ALT_SVC_MAX_AGE_SECS: u32 = 86_400; + +knot_types::scalar_newtype! { + pub struct Port(u16); +} + +pub fn alt_svc_header(port: Port) -> HeaderValue { + let port = port.get(); + HeaderValue::from_str(&format!("h3=\":{port}\"; ma={ALT_SVC_MAX_AGE_SECS}")) + .expect("alt-svc header value is valid ascii") +} + +pub fn with_alt_svc(app: Router, port: Port) -> Router { + let value = alt_svc_header(port); + app.layer(axum::middleware::map_response( + move |mut response: Response| { + let value = value.clone(); + async move { + if response.status() != StatusCode::SWITCHING_PROTOCOLS { + response.headers_mut().insert(ALT_SVC, value); + } + response + } + }, + )) +} + +pub fn with_host_from_authority(app: Router) -> Router { + app.layer(axum::middleware::map_request( + |mut request: Request| async move { + let authority = request + .uri() + .authority() + .map(|authority| HeaderValue::from_str(authority.as_str())); + match ( + request.headers().contains_key(http::header::HOST), + authority, + ) { + (false, Some(Ok(value))) => { + request.headers_mut().insert(http::header::HOST, value); + request + } + _ => request, + } + }, + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + use axum::routing::get; + use tower::ServiceExt; + + #[test] + fn alt_svc_header_advertises_h3() { + assert_eq!( + alt_svc_header(Port::new(443)).to_str().unwrap(), + "h3=\":443\"; ma=86400" + ); + } + + #[tokio::test] + async fn alt_svc_added_to_responses_except_switching_protocols() { + let app = with_alt_svc( + Router::new().route("/ok", get(|| async { "ok" })).route( + "/upgrade", + get(|| async { + Response::builder() + .status(StatusCode::SWITCHING_PROTOCOLS) + .body(Body::empty()) + .unwrap() + }), + ), + Port::new(443), + ); + + let normal = app + .clone() + .oneshot(Request::get("/ok").body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!( + normal.headers().get(ALT_SVC).and_then(|v| v.to_str().ok()), + Some("h3=\":443\"; ma=86400") + ); + + let upgrade = app + .oneshot(Request::get("/upgrade").body(Body::empty()).unwrap()) + .await + .unwrap(); + assert!( + upgrade.headers().get(ALT_SVC).is_none(), + "101 responses mustn't include Alt-Svc" + ); + } +} diff --git a/knot2/crates/knot-edge/src/compression.rs b/knot2/crates/knot-edge/src/compression.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-edge/src/compression.rs @@ -0,0 +1,228 @@ +use http::Response; +use http::header::CONTENT_TYPE; +use http_body::Body; +use tower_http::compression::CompressionLayer; +use tower_http::compression::predicate::{And, Predicate, SizeAbove}; + +const MIN_COMPRESS_BYTES: u64 = 256; + +#[derive(Clone, Copy)] +pub(crate) struct CompressibleResponse; + +impl Predicate for CompressibleResponse { + fn should_compress(&self, response: &Response) -> bool + where + B: Body, + { + response + .headers() + .get(CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .map(|value| { + value + .split(';') + .next() + .unwrap_or(value) + .trim() + .to_ascii_lowercase() + }) + .is_some_and(|base| { + matches!( + base.as_str(), + "application/json" | "application/x-git-upload-pack-advertisement" + ) + }) + } +} + +pub(crate) fn layer() -> CompressionLayer> { + CompressionLayer::new() + .compress_when(SizeAbove::new(MIN_COMPRESS_BYTES).and(CompressibleResponse)) +} + +#[cfg(test)] +mod tests { + use axum::Router; + use axum::response::IntoResponse; + use axum::routing::get; + use http::StatusCode; + use http::header::{ACCEPT_ENCODING, CONTENT_ENCODING, CONTENT_RANGE, CONTENT_TYPE, RANGE}; + use tower::ServiceExt; + + use super::layer; + + async fn json_body() -> impl IntoResponse { + ([(CONTENT_TYPE, "application/json")], "x".repeat(4096)) + } + + async fn advertisement() -> impl IntoResponse { + ( + [(CONTENT_TYPE, "application/x-git-upload-pack-advertisement")], + "x".repeat(4096), + ) + } + + async fn pack_result() -> impl IntoResponse { + ( + [(CONTENT_TYPE, "application/x-git-upload-pack-result")], + "x".repeat(4096), + ) + } + + async fn targz_archive() -> impl IntoResponse { + ([(CONTENT_TYPE, "application/gzip")], "x".repeat(4096)) + } + + async fn zip_archive() -> impl IntoResponse { + ([(CONTENT_TYPE, "application/zip")], "x".repeat(4096)) + } + + async fn small_json() -> impl IntoResponse { + ([(CONTENT_TYPE, "application/json")], "{}") + } + + async fn cased_json() -> impl IntoResponse { + ( + [(CONTENT_TYPE, "Application/JSON; charset=utf-8")], + "x".repeat(4096), + ) + } + + async fn partial_archive() -> impl IntoResponse { + ( + StatusCode::PARTIAL_CONTENT, + [ + (CONTENT_TYPE, "application/gzip"), + (CONTENT_RANGE, "bytes 0-3/4096"), + ], + "xxxx", + ) + } + + fn app() -> Router { + Router::new() + .route("/json", get(json_body)) + .route("/adv", get(advertisement)) + .route("/pack", get(pack_result)) + .route("/targz", get(targz_archive)) + .route("/zip", get(zip_archive)) + .route("/small", get(small_json)) + .route("/cased", get(cased_json)) + .route("/partial", get(partial_archive)) + .layer(layer()) + } + + async fn content_encoding(path: &str, accept: Option<&str>) -> Option { + let mut builder = http::Request::builder().method("GET").uri(path); + if let Some(value) = accept { + builder = builder.header(ACCEPT_ENCODING, value); + } + let request = builder.body(axum::body::Body::empty()).unwrap(); + app() + .oneshot(request) + .await + .unwrap() + .headers() + .get(CONTENT_ENCODING) + .map(|value| value.to_str().unwrap().to_string()) + } + + #[tokio::test] + async fn json_and_advertisement_compress_but_pack_bytes_pass_through() { + let negotiated = ["zstd", "br", "gzip"]; + let json = content_encoding("/json", Some("zstd, br, gzip")).await; + assert!( + json.as_deref().is_some_and(|enc| negotiated.contains(&enc)), + "json negotiates an encoding, got {json:?}" + ); + let adv = content_encoding("/adv", Some("zstd, br, gzip")).await; + assert!( + adv.as_deref().is_some_and(|enc| negotiated.contains(&enc)), + "the ref advertisement negotiates an encoding, got {adv:?}" + ); + assert_eq!( + content_encoding("/pack", Some("zstd, br, gzip")).await, + None, + "an already-compressed pack stream is never re-encoded" + ); + } + + #[tokio::test] + async fn already_compressed_archives_pass_through() { + assert_eq!( + content_encoding("/targz", Some("zstd, br, gzip")).await, + None, + "a gzip archive is never re-encoded" + ); + assert_eq!( + content_encoding("/zip", Some("zstd, br, gzip")).await, + None, + "a zip archive is never re-encoded" + ); + } + + #[tokio::test] + async fn zstd_outranks_brotli_outranks_gzip_on_equal_quality() { + assert_eq!( + content_encoding("/json", Some("gzip, br, zstd")) + .await + .as_deref(), + Some("zstd"), + "zstd wins over brotli and gzip at equal q" + ); + assert_eq!( + content_encoding("/json", Some("gzip, br")).await.as_deref(), + Some("br"), + "brotli wins over gzip at equal q" + ); + assert_eq!( + content_encoding("/json", Some("gzip")).await.as_deref(), + Some("gzip"), + "gzip serves the client that offers only gzip" + ); + } + + #[tokio::test] + async fn nothing_compresses_without_accept_encoding_or_below_the_floor() { + assert_eq!(content_encoding("/json", None).await, None); + assert_eq!( + content_encoding("/small", Some("zstd, br, gzip")).await, + None, + "a body below the size floor is left alone" + ); + } + + #[tokio::test] + async fn a_mixed_case_content_type_still_compresses() { + let negotiated = ["zstd", "br", "gzip"]; + let cased = content_encoding("/cased", Some("zstd, br, gzip")).await; + assert!( + cased + .as_deref() + .is_some_and(|enc| negotiated.contains(&enc)), + "content-type matching is case insensitive, got {cased:?}" + ); + } + + #[tokio::test] + async fn a_partial_archive_passes_through_with_its_range_intact() { + let mut builder = http::Request::builder().method("GET").uri("/partial"); + builder = builder.header(ACCEPT_ENCODING, "zstd, br, gzip"); + builder = builder.header(RANGE, "bytes=0-3"); + let response = app() + .oneshot(builder.body(axum::body::Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!( + response.headers().get(CONTENT_RANGE).unwrap(), + "bytes 0-3/4096", + "the range survives the compression layer untouched" + ); + assert_eq!( + response.headers().get(CONTENT_ENCODING), + None, + "a partial archive is never re-encoded" + ); + } +} diff --git a/knot2/crates/knot-edge/src/lib.rs b/knot2/crates/knot-edge/src/lib.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-edge/src/lib.rs @@ -0,0 +1,391 @@ +mod acme; +mod altsvc; +mod compression; +mod limits; +mod peer; +mod protocol; +mod quic; +mod robustness; +mod tcp; +mod tls; +mod zerortt; + +use std::future::Future; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::pin::Pin; +use std::sync::Arc; + +use axum::Router; +use axum::middleware::from_fn; +use rustls::server::ResolvesServerCert; +use tokio::net::TcpListener; +use tokio_util::sync::CancellationToken; + +pub use acme::{AcmeCacheDir, AcmeContact, AcmeContactError, AcmeError, AcmeParams}; +pub use limits::{ + ConnectionBudget, HeaderTimeout, IdleTimeout, ListenLimits, MaxConcurrentStreams, +}; +pub use peer::SocketPeer; +pub use protocol::NegotiatedProtocol; +pub use quic::EndpointError; +pub use robustness::{ + BodyInactivityTimeout, BurstSize, EdgeGuards, MaxInflightRequests, RequestTimeout, + RequestsPerSecond, WriteRequestTimeout, +}; +pub use tls::{ReloadableCertResolver, SpkiPin, TlsError, load_certified_key}; +pub use zerortt::{EarlyData, RequiresFullHandshake, ZeroRttRoutes, ZeroRttSafe}; + +pub mod fuzz { + pub fn spki_of_certificate(data: &[u8]) { + crate::tls::fuzz_of_certificate(data); + } + + pub fn spki_pin(data: &[u8]) { + let _ = crate::SpkiPin::from_base64(&String::from_utf8_lossy(data)); + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CertChainPath(PathBuf); + +impl CertChainPath { + pub fn new(path: impl Into) -> Self { + Self(path.into()) + } + + pub fn as_path(&self) -> &Path { + &self.0 + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PrivateKeyPath(PathBuf); + +impl PrivateKeyPath { + pub fn new(path: impl Into) -> Self { + Self(path.into()) + } + + pub fn as_path(&self) -> &Path { + &self.0 + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ClientCaPath(PathBuf); + +impl ClientCaPath { + pub fn new(path: impl Into) -> Self { + Self(path.into()) + } + + pub fn as_path(&self) -> &Path { + &self.0 + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PublicBind(SocketAddr); + +impl PublicBind { + pub const fn new(addr: SocketAddr) -> Self { + Self(addr) + } + + pub const fn get(self) -> SocketAddr { + self.0 + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct InternalBind(SocketAddr); + +impl InternalBind { + pub const fn new(addr: SocketAddr) -> Self { + Self(addr) + } + + pub const fn get(self) -> SocketAddr { + self.0 + } +} + +pub struct StaticCertPaths { + pub cert_path: CertChainPath, + pub key_path: PrivateKeyPath, +} + +pub enum CertSource { + Static(StaticCertPaths), + Acme(AcmeParams), +} + +pub struct InternalTls { + pub addr: InternalBind, + pub client_ca_path: ClientCaPath, + pub spki_pin: SpkiPin, +} + +pub struct TlsSetup { + pub source: CertSource, + pub http3: bool, + pub internal: Option, +} + +pub struct EdgeConfig { + pub http_addr: PublicBind, + pub limits: ListenLimits, + pub guards: EdgeGuards, + pub tls: Option, +} + +#[derive(Debug, thiserror::Error)] +pub enum EdgeError { + #[error(transparent)] + Io(#[from] std::io::Error), + #[error(transparent)] + Tls(#[from] TlsError), + #[error(transparent)] + Acme(#[from] AcmeError), + #[error(transparent)] + Endpoint(#[from] EndpointError), +} + +type Served = Pin> + Send>>; + +fn base_router(app: RequiresFullHandshake, early_data_safe: ZeroRttRoutes) -> Router { + early_data_safe + .into_router() + .merge(app.into_router()) + .layer(compression::layer()) + .layer(from_fn(zerortt::tag_from_header)) +} + +fn finish(router: Router) -> Router { + altsvc::with_host_from_authority(router.layer(from_fn(protocol::tag))) +} + +pub async fn serve( + config: EdgeConfig, + app: RequiresFullHandshake, + early_data_safe: ZeroRttRoutes, + shutdown: CancellationToken, +) -> Result<(), EdgeError> { + let EdgeConfig { + http_addr, + limits, + guards, + tls, + } = config; + let layers = guards.prepare(&shutdown); + let early_data = early_data_safe.early_data_policy(); + let wants_internal = tls + .as_ref() + .and_then(|setup| setup.internal.as_ref()) + .is_some(); + let base = base_router(app, early_data_safe); + let internal_router = wants_internal.then(|| finish(base.clone())); + let router = finish(robustness::apply(base, layers)); + let listener = TcpListener::bind(http_addr.get()).await?; + + let Some(setup) = tls else { + return Ok(tcp::serve_plaintext(listener, router, limits, shutdown).await?); + }; + + let (resolver, acme): (Arc, bool) = match setup.source { + CertSource::Static(paths) => { + let certified = tls::load_certified_key(&paths)?; + let reloadable = Arc::new(ReloadableCertResolver::new(certified)); + tls::spawn_cert_reload(Arc::clone(&reloadable), paths, shutdown.clone()); + (reloadable, false) + } + CertSource::Acme(params) => (acme::start(params, shutdown.clone())?, true), + }; + + let extra_alpn: &[&[u8]] = if acme { &[tls::ACME_TLS_ALPN] } else { &[] }; + let tcp_config = Arc::new(tls::build_tls_server_config( + Arc::clone(&resolver), + extra_alpn, + )?); + let port = http_addr.get().port(); + + let mut servers: Vec = Vec::new(); + + servers.push({ + let app = match setup.http3 { + true => altsvc::with_alt_svc(router.clone(), altsvc::Port::new(port)), + false => router.clone(), + }; + let shutdown = shutdown.clone(); + Box::pin(async move { + let result = tcp::serve_tls(listener, app, tcp_config, limits, shutdown.clone()).await; + shutdown.cancel(); + Ok(result?) + }) + }); + + if setup.http3 { + let endpoint = + quic::build_endpoint(http_addr.get(), Arc::clone(&resolver), limits, early_data)?; + let app = router.clone(); + let shutdown = shutdown.clone(); + servers.push(Box::pin(async move { + quic::serve_http3(endpoint, app, limits, shutdown.clone()).await; + shutdown.cancel(); + Ok(()) + })); + } + + if let Some(internal) = setup.internal { + let internal_listener = TcpListener::bind(internal.addr.get()).await?; + let mtls_config = Arc::new(tls::build_mtls_server_config( + Arc::clone(&resolver), + &internal.client_ca_path, + internal.spki_pin, + )?); + let app = internal_router + .expect("an internal router is built whenever an internal bind is configured"); + let shutdown = shutdown.clone(); + servers.push(Box::pin(async move { + let result = tcp::serve_tls( + internal_listener, + app, + mtls_config, + limits, + shutdown.clone(), + ) + .await; + shutdown.cancel(); + Ok(result?) + })); + } + + futures::future::join_all(servers) + .await + .into_iter() + .collect::, EdgeError>>() + .map(drop) +} + +#[cfg(test)] +mod tests { + use super::*; + + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::routing::post; + use http::{Request, StatusCode}; + use tower::ServiceExt; + + use std::num::{NonZeroU32, NonZeroU64}; + + use zerortt::ZeroRttSafe; + + fn test_layers() -> robustness::GuardLayers { + EdgeGuards::new( + RequestsPerSecond::new(NonZeroU32::new(10_000).unwrap()), + BurstSize::new(NonZeroU32::new(10_000).unwrap()), + MaxInflightRequests::new(NonZeroU32::new(1_024).unwrap()), + RequestTimeout::from_millis(NonZeroU64::new(30_000).unwrap()), + BodyInactivityTimeout::from_millis(NonZeroU64::new(30_000).unwrap()), + WriteRequestTimeout::from_millis(NonZeroU64::new(1_800_000).unwrap()), + None, + ) + .prepare(&CancellationToken::new()) + } + + fn wired() -> Router { + let safe = + ZeroRttRoutes::new().get("/info/refs", ZeroRttSafe::new(|| async { "advertisement" })); + let full = RequiresFullHandshake::new( + Router::new().route("/git-upload-pack", post(|| async { "pack" })), + ); + finish(robustness::apply(base_router(full, safe), test_layers())) + } + + fn tight_layers() -> robustness::GuardLayers { + EdgeGuards::new( + RequestsPerSecond::new(NonZeroU32::new(1).unwrap()), + BurstSize::new(NonZeroU32::new(2).unwrap()), + MaxInflightRequests::new(NonZeroU32::new(1_024).unwrap()), + RequestTimeout::from_millis(NonZeroU64::new(30_000).unwrap()), + BodyInactivityTimeout::from_millis(NonZeroU64::new(30_000).unwrap()), + WriteRequestTimeout::from_millis(NonZeroU64::new(1_800_000).unwrap()), + None, + ) + .prepare(&CancellationToken::new()) + } + + async fn status_of(mut request: Request) -> StatusCode { + request + .extensions_mut() + .insert(ConnectInfo(SocketAddr::from(([127, 0, 0, 1], 41001)))); + wired().oneshot(request).await.unwrap().status() + } + + #[tokio::test] + async fn a_write_in_early_data_is_refused_with_425() { + let request = Request::post("/git-upload-pack") + .header("early-data", "1") + .body(Body::empty()) + .unwrap(); + assert_eq!(status_of(request).await, StatusCode::TOO_EARLY); + } + + #[tokio::test] + async fn a_write_after_the_handshake_is_served() { + let request = Request::post("/git-upload-pack") + .body(Body::empty()) + .unwrap(); + assert_eq!(status_of(request).await, StatusCode::OK); + } + + #[tokio::test] + async fn the_advertisement_is_served_even_in_early_data() { + let request = Request::get("/info/refs") + .header("early-data", "1") + .body(Body::empty()) + .unwrap(); + assert_eq!(status_of(request).await, StatusCode::OK); + } + + #[tokio::test] + async fn the_internal_admin_router_shares_no_rate_limit_budget_with_the_data_plane() { + let safe = ZeroRttRoutes::new().get("/info/refs", ZeroRttSafe::new(|| async { "ok" })); + let full = RequiresFullHandshake::new( + Router::new().route("/git-upload-pack", post(|| async { "pack" })), + ); + let base = base_router(full, safe); + let public = finish(robustness::apply(base.clone(), tight_layers())); + let internal = finish(base); + + let request = || { + let mut request = Request::get("/info/refs").body(Body::empty()).unwrap(); + request + .extensions_mut() + .insert(ConnectInfo(SocketAddr::from(([127, 0, 0, 1], 41001)))); + request + }; + + let p1 = public.clone().oneshot(request()).await.unwrap().status(); + let p2 = public.clone().oneshot(request()).await.unwrap().status(); + let p3 = public.clone().oneshot(request()).await.unwrap().status(); + assert_eq!([p1, p2], [StatusCode::OK, StatusCode::OK]); + assert_eq!( + p3, + StatusCode::TOO_MANY_REQUESTS, + "the public edge still enforces the per-IP burst" + ); + + let i1 = internal.clone().oneshot(request()).await.unwrap().status(); + let i2 = internal.clone().oneshot(request()).await.unwrap().status(); + let i3 = internal.clone().oneshot(request()).await.unwrap().status(); + let i4 = internal.clone().oneshot(request()).await.unwrap().status(); + assert_eq!( + [i1, i2, i3, i4], + [StatusCode::OK; 4], + "the internal admin bind is unguarded, so a public flood never sheds admin requests" + ); + } +} diff --git a/knot2/crates/knot-edge/src/limits.rs b/knot2/crates/knot-edge/src/limits.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-edge/src/limits.rs @@ -0,0 +1,104 @@ +use std::num::{NonZeroU32, NonZeroU64}; +use std::time::Duration; + +const MAX_CONCURRENT_STREAMS: u32 = 256; +const STREAM_RECEIVE_WINDOW: u32 = 8 * 1024 * 1024; +const CONNECTION_RECEIVE_WINDOW: u32 = 32 * 1024 * 1024; + +knot_types::scalar_newtype! { + pub struct MaxConcurrentStreams(u32) => sealed; +} + +#[derive(Debug, Clone, Copy)] +pub struct ConnectionBudget { + max_concurrent_streams: MaxConcurrentStreams, + stream_receive_window: u32, + connection_receive_window: u32, +} + +impl ConnectionBudget { + const DEFAULT: Self = Self { + max_concurrent_streams: MaxConcurrentStreams::new(MAX_CONCURRENT_STREAMS), + stream_receive_window: STREAM_RECEIVE_WINDOW, + connection_receive_window: CONNECTION_RECEIVE_WINDOW, + }; + + pub fn max_concurrent_streams(self) -> MaxConcurrentStreams { + self.max_concurrent_streams + } + + pub fn stream_receive_window(self) -> u32 { + self.stream_receive_window + } + + pub fn connection_receive_window(self) -> u32 { + self.connection_receive_window + } +} + +// Separate types that `ListenLimits::new` used to take as +// a bunch of `NonZeroU64` in a row. +// Swapping them would incorrectly compile and +// gave the conn the wrong deadline. +#[derive(Debug, Clone, Copy)] +pub struct HeaderTimeout(Duration); + +impl HeaderTimeout { + pub fn from_millis(millis: NonZeroU64) -> Self { + Self(Duration::from_millis(millis.get())) + } + + pub fn get(self) -> Duration { + self.0 + } +} + +#[derive(Debug, Clone, Copy)] +pub struct IdleTimeout(Duration); + +impl IdleTimeout { + pub fn from_millis(millis: NonZeroU64) -> Self { + Self(Duration::from_millis(millis.get())) + } + + pub fn get(self) -> Duration { + self.0 + } +} + +#[derive(Debug, Clone, Copy)] +pub struct ListenLimits { + header_timeout: HeaderTimeout, + idle_timeout: IdleTimeout, + max_connections: usize, +} + +impl ListenLimits { + pub fn new( + header_timeout: HeaderTimeout, + idle_timeout: IdleTimeout, + max_connections: NonZeroU32, + ) -> Self { + Self { + header_timeout, + idle_timeout, + max_connections: max_connections.get() as usize, + } + } + + pub fn header_timeout(&self) -> HeaderTimeout { + self.header_timeout + } + + pub fn idle_timeout(&self) -> IdleTimeout { + self.idle_timeout + } + + pub fn max_connections(&self) -> usize { + self.max_connections + } + + pub fn connection_budget(&self) -> ConnectionBudget { + ConnectionBudget::DEFAULT + } +} diff --git a/knot2/crates/knot-edge/src/peer.rs b/knot2/crates/knot-edge/src/peer.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-edge/src/peer.rs @@ -0,0 +1,58 @@ +use std::convert::Infallible; +use std::net::{IpAddr, SocketAddr}; + +use axum::extract::{ConnectInfo, FromRequestParts}; +use http::request::Parts; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SocketPeer(Option); + +impl SocketPeer { + pub fn ip(self) -> Option { + self.0 + } +} + +impl FromRequestParts for SocketPeer { + type Rejection = Infallible; + + async fn from_request_parts(parts: &mut Parts, _state: &S) -> Result { + Ok(Self( + parts + .extensions + .get::>() + .map(|connect| connect.0.ip()), + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::body::Body; + use std::net::Ipv4Addr; + + #[tokio::test] + async fn the_extractor_reads_connect_info_and_tolerates_its_absence() { + let with = { + let mut request = http::Request::builder().body(Body::empty()).unwrap(); + request + .extensions_mut() + .insert(ConnectInfo(SocketAddr::from(([203, 0, 113, 7], 443)))); + let (mut parts, _) = request.into_parts(); + SocketPeer::from_request_parts(&mut parts, &()) + .await + .unwrap() + }; + assert_eq!(with.ip(), Some(IpAddr::V4(Ipv4Addr::new(203, 0, 113, 7)))); + + let without = { + let request = http::Request::builder().body(Body::empty()).unwrap(); + let (mut parts, _) = request.into_parts(); + SocketPeer::from_request_parts(&mut parts, &()) + .await + .unwrap() + }; + assert_eq!(without.ip(), None); + } +} diff --git a/knot2/crates/knot-edge/src/protocol.rs b/knot2/crates/knot-edge/src/protocol.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-edge/src/protocol.rs @@ -0,0 +1,78 @@ +use axum::extract::Request; +use axum::middleware::Next; +use axum::response::Response; +use http::Version; +use tracing::Instrument; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NegotiatedProtocol { + H1, + H2, + H3, +} + +impl NegotiatedProtocol { + pub fn as_str(self) -> &'static str { + match self { + NegotiatedProtocol::H1 => "h1", + NegotiatedProtocol::H2 => "h2", + NegotiatedProtocol::H3 => "h3", + } + } + + pub fn from_version(version: Version) -> Option { + match version { + Version::HTTP_10 | Version::HTTP_11 => Some(NegotiatedProtocol::H1), + Version::HTTP_2 => Some(NegotiatedProtocol::H2), + Version::HTTP_3 => Some(NegotiatedProtocol::H3), + _ => None, + } + } +} + +pub(crate) async fn tag(mut request: Request, next: Next) -> Response { + let protocol = NegotiatedProtocol::from_version(request.version()); + if let Some(protocol) = protocol { + request.extensions_mut().insert(protocol); + } + let span = tracing::debug_span!( + "request", + protocol = protocol + .map(NegotiatedProtocol::as_str) + .unwrap_or("unknown") + ); + next.run(request).instrument(span).await +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn each_http_version_maps_to_its_negotiated_protocol() { + assert_eq!( + NegotiatedProtocol::from_version(Version::HTTP_11), + Some(NegotiatedProtocol::H1) + ); + assert_eq!( + NegotiatedProtocol::from_version(Version::HTTP_10), + Some(NegotiatedProtocol::H1) + ); + assert_eq!( + NegotiatedProtocol::from_version(Version::HTTP_2), + Some(NegotiatedProtocol::H2) + ); + assert_eq!( + NegotiatedProtocol::from_version(Version::HTTP_3), + Some(NegotiatedProtocol::H3) + ); + assert_eq!(NegotiatedProtocol::from_version(Version::HTTP_09), None); + } + + #[test] + fn the_label_is_stable_for_logging_and_metrics() { + assert_eq!(NegotiatedProtocol::H1.as_str(), "h1"); + assert_eq!(NegotiatedProtocol::H2.as_str(), "h2"); + assert_eq!(NegotiatedProtocol::H3.as_str(), "h3"); + } +} diff --git a/knot2/crates/knot-edge/src/quic.rs b/knot2/crates/knot-edge/src/quic.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-edge/src/quic.rs @@ -0,0 +1,501 @@ +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; + +use axum::Router; +use axum::body::Body; +use axum::extract::ConnectInfo; +use bytes::{Buf, Bytes}; +use futures::{FutureExt, StreamExt}; +use http::{Request, Response, Version}; +use quinn::{Endpoint, Incoming}; +use tokio::sync::Semaphore; +use tokio_util::sync::CancellationToken; +use tokio_util::task::TaskTracker; +use tower::ServiceExt; + +use rustls::server::ResolvesServerCert; + +use crate::limits::ListenLimits; +use crate::tls::{self, TlsError}; +use crate::zerortt::{EarlyData, EarlyDataPolicy}; + +fn early_data(confirmed: bool) -> EarlyData { + match confirmed { + true => EarlyData::No, + false => EarlyData::Yes, + } +} + +const LISTENER_DRAIN_GRACE: Duration = Duration::from_secs(30); +const ENDPOINT_DRAIN_GRACE: Duration = Duration::from_secs(5); +const CONNECTION_DRAIN_GRACE: Duration = Duration::from_secs(10); + +pub fn build_endpoint( + addr: SocketAddr, + resolver: Arc, + limits: ListenLimits, + early_data: EarlyDataPolicy, +) -> Result { + let server_config = tls::build_quic_server_config(resolver, limits, early_data)?; + let endpoint = Endpoint::server(server_config, addr)?; + Ok(endpoint) +} + +#[derive(Debug, thiserror::Error)] +pub enum EndpointError { + #[error(transparent)] + Tls(#[from] TlsError), + #[error("binding quic socket: {0}")] + Bind(#[from] std::io::Error), +} + +pub async fn serve_http3( + endpoint: Endpoint, + app: Router, + limits: ListenLimits, + shutdown: CancellationToken, +) { + let tracker = TaskTracker::new(); + let connections = Arc::new(Semaphore::new(limits.max_connections())); + loop { + let incoming = tokio::select! { + () = shutdown.cancelled() => break, + incoming = endpoint.accept() => incoming, + }; + let Some(incoming) = incoming else { break }; + let Ok(permit) = Arc::clone(&connections).try_acquire_owned() else { + incoming.refuse(); + continue; + }; + let app = app.clone(); + let conn_shutdown = shutdown.clone(); + let conn_tracker = tracker.clone(); + tracker.spawn(async move { + let _permit = permit; + if let Err(error) = serve_connection(incoming, app, conn_shutdown, conn_tracker).await { + tracing::debug!("h3 connection ended: {error}"); + } + }); + } + tracker.close(); + let _ = tokio::time::timeout(LISTENER_DRAIN_GRACE, tracker.wait()).await; + endpoint.close(0u32.into(), b"shutdown"); + let _ = tokio::time::timeout(ENDPOINT_DRAIN_GRACE, endpoint.wait_idle()).await; +} + +async fn serve_connection( + incoming: Incoming, + app: Router, + shutdown: CancellationToken, + tracker: TaskTracker, +) -> Result<(), Box> { + let (conn, confirmation, mut confirmed) = match incoming.accept()?.into_0rtt() { + Ok((conn, accepted)) => (conn, accepted.map(|_| ()).left_future(), false), + Err(connecting) => ( + connecting.await?, + std::future::pending::<()>().right_future(), + true, + ), + }; + tokio::pin!(confirmation); + let remote = conn.remote_address(); + let quic = conn.clone(); + tracing::trace!("h3 connection from {remote} accepted"); + let mut h3_conn = + h3::server::Connection::<_, Bytes>::new(h3_quinn::Connection::new(conn)).await?; + tracing::trace!("h3 connection from {remote} established"); + + let drain_deadline = tokio::time::sleep(CONNECTION_DRAIN_GRACE); + tokio::pin!(drain_deadline); + let mut draining = false; + loop { + tokio::select! { + biased; + () = shutdown.cancelled(), if !draining => { + draining = true; + drain_deadline + .as_mut() + .reset(tokio::time::Instant::now() + CONNECTION_DRAIN_GRACE); + let _ = h3_conn.shutdown(0).await; + } + () = &mut drain_deadline, if draining => { + tracing::debug!("h3 connection from {remote} drain timed out, closing"); + quic.close(0u32.into(), b"drain timeout"); + break; + } + resolved = h3_conn.accept() => match resolved { + Ok(Some(resolver)) => { + let app = app.clone(); + let early = early_data(confirmed); + tracker.spawn(async move { + if let Err(error) = serve_request(resolver, app, remote, early).await { + tracing::debug!("h3 request from {remote} failed: {error}"); + } + }); + } + Ok(None) => { + tracing::debug!("h3 connection from {remote} closed by the client"); + break; + } + Err(error) => { + tracing::debug!("h3 accept from {remote} error: {error}"); + break; + } + }, + () = &mut confirmation, if !confirmed => { + confirmed = true; + } + } + } + Ok(()) +} + +async fn serve_request( + resolver: h3::server::RequestResolver, + app: Router, + remote: SocketAddr, + early: EarlyData, +) -> Result<(), Box> { + let (request, stream) = resolver.resolve_request().await?; + let (mut send, recv) = stream.split(); + + let (mut parts, ()) = request.into_parts(); + parts.version = Version::HTTP_3; + parts.extensions.insert(ConnectInfo(remote)); + parts.extensions.insert(early); + let request = Request::from_parts(parts, request_body(recv)); + + let response = match app.oneshot(request).await { + Ok(response) => response, + Err(infallible) => match infallible {}, + }; + + let (parts, body) = response.into_parts(); + send.send_response(Response::from_parts(parts, ())).await?; + + let mut data = body.into_data_stream(); + while let Some(chunk) = data.next().await { + match chunk { + Ok(bytes) if bytes.has_remaining() => send.send_data(bytes).await?, + Ok(_) => {} + Err(error) => { + tracing::debug!("h3 response body to {remote} errored: {error}"); + send.stop_stream(h3::error::Code::H3_INTERNAL_ERROR); + return Ok(()); + } + } + } + send.finish().await?; + Ok(()) +} + +struct RecvGuard { + stream: h3::server::RequestStream, + ended: bool, +} + +impl Drop for RecvGuard { + fn drop(&mut self) { + if !self.ended { + self.stream.stop_sending(h3::error::Code::H3_NO_ERROR); + } + } +} + +fn request_body(recv: h3::server::RequestStream) -> Body { + let guard = RecvGuard { + stream: recv, + ended: false, + }; + let stream = futures::stream::unfold(Some(guard), |state| async move { + let mut guard = state?; + match guard.stream.recv_data().await { + Ok(Some(mut buf)) => { + let bytes = buf.copy_to_bytes(buf.remaining()); + Some((Ok::(bytes), Some(guard))) + } + Ok(None) => { + guard.ended = true; + None + } + Err(error) => { + guard.ended = true; + Some((Err(std::io::Error::other(error.to_string())), None)) + } + } + }); + Body::from_stream(stream) +} + +#[cfg(test)] +mod tests { + use super::*; + + use axum::routing::get; + use rustls::crypto::aws_lc_rs; + + use crate::tls; + + #[test] + fn an_unconfirmed_handshake_is_early_data_and_a_confirmed_one_is_not() { + assert_eq!( + early_data(false), + EarlyData::Yes, + "data before handshake confirmation is early data, fail closed" + ); + assert_eq!(early_data(true), EarlyData::No); + } + + fn client_endpoint() -> Endpoint { + client_endpoint_with_provider(aws_lc_rs::default_provider()) + } + + fn client_endpoint_with_provider(provider: rustls::crypto::CryptoProvider) -> Endpoint { + let mut crypto = rustls::ClientConfig::builder_with_provider(Arc::new(provider)) + .with_protocol_versions(&[&rustls::version::TLS13]) + .unwrap() + .dangerous() + .with_custom_certificate_verifier(Arc::new(tls::test_support::AcceptAnyServerCert)) + .with_no_client_auth(); + crypto.alpn_protocols = vec![b"h3".to_vec()]; + let quic = quinn::crypto::rustls::QuicClientConfig::try_from(crypto).unwrap(); + let mut endpoint = Endpoint::client("[::1]:0".parse().unwrap()).unwrap(); + endpoint.set_default_client_config(quinn::ClientConfig::new(Arc::new(quic))); + endpoint + } + + fn spawn_h3_server( + app: Router, + limits: ListenLimits, + early_data: EarlyDataPolicy, + ) -> (SocketAddr, CancellationToken) { + let endpoint = build_endpoint( + "[::1]:0".parse().unwrap(), + tls::test_support::resolver(), + limits, + early_data, + ) + .unwrap(); + let addr = endpoint.local_addr().unwrap(); + let shutdown = CancellationToken::new(); + tokio::spawn(serve_http3( + endpoint, + crate::altsvc::with_host_from_authority(app), + limits, + shutdown.clone(), + )); + (addr, shutdown) + } + + async fn h3_client_connect( + client: &Endpoint, + addr: SocketAddr, + ) -> h3::client::SendRequest { + let conn = client.connect(addr, "localhost").unwrap().await.unwrap(); + let (mut driver, send_request) = h3::client::new(h3_quinn::Connection::new(conn)) + .await + .unwrap(); + tokio::spawn(async move { std::future::poll_fn(|cx| driver.poll_close(cx)).await }); + send_request + } + + async fn h3_request( + send_request: &mut h3::client::SendRequest, + path: &str, + ) -> (http::StatusCode, Vec) { + let request = http::Request::get(format!("https://localhost{path}")) + .body(()) + .unwrap(); + let mut stream = send_request.send_request(request).await.unwrap(); + stream.finish().await.unwrap(); + let status = stream.recv_response().await.unwrap().status(); + let chunks: Vec = futures::stream::unfold(stream, |mut stream| async move { + stream + .recv_data() + .await + .unwrap() + .map(|mut buf| (buf.copy_to_bytes(buf.remaining()), stream)) + }) + .collect() + .await; + let body = chunks + .iter() + .flat_map(|chunk| chunk.iter().copied()) + .collect(); + (status, body) + } + + #[tokio::test] + async fn an_h3_get_roundtrips_through_the_router() { + let app = Router::new().route( + "/", + get(|ConnectInfo(peer): ConnectInfo| async move { peer.to_string() }), + ); + let (addr, shutdown) = + spawn_h3_server(app, tls::test_support::limits(), EarlyDataPolicy::Disabled); + + let client = client_endpoint(); + let client_port = client.local_addr().unwrap().port(); + let mut send_request = h3_client_connect(&client, addr).await; + let (status, body) = h3_request(&mut send_request, "/").await; + assert_eq!(status, 200); + let reported: SocketAddr = String::from_utf8(body).unwrap().parse().unwrap(); + assert_eq!( + reported.port(), + client_port, + "the h3 handler must see the QUIC remote address via ConnectInfo" + ); + shutdown.cancel(); + } + + #[tokio::test] + async fn an_h3_request_is_tagged_with_the_h3_protocol() { + use axum::middleware::from_fn; + + let app = Router::new() + .route( + "/proto", + get(|req: axum::extract::Request| async move { + req.extensions() + .get::() + .map(|protocol| protocol.as_str()) + .unwrap_or("missing") + .to_string() + }), + ) + .layer(from_fn(crate::protocol::tag)); + let (addr, shutdown) = + spawn_h3_server(app, tls::test_support::limits(), EarlyDataPolicy::Disabled); + + let client = client_endpoint(); + let mut send_request = h3_client_connect(&client, addr).await; + let (status, body) = h3_request(&mut send_request, "/proto").await; + assert_eq!(status, 200); + assert_eq!( + String::from_utf8(body).unwrap(), + "h3", + "a request served over QUIC must be tagged as the h3 negotiated protocol" + ); + shutdown.cancel(); + } + + #[tokio::test] + async fn an_early_data_enabled_endpoint_still_serves_through_the_zero_rtt_path() { + let app = Router::new().route("/", get(|| async { "ok" })); + let (addr, shutdown) = + spawn_h3_server(app, tls::test_support::limits(), EarlyDataPolicy::Enabled); + + let client = client_endpoint(); + let mut send_request = h3_client_connect(&client, addr).await; + let (status, _) = h3_request(&mut send_request, "/").await; + assert_eq!( + status, 200, + "a server with early data enabled must serve requests through the 0-RTT acceptance path" + ); + shutdown.cancel(); + } + + #[tokio::test] + async fn a_classical_only_h3_client_completes_over_x25519() { + let app = Router::new().route("/", get(|| async { "ok" })); + let (addr, shutdown) = + spawn_h3_server(app, tls::test_support::limits(), EarlyDataPolicy::Disabled); + + let client = client_endpoint_with_provider(tls::test_support::classical_only_provider()); + let mut send_request = h3_client_connect(&client, addr).await; + let (status, _) = h3_request(&mut send_request, "/").await; + assert_eq!( + status, 200, + "a QUIC client without ML-KEM must still complete the h3 handshake over classical X25519" + ); + shutdown.cancel(); + } + + #[tokio::test] + async fn an_in_flight_h3_request_finishes_during_drain() { + let app = Router::new().route( + "/slow", + get(|| async { + tokio::time::sleep(Duration::from_millis(300)).await; + "drained-clean" + }), + ); + let (addr, shutdown) = + spawn_h3_server(app, tls::test_support::limits(), EarlyDataPolicy::Disabled); + + let client = client_endpoint(); + let mut send_request = h3_client_connect(&client, addr).await; + let ((status, body), ()) = tokio::join!(h3_request(&mut send_request, "/slow"), async { + tokio::time::sleep(Duration::from_millis(100)).await; + shutdown.cancel(); + }); + assert_eq!( + status, 200, + "an in-flight h3 request must complete through the graceful drain" + ); + assert_eq!(body, b"drained-clean"); + shutdown.cancel(); + } + + #[tokio::test] + async fn an_h3_connection_survives_repeated_client_path_migration() { + let app = Router::new().route("/echo", get(|| async { "migrated-clean" })); + let (addr, shutdown) = + spawn_h3_server(app, tls::test_support::limits(), EarlyDataPolicy::Disabled); + + let client = client_endpoint(); + let send_request = h3_client_connect(&client, addr).await; + futures::stream::iter(0u8..3) + .fold( + (client, send_request), + |(client, mut send_request), round| async move { + client + .rebind(std::net::UdpSocket::bind("[::1]:0").unwrap()) + .unwrap(); + let (status, body) = h3_request(&mut send_request, "/echo").await; + assert_eq!( + status, 200, + "round {round}: a request after path migration must still be served" + ); + assert_eq!( + body, b"migrated-clean", + "round {round}: the migrated connection must deliver the response uncorrupted" + ); + (client, send_request) + }, + ) + .await; + + shutdown.cancel(); + } + + #[tokio::test] + async fn a_slow_h3_response_survives_the_idle_timeout() { + use std::num::{NonZeroU32, NonZeroU64}; + + let limits = ListenLimits::new( + crate::limits::HeaderTimeout::from_millis(NonZeroU64::new(1_000).unwrap()), + crate::limits::IdleTimeout::from_millis(NonZeroU64::new(2_000).unwrap()), + NonZeroU32::new(64).unwrap(), + ); + let app = Router::new().route( + "/", + get(|| async { + tokio::time::sleep(Duration::from_secs(3)).await; + "ok" + }), + ); + let (addr, shutdown) = spawn_h3_server(app, limits, EarlyDataPolicy::Disabled); + + let client = client_endpoint(); + let mut send_request = h3_client_connect(&client, addr).await; + let (status, body) = h3_request(&mut send_request, "/").await; + assert_eq!( + status, 200, + "keep-alive must hold the connection through a response slower than the idle timeout" + ); + assert_eq!(body, b"ok"); + shutdown.cancel(); + } +} diff --git a/knot2/crates/knot-edge/src/robustness.rs b/knot2/crates/knot-edge/src/robustness.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-edge/src/robustness.rs @@ -0,0 +1,562 @@ +use std::net::{IpAddr, SocketAddr}; +use std::num::{NonZeroU32, NonZeroU64}; +use std::sync::Arc; +use std::time::Duration; + +use axum::Router; +use axum::body::Body; +use axum::error_handling::HandleErrorLayer; +use axum::extract::{ConnectInfo, State}; +use axum::middleware::{Next, from_fn_with_state}; +use axum::response::{IntoResponse, Response}; +use governor::middleware::NoOpMiddleware; +use http::{HeaderName, Method, Request, StatusCode}; +use tokio_util::sync::CancellationToken; +use tower::limit::GlobalConcurrencyLimitLayer; +use tower::load_shed::LoadShedLayer; +use tower::{BoxError, ServiceBuilder}; +use tower_governor::GovernorLayer; +use tower_governor::errors::GovernorError; +use tower_governor::governor::{GovernorConfig, GovernorConfigBuilder}; +use tower_governor::key_extractor::KeyExtractor; +use tower_http::map_request_body::MapRequestBodyLayer; +use tower_http::timeout::{RequestBodyTimeoutLayer, TimeoutBody}; + +const NANOS_PER_SECOND: u64 = 1_000_000_000; +const CLEANUP_INTERVAL: Duration = Duration::from_secs(60); + +#[derive(Debug, Clone, Copy)] +pub struct RequestsPerSecond(NonZeroU32); + +impl RequestsPerSecond { + pub fn new(value: NonZeroU32) -> Self { + Self(value) + } + + fn period(self) -> Duration { + Duration::from_nanos((NANOS_PER_SECOND / u64::from(self.0.get())).max(1)) + } +} + +knot_types::scalar_newtype! { + pub struct BurstSize(NonZeroU32); + pub struct MaxInflightRequests(NonZeroU32); +} + +#[derive(Debug, Clone, Copy)] +pub struct RequestTimeout(Duration); + +impl RequestTimeout { + pub fn from_millis(millis: NonZeroU64) -> Self { + Self(Duration::from_millis(millis.get())) + } +} + +#[derive(Debug, Clone, Copy)] +pub struct BodyInactivityTimeout(Duration); + +impl BodyInactivityTimeout { + pub fn from_millis(millis: NonZeroU64) -> Self { + Self(Duration::from_millis(millis.get())) + } +} + +#[derive(Debug, Clone, Copy)] +pub struct WriteRequestTimeout(Duration); + +impl WriteRequestTimeout { + pub fn from_millis(millis: NonZeroU64) -> Self { + Self(Duration::from_millis(millis.get())) + } +} + +pub struct EdgeGuards { + rate: RequestsPerSecond, + burst: BurstSize, + max_inflight: MaxInflightRequests, + request_timeout: RequestTimeout, + body_timeout: BodyInactivityTimeout, + write_request_timeout: WriteRequestTimeout, + proxy_header: Option, +} + +impl EdgeGuards { + pub fn new( + rate: RequestsPerSecond, + burst: BurstSize, + max_inflight: MaxInflightRequests, + request_timeout: RequestTimeout, + body_timeout: BodyInactivityTimeout, + write_request_timeout: WriteRequestTimeout, + proxy_header: Option, + ) -> Self { + Self { + rate, + burst, + max_inflight, + request_timeout, + body_timeout, + write_request_timeout, + proxy_header, + } + } + + pub(crate) fn prepare(self, shutdown: &CancellationToken) -> GuardLayers { + let governor = build_governor(self.rate, self.burst, self.proxy_header); + spawn_state_cleanup(Arc::clone(&governor), shutdown.clone()); + GuardLayers { + governor, + max_inflight: self.max_inflight.0.get() as usize, + body_timeout: self.body_timeout.0, + timeout: TimeoutBudget { + standard: self.request_timeout.0, + extended: self.write_request_timeout.0, + }, + } + } +} + +type GuardGovernor = GovernorConfig; + +pub(crate) struct GuardLayers { + governor: Arc, + max_inflight: usize, + body_timeout: Duration, + timeout: TimeoutBudget, +} + +#[derive(Clone, Copy)] +struct TimeoutBudget { + standard: Duration, + extended: Duration, +} + +#[derive(Clone)] +struct ProxyAwareIp { + header: Option, +} + +impl KeyExtractor for ProxyAwareIp { + type Key = IpAddr; + + fn extract(&self, request: &Request) -> Result { + let from_header = self + .header + .as_ref() + .and_then(|header| knot_types::forwarded_peer(request.headers(), header)); + from_header + .or_else(|| { + request + .extensions() + .get::>() + .map(|info| info.0.ip()) + }) + .ok_or(GovernorError::UnableToExtractKey) + } +} + +fn build_governor( + rate: RequestsPerSecond, + burst: BurstSize, + proxy_header: Option, +) -> Arc { + let mut builder = GovernorConfigBuilder::default(); + builder.period(rate.period()).burst_size(burst.0.get()); + let config = builder + .key_extractor(ProxyAwareIp { + header: proxy_header, + }) + .finish() + .expect("a non-zero rate period and burst size always yield a governor config"); + Arc::new(config) +} + +fn spawn_state_cleanup(governor: Arc, shutdown: CancellationToken) { + tokio::spawn(async move { + let mut ticker = tokio::time::interval(CLEANUP_INTERVAL); + ticker.tick().await; + loop { + tokio::select! { + () = shutdown.cancelled() => break, + _ = ticker.tick() => governor.limiter().retain_recent(), + } + } + }); +} + +async fn shed_overloaded(_error: BoxError) -> StatusCode { + StatusCode::SERVICE_UNAVAILABLE +} + +async fn apply_request_timeout( + State(budget): State, + request: Request, + next: Next, +) -> Response { + let limit = match is_streaming_write(&request) { + true => budget.extended, + false => budget.standard, + }; + match tokio::time::timeout(limit, next.run(request)).await { + Ok(response) => response, + Err(_) => StatusCode::REQUEST_TIMEOUT.into_response(), + } +} + +fn is_streaming_write(request: &Request) -> bool { + let path = request.uri().path(); + match *request.method() { + Method::POST => path.ends_with("/git-receive-pack"), + Method::PUT => path.contains("/info/lfs/objects/"), + _ => false, + } +} + +fn rewrap_body(body: TimeoutBody) -> axum::body::Body { + axum::body::Body::new(body) +} + +pub(crate) fn apply(router: Router, layers: GuardLayers) -> Router { + let GuardLayers { + governor, + max_inflight, + body_timeout, + timeout, + } = layers; + let rate_limit: GovernorLayer = + GovernorLayer::new(governor); + router + .layer( + ServiceBuilder::new() + .layer(RequestBodyTimeoutLayer::new(body_timeout)) + .layer(MapRequestBodyLayer::new(rewrap_body)), + ) + .layer(from_fn_with_state(timeout, apply_request_timeout)) + .layer( + ServiceBuilder::new() + .layer(HandleErrorLayer::new(shed_overloaded)) + .layer(LoadShedLayer::new()) + .layer(GlobalConcurrencyLimitLayer::new(max_inflight)), + ) + .layer(rate_limit) +} + +#[cfg(test)] +mod tests { + use super::*; + + use axum::body::{Body, Bytes}; + use axum::routing::{get, post}; + use futures::StreamExt; + use http::StatusCode; + use tower::ServiceExt; + + fn guards( + rate: u32, + burst: u32, + inflight: u32, + request_timeout_ms: u64, + body_timeout_ms: u64, + proxy_header: Option<&str>, + ) -> EdgeGuards { + guards_with_write( + rate, + burst, + inflight, + request_timeout_ms, + body_timeout_ms, + request_timeout_ms, + proxy_header, + ) + } + + #[allow(clippy::too_many_arguments)] + fn guards_with_write( + rate: u32, + burst: u32, + inflight: u32, + request_timeout_ms: u64, + body_timeout_ms: u64, + write_request_timeout_ms: u64, + proxy_header: Option<&str>, + ) -> EdgeGuards { + EdgeGuards::new( + RequestsPerSecond::new(NonZeroU32::new(rate).unwrap()), + BurstSize::new(NonZeroU32::new(burst).unwrap()), + MaxInflightRequests::new(NonZeroU32::new(inflight).unwrap()), + RequestTimeout::from_millis(NonZeroU64::new(request_timeout_ms).unwrap()), + BodyInactivityTimeout::from_millis(NonZeroU64::new(body_timeout_ms).unwrap()), + WriteRequestTimeout::from_millis(NonZeroU64::new(write_request_timeout_ms).unwrap()), + proxy_header.map(|header| HeaderName::from_bytes(header.as_bytes()).unwrap()), + ) + } + + fn guarded_router(router: Router, guards: EdgeGuards) -> Router { + apply(router, guards.prepare(&CancellationToken::new())) + } + + fn from_peer(request: Request, host: u8) -> Request { + let mut request = request; + request + .extensions_mut() + .insert(ConnectInfo(SocketAddr::from(([127, 0, 0, host], 47000)))); + request + } + + fn get_request() -> Request { + Request::get("/").body(Body::empty()).unwrap() + } + + #[test] + fn the_extractor_keys_on_the_trusted_proxy_header_when_configured() { + let extractor = ProxyAwareIp { + header: Some(HeaderName::from_static("x-forwarded-for")), + }; + let request = Request::get("/") + .header("x-forwarded-for", "203.0.113.7, 198.51.100.4") + .extension(ConnectInfo(SocketAddr::from(([127, 0, 0, 1], 5000)))) + .body(()) + .unwrap(); + assert_eq!( + extractor.extract(&request).unwrap(), + "198.51.100.4".parse::().unwrap(), + "the rightmost forwarded entry is the client the proxy appended" + ); + } + + #[test] + fn the_extractor_ignores_a_forgeable_header_when_no_proxy_is_trusted() { + let extractor = ProxyAwareIp { header: None }; + let request = Request::get("/") + .header("x-forwarded-for", "203.0.113.7") + .extension(ConnectInfo(SocketAddr::from(([10, 0, 0, 9], 5000)))) + .body(()) + .unwrap(); + assert_eq!( + extractor.extract(&request).unwrap(), + "10.0.0.9".parse::().unwrap(), + "with no trusted proxy the socket peer wins over a client-forgeable header" + ); + } + + #[test] + fn the_extractor_falls_back_to_the_peer_when_the_trusted_header_is_absent() { + let extractor = ProxyAwareIp { + header: Some(HeaderName::from_static("x-forwarded-for")), + }; + let request = Request::get("/") + .extension(ConnectInfo(SocketAddr::from(([10, 0, 0, 9], 5000)))) + .body(()) + .unwrap(); + assert_eq!( + extractor.extract(&request).unwrap(), + "10.0.0.9".parse::().unwrap() + ); + } + + #[test] + fn the_extractor_fails_when_no_peer_can_be_identified() { + let extractor = ProxyAwareIp { header: None }; + let request = Request::get("/").body(()).unwrap(); + assert!(matches!( + extractor.extract(&request), + Err(GovernorError::UnableToExtractKey) + )); + } + + #[tokio::test] + async fn a_well_behaved_request_passes_every_guard() { + let app = guarded_router( + Router::new().route("/", get(|| async { "ok" })), + guards(50, 200, 1_024, 60_000, 30_000, None), + ); + let status = app + .oneshot(from_peer(get_request(), 1)) + .await + .unwrap() + .status(); + assert_eq!(status, StatusCode::OK); + } + + #[tokio::test] + async fn a_burst_beyond_the_per_ip_limit_is_rejected_with_429() { + let app = guarded_router( + Router::new().route("/", get(|| async { "ok" })), + guards(1, 2, 1_024, 60_000, 30_000, None), + ); + let first = app + .clone() + .oneshot(from_peer(get_request(), 7)) + .await + .unwrap() + .status(); + let second = app + .clone() + .oneshot(from_peer(get_request(), 7)) + .await + .unwrap() + .status(); + let third = app + .clone() + .oneshot(from_peer(get_request(), 7)) + .await + .unwrap() + .status(); + let other_ip = app + .clone() + .oneshot(from_peer(get_request(), 8)) + .await + .unwrap() + .status(); + assert_eq!(first, StatusCode::OK); + assert_eq!(second, StatusCode::OK); + assert_eq!( + third, + StatusCode::TOO_MANY_REQUESTS, + "a third request inside the window exhausts the burst for this IP" + ); + assert_eq!( + other_ip, + StatusCode::OK, + "a different IP keeps its own independent quota" + ); + } + + #[tokio::test] + async fn requests_beyond_the_inflight_limit_are_shed_with_503() { + let app = guarded_router( + Router::new().route( + "/slow", + get(|| async { + tokio::time::sleep(Duration::from_millis(300)).await; + "ok" + }), + ), + guards(10_000, 10_000, 1, 60_000, 30_000, None), + ); + let holder = { + let app = app.clone(); + tokio::spawn(async move { + let request = from_peer(Request::get("/slow").body(Body::empty()).unwrap(), 1); + app.oneshot(request).await.unwrap().status() + }) + }; + tokio::time::sleep(Duration::from_millis(50)).await; + let shed = app + .clone() + .oneshot(from_peer( + Request::get("/slow").body(Body::empty()).unwrap(), + 2, + )) + .await + .unwrap() + .status(); + assert_eq!( + shed, + StatusCode::SERVICE_UNAVAILABLE, + "with the single inflight slot held, the next request sheds rather than queues" + ); + assert_eq!(holder.await.unwrap(), StatusCode::OK); + } + + #[tokio::test] + async fn a_request_slower_than_the_timeout_is_cut_with_408() { + let app = guarded_router( + Router::new().route( + "/slow", + get(|| async { + tokio::time::sleep(Duration::from_millis(500)).await; + "ok" + }), + ), + guards(10_000, 10_000, 1_024, 80, 30_000, None), + ); + let status = app + .oneshot(from_peer( + Request::get("/slow").body(Body::empty()).unwrap(), + 1, + )) + .await + .unwrap() + .status(); + assert_eq!(status, StatusCode::REQUEST_TIMEOUT); + } + + #[tokio::test] + async fn a_streaming_write_runs_under_the_extended_budget_while_reads_keep_the_standard_one() { + let app = guarded_router( + Router::new() + .route( + "/did/name/git-receive-pack", + post(|| async { + tokio::time::sleep(Duration::from_millis(200)).await; + "ok" + }), + ) + .route( + "/did/name/git-upload-pack", + post(|| async { + tokio::time::sleep(Duration::from_millis(200)).await; + "ok" + }), + ), + guards_with_write(10_000, 10_000, 1_024, 80, 30_000, 5_000, None), + ); + let push = app + .clone() + .oneshot(from_peer( + Request::post("/did/name/git-receive-pack") + .body(Body::empty()) + .unwrap(), + 1, + )) + .await + .unwrap() + .status(); + assert_eq!( + push, + StatusCode::OK, + "a push slower than the standard timeout survives on the extended write budget" + ); + let fetch = app + .oneshot(from_peer( + Request::post("/did/name/git-upload-pack") + .body(Body::empty()) + .unwrap(), + 1, + )) + .await + .unwrap() + .status(); + assert_eq!( + fetch, + StatusCode::REQUEST_TIMEOUT, + "a non-write request past the standard timeout is still cut" + ); + } + + #[tokio::test] + async fn a_stalled_request_body_is_cut_and_never_hangs() { + let app = guarded_router( + Router::new().route("/upload", post(|_body: Bytes| async { "ok" })), + guards(10_000, 10_000, 1_024, 60_000, 80, None), + ); + let body = Body::from_stream( + futures::stream::once(async { + Ok::<_, std::io::Error>(Bytes::from_static(b"partial")) + }) + .chain(futures::stream::pending::>()), + ); + let request = from_peer(Request::post("/upload").body(body).unwrap(), 1); + let status = tokio::time::timeout(Duration::from_secs(5), app.oneshot(request)) + .await + .expect("the body inactivity timeout must cut a stalled upload instead of hanging") + .unwrap() + .status(); + assert_ne!( + status, + StatusCode::OK, + "a body that stalls past the inactivity timeout is never accepted" + ); + } +} diff --git a/knot2/crates/knot-edge/src/tcp.rs b/knot2/crates/knot-edge/src/tcp.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-edge/src/tcp.rs @@ -0,0 +1,521 @@ +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; + +use axum::Router; +use axum::body::Body; +use hyper::body::Incoming; +use hyper::service::service_fn; +use hyper_util::rt::{TokioExecutor, TokioIo, TokioTimer}; +use hyper_util::server::conn::auto::Builder; +use rustls::ServerConfig; +use tokio::io::{AsyncRead, AsyncWrite}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::sync::Semaphore; +use tokio_rustls::TlsAcceptor; +use tokio_util::sync::CancellationToken; +use tokio_util::task::TaskTracker; +use tower::{Service, ServiceExt}; + +use crate::limits::ListenLimits; + +const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10); +const CONNECTION_DRAIN_GRACE: Duration = Duration::from_secs(10); +const LISTENER_DRAIN_GRACE: Duration = Duration::from_secs(30); +const ACCEPT_BACKOFF: Duration = Duration::from_millis(250); + +pub async fn serve_plaintext( + listener: TcpListener, + router: Router, + limits: ListenLimits, + shutdown: CancellationToken, +) -> std::io::Result<()> { + run_listener(listener, router, limits, shutdown, |stream| async move { + Some(stream) + }) + .await +} + +pub async fn serve_tls( + listener: TcpListener, + router: Router, + server_config: Arc, + limits: ListenLimits, + shutdown: CancellationToken, +) -> std::io::Result<()> { + let acceptor = TlsAcceptor::from(server_config); + run_listener(listener, router, limits, shutdown, move |stream| { + let acceptor = acceptor.clone(); + async move { + match tokio::time::timeout(HANDSHAKE_TIMEOUT, acceptor.accept(stream)).await { + Ok(Ok(tls_stream)) => Some(tls_stream), + Ok(Err(error)) => { + tracing::debug!("tls handshake failed: {error}"); + None + } + Err(_) => { + tracing::debug!("tls handshake timed out after {HANDSHAKE_TIMEOUT:?}"); + None + } + } + } + }) + .await +} + +async fn run_listener( + listener: TcpListener, + router: Router, + limits: ListenLimits, + shutdown: CancellationToken, + upgrade: Upgrade, +) -> std::io::Result<()> +where + IO: AsyncRead + AsyncWrite + Unpin + Send + 'static, + Upgrade: Fn(TcpStream) -> Fut + Send + Sync + 'static, + Fut: std::future::Future> + Send + 'static, +{ + let slots = Arc::new(Semaphore::new(limits.max_connections())); + let mut make_service = router.into_make_service_with_connect_info::(); + let tracker = TaskTracker::new(); + let upgrade = Arc::new(upgrade); + + loop { + let accepted = tokio::select! { + () = shutdown.cancelled() => break, + accepted = listener.accept() => accepted, + }; + let (stream, peer) = match accepted { + Ok(pair) => pair, + Err(error) if is_connection_error(&error) => continue, + Err(error) => { + tracing::warn!("tcp accept failed, backing off: {error}"); + tokio::select! { + () = shutdown.cancelled() => break, + () = tokio::time::sleep(ACCEPT_BACKOFF) => {} + } + continue; + } + }; + let Ok(slot) = Arc::clone(&slots).try_acquire_owned() else { + continue; + }; + let service = match make_service.call(peer).await { + Ok(service) => service, + Err(never) => match never {}, + }; + let header_timeout = limits.header_timeout().get(); + let conn_shutdown = shutdown.clone(); + let upgrade = Arc::clone(&upgrade); + tracker.spawn(async move { + let _slot = slot; + let Some(io) = upgrade(stream).await else { + return; + }; + let hyper_service = service_fn(move |request: hyper::Request| { + service.clone().oneshot(request.map(Body::new)) + }); + let budget = limits.connection_budget(); + let mut builder = Builder::new(TokioExecutor::new()); + builder + .http1() + .timer(TokioTimer::new()) + .header_read_timeout(header_timeout); + builder + .http2() + .timer(TokioTimer::new()) + .max_concurrent_streams(budget.max_concurrent_streams().get()) + .initial_stream_window_size(budget.stream_receive_window()) + .initial_connection_window_size(budget.connection_receive_window()) + .keep_alive_interval(Some(header_timeout)) + .keep_alive_timeout(header_timeout); + let connection = + builder.serve_connection_with_upgrades(TokioIo::new(io), hyper_service); + tokio::pin!(connection); + + tokio::select! { + served = connection.as_mut() => drop(served), + () = conn_shutdown.cancelled() => { + connection.as_mut().graceful_shutdown(); + let _ = tokio::time::timeout(CONNECTION_DRAIN_GRACE, connection.as_mut()).await; + } + } + }); + } + + tracker.close(); + let _ = tokio::time::timeout(LISTENER_DRAIN_GRACE, tracker.wait()).await; + Ok(()) +} + +fn is_connection_error(error: &std::io::Error) -> bool { + matches!( + error.kind(), + std::io::ErrorKind::ConnectionRefused + | std::io::ErrorKind::ConnectionAborted + | std::io::ErrorKind::ConnectionReset + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + use std::num::{NonZeroU32, NonZeroU64}; + + use axum::routing::get; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpStream; + + fn limits(header_timeout_ms: u64, max_connections: u32) -> ListenLimits { + ListenLimits::new( + crate::limits::HeaderTimeout::from_millis(NonZeroU64::new(header_timeout_ms).unwrap()), + crate::limits::IdleTimeout::from_millis(NonZeroU64::new(30_000).unwrap()), + NonZeroU32::new(max_connections).unwrap(), + ) + } + + async fn bind_and_serve(limits: ListenLimits) -> (SocketAddr, CancellationToken) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let router = Router::new().route("/", get(|| async { "ok" })); + let shutdown = CancellationToken::new(); + tokio::spawn(serve_plaintext(listener, router, limits, shutdown.clone())); + (addr, shutdown) + } + + async fn read_to_close(stream: &mut TcpStream) -> Vec { + let mut collected = Vec::new(); + stream.read_to_end(&mut collected).await.unwrap(); + collected + } + + #[tokio::test] + async fn a_full_request_is_answered() { + let (addr, _shutdown) = bind_and_serve(limits(5_000, 4)).await; + let mut stream = TcpStream::connect(addr).await.unwrap(); + stream + .write_all(b"GET / HTTP/1.1\r\nhost: oyster.cafe\r\nconnection: close\r\n\r\n") + .await + .unwrap(); + let answer = read_to_close(&mut stream).await; + let head = String::from_utf8_lossy(&answer); + assert!(head.starts_with("HTTP/1.1 200"), "got: {head}"); + } + + #[tokio::test] + async fn a_slowloris_connection_is_cut_at_the_header_timeout() { + let (addr, _shutdown) = bind_and_serve(limits(200, 4)).await; + let mut stream = TcpStream::connect(addr).await.unwrap(); + stream.write_all(b"GET / HTT").await.unwrap(); + let closed = tokio::time::timeout(Duration::from_secs(5), read_to_close(&mut stream)) + .await + .expect("server cuts connection instead of waiting forever"); + let head = String::from_utf8_lossy(&closed); + assert!( + !head.contains("200"), + "half-sent request must never be answered, got: {head}" + ); + } + + #[tokio::test] + async fn an_idle_keep_alive_connection_is_cut_at_the_header_timeout() { + let (addr, _shutdown) = bind_and_serve(limits(200, 4)).await; + let mut stream = TcpStream::connect(addr).await.unwrap(); + stream + .write_all(b"GET / HTTP/1.1\r\nhost: oyster.cafe\r\n\r\n") + .await + .unwrap(); + let answer = tokio::time::timeout(Duration::from_secs(5), read_to_close(&mut stream)) + .await + .expect("idle keep-alive connection is cut after answered request"); + let head = String::from_utf8_lossy(&answer); + assert!(head.starts_with("HTTP/1.1 200"), "got: {head}"); + } + + #[tokio::test] + async fn a_connection_beyond_the_limit_is_refused() { + let (addr, _shutdown) = bind_and_serve(limits(5_000, 1)).await; + let mut held = TcpStream::connect(addr).await.unwrap(); + held.write_all(b"GET / HTT").await.unwrap(); + tokio::time::sleep(Duration::from_millis(100)).await; + + let mut refused = TcpStream::connect(addr).await.unwrap(); + refused.write_all(b"GET / HTT").await.unwrap(); + let mut answer = Vec::new(); + let outcome = + tokio::time::timeout(Duration::from_secs(1), refused.read_to_end(&mut answer)) + .await + .expect("over-limit connection is dropped at accept instead of held to timeout"); + match outcome { + Ok(_) => assert!( + answer.is_empty(), + "over-limit connection gets no bytes, got: {}", + String::from_utf8_lossy(&answer) + ), + Err(reset) => assert_eq!(reset.kind(), std::io::ErrorKind::ConnectionReset), + } + } + + #[tokio::test] + async fn an_in_flight_request_finishes_during_drain() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let router = Router::new().route( + "/slow", + get(|| async { + tokio::time::sleep(Duration::from_millis(300)).await; + "drained-clean" + }), + ); + let shutdown = CancellationToken::new(); + tokio::spawn(serve_plaintext( + listener, + router, + limits(5_000, 4), + shutdown.clone(), + )); + + let mut stream = TcpStream::connect(addr).await.unwrap(); + stream + .write_all(b"GET /slow HTTP/1.1\r\nhost: oyster.cafe\r\nconnection: close\r\n\r\n") + .await + .unwrap(); + + tokio::time::sleep(Duration::from_millis(100)).await; + shutdown.cancel(); + + let answer = tokio::time::timeout(Duration::from_secs(5), read_to_close(&mut stream)) + .await + .expect("an in-flight request is answered through the graceful drain"); + let text = String::from_utf8_lossy(&answer); + assert!(text.starts_with("HTTP/1.1 200"), "got: {text}"); + assert!( + text.trim_end().ends_with("drained-clean"), + "the in-flight response must complete during drain, got: {text}" + ); + } + + #[tokio::test] + async fn cancelling_the_token_stops_the_listener() { + let (addr, shutdown) = bind_and_serve(limits(5_000, 4)).await; + let mut stream = TcpStream::connect(addr).await.unwrap(); + stream + .write_all(b"GET / HTTP/1.1\r\nhost: oyster.cafe\r\nconnection: close\r\n\r\n") + .await + .unwrap(); + let _ = read_to_close(&mut stream).await; + + shutdown.cancel(); + tokio::time::sleep(Duration::from_millis(100)).await; + let refused = TcpStream::connect(addr).await; + if let Ok(mut late) = refused { + late.write_all(b"GET / HTTP/1.1\r\nhost: oyster.cafe\r\nconnection: close\r\n\r\n") + .await + .ok(); + let mut answer = Vec::new(); + let _ = + tokio::time::timeout(Duration::from_secs(1), late.read_to_end(&mut answer)).await; + assert!( + !String::from_utf8_lossy(&answer).contains("200"), + "a drained listener mustn't answer new requests" + ); + } + } +} + +#[cfg(test)] +mod tls_tests { + use super::*; + + use std::num::{NonZeroU32, NonZeroU64}; + + use axum::routing::get; + use futures::StreamExt; + use rustls::NamedGroup; + use rustls::crypto::aws_lc_rs; + use rustls::pki_types::ServerName; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpStream; + use tokio_rustls::TlsConnector; + + use crate::tls; + + fn client(alpn: &[&[u8]]) -> TlsConnector { + client_with_provider(aws_lc_rs::default_provider(), alpn) + } + + fn client_with_provider( + provider: rustls::crypto::CryptoProvider, + alpn: &[&[u8]], + ) -> TlsConnector { + let mut config = rustls::ClientConfig::builder_with_provider(Arc::new(provider)) + .with_safe_default_protocol_versions() + .unwrap() + .dangerous() + .with_custom_certificate_verifier(Arc::new(tls::test_support::AcceptAnyServerCert)) + .with_no_client_auth(); + config.alpn_protocols = alpn.iter().map(|p| p.to_vec()).collect(); + TlsConnector::from(Arc::new(config)) + } + + async fn spawn_tls_server( + resolver: Arc, + limits: ListenLimits, + ) -> (SocketAddr, CancellationToken) { + let server_config = Arc::new(tls::build_tls_server_config(resolver, &[]).unwrap()); + let listener = TcpListener::bind("[::1]:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let router = Router::new().route("/", get(|| async { "ok" })); + let shutdown = CancellationToken::new(); + tokio::spawn(serve_tls( + listener, + router, + server_config, + limits, + shutdown.clone(), + )); + (addr, shutdown) + } + + async fn serve( + alpn_offer: &[&[u8]], + ) -> ( + SocketAddr, + CancellationToken, + tokio_rustls::client::TlsStream, + ) { + let (addr, shutdown) = + spawn_tls_server(tls::test_support::resolver(), tls::test_support::limits()).await; + let tcp = TcpStream::connect(addr).await.unwrap(); + let name = ServerName::try_from("localhost").unwrap(); + let tls = client(alpn_offer).connect(name, tcp).await.unwrap(); + (addr, shutdown, tls) + } + + #[tokio::test] + async fn it_terminates_tls_over_http1_and_negotiates_post_quantum() { + let (_addr, shutdown, mut tls) = serve(&[b"http/1.1"]).await; + + let group = tls.get_ref().1.negotiated_key_exchange_group().unwrap(); + assert_eq!( + group.name(), + NamedGroup::X25519MLKEM768, + "prefer-post-quantum must select X25519MLKEM768 with a capable client" + ); + assert_eq!( + tls.get_ref().1.alpn_protocol(), + Some(b"http/1.1".as_slice()) + ); + + tls.write_all(b"GET / HTTP/1.1\r\nhost: localhost\r\nconnection: close\r\n\r\n") + .await + .unwrap(); + let mut response = Vec::new(); + tls.read_to_end(&mut response).await.unwrap(); + let text = String::from_utf8_lossy(&response); + assert!(text.starts_with("HTTP/1.1 200"), "got: {text}"); + assert!(text.trim_end().ends_with("ok"), "got: {text}"); + + shutdown.cancel(); + } + + #[tokio::test] + async fn it_negotiates_h2_when_the_client_offers_only_h2() { + let (_addr, shutdown, tls) = serve(&[b"h2"]).await; + assert_eq!(tls.get_ref().1.alpn_protocol(), Some(b"h2".as_slice())); + shutdown.cancel(); + } + + #[tokio::test] + async fn a_classical_only_client_completes_over_x25519() { + let (addr, shutdown) = + spawn_tls_server(tls::test_support::resolver(), tls::test_support::limits()).await; + + let tcp = TcpStream::connect(addr).await.unwrap(); + let name = ServerName::try_from("localhost").unwrap(); + let tls = + client_with_provider(tls::test_support::classical_only_provider(), &[b"http/1.1"]) + .connect(name, tcp) + .await + .unwrap(); + + let group = tls.get_ref().1.negotiated_key_exchange_group().unwrap(); + assert_eq!( + group.name(), + NamedGroup::X25519, + "a client without ML-KEM must still complete the handshake over classical X25519" + ); + + shutdown.cancel(); + } + + #[tokio::test(flavor = "multi_thread")] + async fn concurrent_handshakes_survive_a_cert_reload_under_load() { + let high_limit = ListenLimits::new( + crate::limits::HeaderTimeout::from_millis(NonZeroU64::new(5_000).unwrap()), + crate::limits::IdleTimeout::from_millis(NonZeroU64::new(30_000).unwrap()), + NonZeroU32::new(512).unwrap(), + ); + let resolver = tls::test_support::resolver(); + let (addr, shutdown) = spawn_tls_server(resolver.clone(), high_limit).await; + + let churn = { + let resolver = resolver.clone(); + let stop = shutdown.clone(); + tokio::spawn(async move { + futures::stream::unfold(0u32, move |swaps| { + let resolver = resolver.clone(); + let stop = stop.clone(); + async move { + match stop.is_cancelled() { + true => None, + false => { + resolver.store(tls::test_support::self_signed()); + tokio::time::sleep(Duration::from_millis(1)).await; + Some((swaps + 1, swaps + 1)) + } + } + } + }) + .fold(0u32, |_, swaps| async move { swaps }) + .await + }) + }; + + let clients: Vec<_> = (0..48) + .map(|_| { + tokio::spawn(async move { + let tcp = TcpStream::connect(addr).await.unwrap(); + let name = ServerName::try_from("localhost").unwrap(); + let mut tls = client(&[b"http/1.1"]).connect(name, tcp).await.unwrap(); + tls.write_all( + b"GET / HTTP/1.1\r\nhost: localhost\r\nconnection: close\r\n\r\n", + ) + .await + .unwrap(); + let mut response = Vec::new(); + tls.read_to_end(&mut response).await.unwrap(); + let text = String::from_utf8_lossy(&response).into_owned(); + text.starts_with("HTTP/1.1 200") && text.trim_end().ends_with("ok") + }) + }) + .collect(); + + let outcomes: Vec = futures::future::join_all(clients) + .await + .into_iter() + .map(|joined| joined.unwrap()) + .collect(); + assert!( + outcomes.iter().all(|served| *served), + "every handshake racing a cert reload must complete and serve the router uncorrupted" + ); + + shutdown.cancel(); + assert!( + churn.await.unwrap() > 1, + "the resolver must have reloaded the certificate during the load" + ); + } +} diff --git a/knot2/crates/knot-edge/src/tls.rs b/knot2/crates/knot-edge/src/tls.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-edge/src/tls.rs @@ -0,0 +1,679 @@ +use std::io::BufReader; +use std::path::Path; +use std::sync::Arc; + +use arc_swap::ArcSwap; +use base64::Engine; +use quinn::crypto::rustls::QuicServerConfig; +use rustls::RootCertStore; +use rustls::ServerConfig; +use rustls::crypto::aws_lc_rs; +use rustls::pki_types::{CertificateDer, PrivateKeyDer, UnixTime}; +use rustls::server::danger::{ClientCertVerified, ClientCertVerifier}; +use rustls::server::{ClientHello, ResolvesServerCert, WebPkiClientVerifier}; +use rustls::sign::CertifiedKey; +use sha2::{Digest, Sha256}; +use tokio_util::sync::CancellationToken; + +use crate::limits::ListenLimits; +use crate::zerortt::EarlyDataPolicy; + +pub const ACME_TLS_ALPN: &[u8] = rustls_acme::acme::ACME_TLS_ALPN_NAME; + +#[derive(Debug, thiserror::Error)] +pub enum TlsError { + #[error("reading {path}: {source}")] + Read { + path: String, + source: std::io::Error, + }, + #[error("parsing {path}: {message}")] + Parse { path: String, message: String }, + #[error("no certificates found in {0}")] + NoCertificates(String), + #[error("no private key found in {0}")] + NoPrivateKey(String), + #[error("unusable private key: {0}")] + SigningKey(String), + #[error("building server config: {0}")] + Config(String), + #[error("certificate and private key don't match: {0}")] + KeyMismatch(String), + #[error("session ticketer: {0}")] + Ticketer(String), + #[error("client certificate verifier for {path}: {message}")] + ClientVerifier { path: String, message: String }, + #[error("admin SPKI pin: {0}")] + SpkiPin(String), +} + +#[derive(Clone)] +pub struct SpkiPin([u8; 32]); + +impl PartialEq for SpkiPin { + fn eq(&self, other: &Self) -> bool { + self.0 + .iter() + .zip(other.0.iter()) + .fold(0u8, |acc, (left, right)| acc | (left ^ right)) + == 0 + } +} + +impl Eq for SpkiPin {} + +impl SpkiPin { + pub fn from_base64(encoded: &str) -> Result { + let bytes = base64::engine::general_purpose::STANDARD + .decode(encoded.trim()) + .map_err(|error| TlsError::SpkiPin(error.to_string()))?; + let array: [u8; 32] = bytes.try_into().map_err(|bytes: Vec| { + TlsError::SpkiPin(format!("expected 32 bytes, got {}", bytes.len())) + })?; + Ok(Self(array)) + } + + fn of_certificate(cert: &CertificateDer<'_>) -> Result { + let (_, parsed) = x509_parser::parse_x509_certificate(cert.as_ref()).map_err(|error| { + rustls::Error::General(format!("parse client certificate: {error}")) + })?; + Ok(Self( + Sha256::digest(parsed.tbs_certificate.subject_pki.raw).into(), + )) + } +} + +impl std::fmt::Debug for SpkiPin { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_tuple("SpkiPin").finish_non_exhaustive() + } +} + +pub(crate) fn fuzz_of_certificate(data: &[u8]) { + let cert = CertificateDer::from(data.to_vec()); + let _ = SpkiPin::of_certificate(&cert); +} + +pub struct ReloadableCertResolver { + current: ArcSwap, +} + +impl std::fmt::Debug for ReloadableCertResolver { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ReloadableCertResolver") + .finish_non_exhaustive() + } +} + +impl ReloadableCertResolver { + pub fn new(initial: CertifiedKey) -> Self { + Self { + current: ArcSwap::from_pointee(initial), + } + } + + pub fn store(&self, key: CertifiedKey) { + self.current.store(Arc::new(key)); + } +} + +impl ResolvesServerCert for ReloadableCertResolver { + fn resolve(&self, _client_hello: ClientHello<'_>) -> Option> { + Some(self.current.load_full()) + } +} + +pub fn spawn_cert_reload( + resolver: Arc, + paths: crate::StaticCertPaths, + shutdown: CancellationToken, +) { + #[cfg(unix)] + tokio::spawn(async move { + use tokio::signal::unix::{SignalKind, signal}; + let mut hangup = match signal(SignalKind::hangup()) { + Ok(stream) => stream, + Err(error) => { + tracing::error!("install SIGHUP handler: {error}"); + return; + } + }; + loop { + tokio::select! { + () = shutdown.cancelled() => break, + received = hangup.recv() => { + if received.is_none() { + break; + } + match load_certified_key(&paths) { + Ok(key) => { + resolver.store(key); + tracing::info!("reloaded TLS certificate on SIGHUP"); + } + Err(error) => { + tracing::warn!("SIGHUP reload kept the existing certificate: {error}"); + } + } + } + } + } + }); + + #[cfg(not(unix))] + let _ = (resolver, paths, shutdown); +} + +pub fn load_certified_key(paths: &crate::StaticCertPaths) -> Result { + let certs = load_certs(paths.cert_path.as_path())?; + let key = load_private_key(paths.key_path.as_path())?; + let signing_key = aws_lc_rs::sign::any_supported_type(&key) + .map_err(|error| TlsError::SigningKey(error.to_string()))?; + let certified = CertifiedKey::new(certs, signing_key); + certified + .keys_match() + .map_err(|error| TlsError::KeyMismatch(error.to_string()))?; + Ok(certified) +} + +fn load_certs(path: &Path) -> Result>, TlsError> { + let bytes = std::fs::read(path).map_err(|source| TlsError::Read { + path: path.display().to_string(), + source, + })?; + let mut reader = BufReader::new(bytes.as_slice()); + let certs = rustls_pemfile::certs(&mut reader) + .collect::, _>>() + .map_err(|error| TlsError::Parse { + path: path.display().to_string(), + message: error.to_string(), + })?; + match certs.is_empty() { + true => Err(TlsError::NoCertificates(path.display().to_string())), + false => Ok(certs), + } +} + +fn load_private_key(path: &Path) -> Result, TlsError> { + let bytes = std::fs::read(path).map_err(|source| TlsError::Read { + path: path.display().to_string(), + source, + })?; + let mut reader = BufReader::new(bytes.as_slice()); + rustls_pemfile::private_key(&mut reader) + .map_err(|error| TlsError::Parse { + path: path.display().to_string(), + message: error.to_string(), + })? + .ok_or_else(|| TlsError::NoPrivateKey(path.display().to_string())) +} + +fn ticketer() -> Result, TlsError> { + aws_lc_rs::Ticketer::new().map_err(|error| TlsError::Ticketer(error.to_string())) +} + +fn tcp_alpn(extra: &[&[u8]]) -> Vec> { + [b"h2".as_slice(), b"http/1.1".as_slice()] + .into_iter() + .chain(extra.iter().copied()) + .map(<[u8]>::to_vec) + .collect() +} + +pub fn build_tls_server_config( + resolver: Arc, + extra_alpn: &[&[u8]], +) -> Result { + let provider = Arc::new(aws_lc_rs::default_provider()); + let mut config = ServerConfig::builder_with_provider(provider) + .with_safe_default_protocol_versions() + .map_err(|error| TlsError::Config(error.to_string()))? + .with_no_client_auth() + .with_cert_resolver(resolver); + config.alpn_protocols = tcp_alpn(extra_alpn); + config.ticketer = ticketer()?; + Ok(config) +} + +pub fn build_mtls_server_config( + resolver: Arc, + client_ca: &crate::ClientCaPath, + pin: SpkiPin, +) -> Result { + let provider = Arc::new(aws_lc_rs::default_provider()); + let roots = load_client_ca(client_ca.as_path())?; + let webpki = + WebPkiClientVerifier::builder_with_provider(Arc::new(roots), Arc::clone(&provider)) + .build() + .map_err(|error| TlsError::ClientVerifier { + path: client_ca.as_path().display().to_string(), + message: error.to_string(), + })?; + let verifier = Arc::new(PinnedClientVerifier { inner: webpki, pin }); + let mut config = ServerConfig::builder_with_provider(provider) + .with_safe_default_protocol_versions() + .map_err(|error| TlsError::Config(error.to_string()))? + .with_client_cert_verifier(verifier) + .with_cert_resolver(resolver); + config.alpn_protocols = tcp_alpn(&[]); + config.ticketer = ticketer()?; + Ok(config) +} + +fn load_client_ca(path: &Path) -> Result { + let certs = load_certs(path)?; + let mut roots = RootCertStore::empty(); + let (added, _) = roots.add_parsable_certificates(certs); + match added { + 0 => Err(TlsError::NoCertificates(path.display().to_string())), + _ => Ok(roots), + } +} + +#[derive(Debug)] +struct PinnedClientVerifier { + inner: Arc, + pin: SpkiPin, +} + +impl ClientCertVerifier for PinnedClientVerifier { + fn root_hint_subjects(&self) -> &[rustls::DistinguishedName] { + self.inner.root_hint_subjects() + } + + fn offer_client_auth(&self) -> bool { + self.inner.offer_client_auth() + } + + fn client_auth_mandatory(&self) -> bool { + self.inner.client_auth_mandatory() + } + + fn verify_client_cert( + &self, + end_entity: &CertificateDer<'_>, + intermediates: &[CertificateDer<'_>], + now: UnixTime, + ) -> Result { + let verified = self + .inner + .verify_client_cert(end_entity, intermediates, now)?; + match SpkiPin::of_certificate(end_entity)? == self.pin { + true => Ok(verified), + false => Err(rustls::Error::General( + "client certificate SPKI doesn't match the pinned admin identity".to_string(), + )), + } + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + self.inner.verify_tls12_signature(message, cert, dss) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + self.inner.verify_tls13_signature(message, cert, dss) + } + + fn supported_verify_schemes(&self) -> Vec { + self.inner.supported_verify_schemes() + } +} + +pub fn build_quic_server_config( + resolver: Arc, + limits: ListenLimits, + early_data: EarlyDataPolicy, +) -> Result { + let provider = Arc::new(aws_lc_rs::default_provider()); + let mut crypto = ServerConfig::builder_with_provider(provider) + .with_protocol_versions(&[&rustls::version::TLS13]) + .map_err(|error| TlsError::Config(error.to_string()))? + .with_no_client_auth() + .with_cert_resolver(resolver); + crypto.alpn_protocols = vec![b"h3".to_vec()]; + crypto.max_early_data_size = early_data.max_early_data_size(); + + let quic_crypto = + QuicServerConfig::try_from(crypto).map_err(|error| TlsError::Config(error.to_string()))?; + let mut config = quinn::ServerConfig::with_crypto(Arc::new(quic_crypto)); + + let budget = limits.connection_budget(); + let mut transport = quinn::TransportConfig::default(); + transport.max_concurrent_bidi_streams(quinn::VarInt::from_u32( + budget.max_concurrent_streams().get(), + )); + transport.stream_receive_window(quinn::VarInt::from_u32(budget.stream_receive_window())); + transport.receive_window(quinn::VarInt::from_u32(budget.connection_receive_window())); + let idle = limits.idle_timeout().get(); + transport.max_idle_timeout(Some( + quinn::IdleTimeout::try_from(idle).map_err(|error| TlsError::Config(error.to_string()))?, + )); + transport.keep_alive_interval(Some(idle / 2)); + config.transport_config(Arc::new(transport)); + Ok(config) +} + +#[cfg(test)] +pub(crate) mod test_support { + use std::num::{NonZeroU32, NonZeroU64}; + + use rustls::pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer}; + use rustls::sign::CertifiedKey; + + use super::*; + use crate::limits::ListenLimits; + + pub(crate) fn self_signed() -> CertifiedKey { + let generated = rcgen::generate_simple_self_signed(vec!["localhost".to_string()]).unwrap(); + let cert_der = generated.cert.der().clone(); + let key_der = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from( + generated.signing_key.serialize_der(), + )); + let signing_key = aws_lc_rs::sign::any_supported_type(&key_der).unwrap(); + CertifiedKey::new(vec![cert_der], signing_key) + } + + pub(crate) fn resolver() -> Arc { + Arc::new(ReloadableCertResolver::new(self_signed())) + } + + pub(crate) fn limits() -> ListenLimits { + ListenLimits::new( + crate::limits::HeaderTimeout::from_millis(NonZeroU64::new(5_000).unwrap()), + crate::limits::IdleTimeout::from_millis(NonZeroU64::new(30_000).unwrap()), + NonZeroU32::new(64).unwrap(), + ) + } + + pub(crate) fn classical_only_provider() -> rustls::crypto::CryptoProvider { + let mut provider = aws_lc_rs::default_provider(); + provider.kx_groups = vec![aws_lc_rs::kx_group::X25519]; + provider + } + + #[derive(Debug)] + pub(crate) struct AcceptAnyServerCert; + + impl rustls::client::danger::ServerCertVerifier for AcceptAnyServerCert { + fn verify_server_cert( + &self, + _end_entity: &rustls::pki_types::CertificateDer<'_>, + _intermediates: &[rustls::pki_types::CertificateDer<'_>], + _server_name: &rustls::pki_types::ServerName<'_>, + _ocsp_response: &[u8], + _now: rustls::pki_types::UnixTime, + ) -> Result { + Ok(rustls::client::danger::ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &rustls::pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature( + message, + cert, + dss, + &aws_lc_rs::default_provider().signature_verification_algorithms, + ) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &rustls::pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature( + message, + cert, + dss, + &aws_lc_rs::default_provider().signature_verification_algorithms, + ) + } + + fn supported_verify_schemes(&self) -> Vec { + aws_lc_rs::default_provider() + .signature_verification_algorithms + .supported_schemes() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_tcp_alpn_set_offers_h2_and_http1_but_not_h3() { + let config = build_tls_server_config(test_support::resolver(), &[]).unwrap(); + assert_eq!( + config.alpn_protocols, + vec![b"h2".to_vec(), b"http/1.1".to_vec()], + "h3 is QUIC-only and must never appear in the TCP ALPN set" + ); + } + + #[test] + fn the_acme_challenge_alpn_joins_only_when_requested() { + let config = build_tls_server_config(test_support::resolver(), &[ACME_TLS_ALPN]).unwrap(); + assert_eq!( + config.alpn_protocols, + vec![b"h2".to_vec(), b"http/1.1".to_vec(), ACME_TLS_ALPN.to_vec()], + "acme-tls/1 must trail h2 and http/1.1 so normal clients never select it" + ); + } + + #[test] + fn the_tcp_config_installs_an_enabled_session_ticketer() { + let config = build_tls_server_config(test_support::resolver(), &[]).unwrap(); + assert!( + config.ticketer.enabled(), + "session resumption requires an enabled ticketer" + ); + } + + #[test] + fn an_spki_pin_round_trips_through_base64() { + let encoded = base64::engine::general_purpose::STANDARD.encode([9u8; 32]); + assert_eq!(SpkiPin::from_base64(&encoded).unwrap(), SpkiPin([9u8; 32])); + } + + #[test] + fn the_spki_pin_matches_the_standard_openssl_recipe() { + const CERT_PEM: &str = "-----BEGIN CERTIFICATE-----\n\ +MIIBhDCCASugAwIBAgIUSkWE4CZvV8B8z9phedKo1PbahDUwCgYIKoZIzj0EAwIw\n\ +GDEWMBQGA1UEAwwNYW5lbW9uZS5hZG1pbjAeFw0yNjA2MjExOTA5MDdaFw0zNjA2\n\ +MTgxOTA5MDdaMBgxFjAUBgNVBAMMDWFuZW1vbmUuYWRtaW4wWTATBgcqhkjOPQIB\n\ +BggqhkjOPQMBBwNCAASFLKd70MtSGSyI2UjdpQyjaJrvXLofac41nI346wK0lC9G\n\ +PjZH/NKqo1iwQn+UfZB7gotfezWrDmAUz5OgT6Rlo1MwUTAdBgNVHQ4EFgQUis7S\n\ +XEFGpe4gQwWnzX/uzjpt274wHwYDVR0jBBgwFoAUis7SXEFGpe4gQwWnzX/uzjpt\n\ +274wDwYDVR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAgNHADBEAiBhJgE4cMP5/FJw\n\ +imc3fYQxOhm5nO59cfG06+0vuDIV1QIgMsKZjFjsch8rbLRNiJL5+bDmlgO7MD14\n\ +0PAyPOyjb+w=\n\ +-----END CERTIFICATE-----\n"; + const OPENSSL_PIN: &str = "EhmM1HyWzC54br06EDvoAaqt1q1h+je3vVFJTcZ9e1U="; + + let der = rustls_pemfile::certs(&mut CERT_PEM.as_bytes()) + .next() + .unwrap() + .unwrap(); + assert_eq!( + SpkiPin::of_certificate(&der).unwrap(), + SpkiPin::from_base64(OPENSSL_PIN).unwrap(), + "of_certificate must hash the same SubjectPublicKeyInfo bytes as openssl pkey -pubin -outform DER | dgst -sha256" + ); + } + + #[test] + fn an_spki_pin_of_the_wrong_length_is_rejected() { + let encoded = base64::engine::general_purpose::STANDARD.encode([9u8; 16]); + assert!(matches!( + SpkiPin::from_base64(&encoded), + Err(TlsError::SpkiPin(_)) + )); + } + + #[test] + fn the_quic_alpn_set_offers_only_h3() { + let quic = build_quic_server_config( + test_support::resolver(), + test_support::limits(), + EarlyDataPolicy::Disabled, + ); + assert!(quic.is_ok()); + } + + #[test] + fn the_default_provider_prefers_post_quantum_key_exchange() { + use rustls::NamedGroup; + + let provider = aws_lc_rs::default_provider(); + let first = provider.kx_groups.first().expect("a key exchange group"); + assert_eq!( + first.name(), + NamedGroup::X25519MLKEM768, + "prefer-post-quantum must order X25519MLKEM768 first for both the TCP and QUIC configs" + ); + } + + #[test] + fn a_mismatched_certificate_and_key_is_rejected() { + let first = test_support::self_signed(); + let second = test_support::self_signed(); + let mismatched = CertifiedKey::new(first.cert.clone(), second.key.clone()); + assert!(mismatched.keys_match().is_err()); + } + + struct ClientIdentity { + ca_path: crate::ClientCaPath, + chain: Vec>, + key_der: Vec, + pin: SpkiPin, + } + + fn issue_client_identity() -> ClientIdentity { + let ca_key = rcgen::KeyPair::generate().unwrap(); + let mut ca_params = rcgen::CertificateParams::new(Vec::::new()).unwrap(); + ca_params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + let ca_cert = ca_params.self_signed(&ca_key).unwrap(); + let issuer = rcgen::Issuer::new(ca_params, ca_key); + + let client_key = rcgen::KeyPair::generate().unwrap(); + let client_params = rcgen::CertificateParams::new(vec!["admin.knot".to_string()]).unwrap(); + let client_cert = client_params.signed_by(&client_key, &issuer).unwrap(); + let client_der = client_cert.der().clone(); + + let ca_path = std::env::temp_dir().join(format!( + "knot_edge_mtls_ca_{}_{:p}.pem", + std::process::id(), + &client_der as *const _ + )); + std::fs::write(&ca_path, ca_cert.pem()).unwrap(); + + ClientIdentity { + ca_path: crate::ClientCaPath::new(ca_path), + pin: SpkiPin::of_certificate(&client_der).unwrap(), + chain: vec![client_der], + key_der: client_key.serialize_der(), + } + } + + async fn mtls_handshake( + server_config: ServerConfig, + identity: Option<&ClientIdentity>, + ) -> bool { + use rustls::pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer, ServerName}; + use tokio::net::{TcpListener, TcpStream}; + use tokio_rustls::{TlsAcceptor, TlsConnector}; + + let acceptor = TlsAcceptor::from(Arc::new(server_config)); + let listener = TcpListener::bind("[::1]:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let (tcp, _) = listener.accept().await.unwrap(); + acceptor.accept(tcp).await.is_ok() + }); + + let verifier = + rustls::ClientConfig::builder_with_provider(Arc::new(aws_lc_rs::default_provider())) + .with_safe_default_protocol_versions() + .unwrap() + .dangerous() + .with_custom_certificate_verifier(Arc::new(test_support::AcceptAnyServerCert)); + let mut client_config = match identity { + Some(identity) => verifier + .with_client_auth_cert( + identity.chain.clone(), + PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(identity.key_der.clone())), + ) + .unwrap(), + None => verifier.with_no_client_auth(), + }; + client_config.alpn_protocols = vec![b"h2".to_vec()]; + let connector = TlsConnector::from(Arc::new(client_config)); + let tcp = TcpStream::connect(addr).await.unwrap(); + let client_ok = connector + .connect(ServerName::try_from("localhost").unwrap(), tcp) + .await + .is_ok(); + let server_ok = server.await.unwrap(); + client_ok && server_ok + } + + #[tokio::test] + async fn mtls_rejects_a_client_presenting_no_certificate() { + let identity = issue_client_identity(); + let config = build_mtls_server_config( + test_support::resolver(), + &identity.ca_path, + identity.pin.clone(), + ) + .unwrap(); + assert!( + !mtls_handshake(config, None).await, + "the mandatory mTLS verifier must reject a client that presents no certificate" + ); + } + + #[tokio::test] + async fn mtls_admits_the_pinned_admin_certificate() { + let identity = issue_client_identity(); + let config = build_mtls_server_config( + test_support::resolver(), + &identity.ca_path, + identity.pin.clone(), + ) + .unwrap(); + assert!( + mtls_handshake(config, Some(&identity)).await, + "a client presenting the pinned admin certificate must complete the mTLS handshake" + ); + } + + #[tokio::test] + async fn mtls_rejects_a_ca_trusted_client_whose_spki_is_not_pinned() { + let identity = issue_client_identity(); + let config = build_mtls_server_config( + test_support::resolver(), + &identity.ca_path, + SpkiPin([0u8; 32]), + ) + .unwrap(); + assert!( + !mtls_handshake(config, Some(&identity)).await, + "a client trusted by the CA but failing the SPKI pin must be rejected" + ); + } +} diff --git a/knot2/crates/knot-edge/src/zerortt.rs b/knot2/crates/knot-edge/src/zerortt.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-edge/src/zerortt.rs @@ -0,0 +1,194 @@ +use axum::Router; +use axum::extract::Request; +use axum::handler::Handler; +use axum::middleware::{Next, from_fn}; +use axum::response::{IntoResponse, Response}; +use axum::routing::get; +use http::StatusCode; + +use crate::protocol::NegotiatedProtocol; + +pub struct ZeroRttSafe { + handler: H, +} + +impl ZeroRttSafe { + pub fn new(handler: H) -> Self { + Self { handler } + } +} + +pub struct RequiresFullHandshake { + router: Router, +} + +impl RequiresFullHandshake { + pub fn new(router: Router) -> Self { + Self { + router: router.layer(from_fn(reject_early_writes)), + } + } + + pub(crate) fn into_router(self) -> Router { + self.router + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EarlyData { + Yes, + No, +} + +impl EarlyData { + pub fn is_early(self) -> bool { + matches!(self, EarlyData::Yes) + } +} + +const EARLY_DATA_HEADER: &str = "early-data"; + +pub struct ZeroRttRoutes { + router: Router, + count: usize, +} + +impl ZeroRttRoutes { + pub fn new() -> Self { + Self { + router: Router::new(), + count: 0, + } + } + + pub fn get(mut self, path: &str, handler: ZeroRttSafe) -> Self + where + H: Handler, + T: 'static, + { + self.router = self.router.route(path, get(handler.handler)); + self.count += 1; + self + } + + pub fn into_router(self) -> Router { + self.router + } + + pub(crate) fn early_data_policy(&self) -> EarlyDataPolicy { + match self.count { + 0 => EarlyDataPolicy::Disabled, + _ => EarlyDataPolicy::Enabled, + } + } +} + +impl Default for ZeroRttRoutes { + fn default() -> Self { + Self::new() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum EarlyDataPolicy { + Disabled, + Enabled, +} + +impl EarlyDataPolicy { + pub(crate) fn max_early_data_size(self) -> u32 { + match self { + EarlyDataPolicy::Disabled => 0, + // rustls takes 0 or u32::MAX, + // and quinn unwraps errors, quite unfortunate. + // + // The 0 above is hard-off marker, for when no route opted in. + EarlyDataPolicy::Enabled => u32::MAX, + } + } +} + +pub(crate) async fn tag_from_header(mut request: Request, next: Next) -> Response { + if request.extensions().get::().is_none() { + let early = request + .headers() + .get(EARLY_DATA_HEADER) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.trim() == "1"); + request + .extensions_mut() + .insert(if early { EarlyData::Yes } else { EarlyData::No }); + } + next.run(request).await +} + +async fn reject_early_writes(request: Request, next: Next) -> Response { + let early = request + .extensions() + .get::() + .copied() + .unwrap_or(EarlyData::Yes); + if early.is_early() { + let protocol = request + .extensions() + .get::() + .map(|protocol| protocol.as_str()) + .unwrap_or("unknown"); + tracing::debug!( + protocol, + path = %request.uri().path(), + "refused an early-data request to a full-handshake route with 425" + ); + return too_early(); + } + next.run(request).await +} + +fn too_early() -> Response { + ( + StatusCode::TOO_EARLY, + "this route refuses early data and needs a completed handshake", + ) + .into_response() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_empty_safe_set_keeps_early_data_disabled() { + assert_eq!( + ZeroRttRoutes::new().early_data_policy(), + EarlyDataPolicy::Disabled + ); + assert_eq!(EarlyDataPolicy::Disabled.max_early_data_size(), 0); + } + + #[test] + fn a_proven_safe_set_unlocks_a_nonzero_early_data_size() { + let routes = ZeroRttRoutes::new().get("/info/refs", ZeroRttSafe::new(|| async { "ok" })); + assert_eq!(routes.early_data_policy(), EarlyDataPolicy::Enabled); + assert_eq!(EarlyDataPolicy::Enabled.max_early_data_size(), u32::MAX); + } + + #[tokio::test] + async fn a_full_handshake_route_fails_closed_when_the_request_is_unclassified() { + use axum::body::Body; + use axum::routing::post; + use http::{Request, StatusCode}; + use tower::ServiceExt; + + let router = Router::new() + .route("/git-upload-pack", post(|| async { "pack" })) + .layer(from_fn(reject_early_writes)); + let request = Request::post("/git-upload-pack") + .body(Body::empty()) + .unwrap(); + assert_eq!( + router.oneshot(request).await.unwrap().status(), + StatusCode::TOO_EARLY, + "a write whose early-data status was never tagged must fail closed with 425" + ); + } +} diff --git a/knot2/crates/knot-edge/tests/fuzz_smoke.rs b/knot2/crates/knot-edge/tests/fuzz_smoke.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-edge/tests/fuzz_smoke.rs @@ -0,0 +1,11 @@ +use proptest::prelude::*; + +proptest! { + #![proptest_config(ProptestConfig::with_cases(1024))] + + #[test] + fn tls_parsers_never_panic(data in proptest::collection::vec(any::(), 0..4096)) { + knot_edge::fuzz::spki_of_certificate(&data); + knot_edge::fuzz::spki_pin(&data); + } +} diff --git a/knot2/crates/knot-events/src/lib.rs b/knot2/crates/knot-events/src/lib.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-events/src/lib.rs @@ -0,0 +1,1129 @@ +use std::collections::{BTreeSet, HashMap, VecDeque}; +use std::net::IpAddr; +use std::sync::{Arc, Mutex}; + +use serde::ser::SerializeMap; +use serde::{Serialize, Serializer}; +use serde_json::Value; +use tokio::sync::{OwnedSemaphorePermit, Semaphore, watch}; + +use knot_runtime::{Clock, UnixMicros}; +use knot_types::{ + AccountDid, ChangedFiles, Email, LanguageBytes, LanguageName, ObjectFormat, Oid, OwnerDid, + PushOptions, RefName, RefTransition, RepoDid, RepoPath, Tid, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)] +#[serde(transparent)] +pub struct EventCursor(i64); + +impl EventCursor { + pub const START: Self = Self(0); + + pub fn new(nanos: i64) -> Self { + Self(nanos) + } + + pub fn get(self) -> i64 { + self.0 + } + + fn from_unix_micros(micros: UnixMicros) -> Self { + Self((micros.get() as i64).saturating_mul(1_000)) + } +} + +pub trait Publish: Serialize { + const NSID: &'static str; +} + +#[derive(Debug, Clone, Serialize)] +pub struct Event { + pub rkey: Tid, + pub nsid: &'static str, + #[serde(rename = "event")] + pub payload: Value, + pub created: EventCursor, +} + +// `sh.tangled.git.refUpdate` requires ref, oldSha and newSha, +// so a record about no ref at all sends "" for the three rather than `null`. +#[derive(Debug, Clone)] +enum RefChange { + Absent, + Applied { + ref_name: RefName, + old_sha: Oid, + new_sha: Oid, + }, +} + +impl Serialize for RefChange { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(Some(3))?; + match self { + Self::Absent => { + map.serialize_entry("ref", "")?; + map.serialize_entry("oldSha", "")?; + map.serialize_entry("newSha", "")?; + } + Self::Applied { + ref_name, + old_sha, + new_sha, + } => { + map.serialize_entry("ref", ref_name.as_str())?; + map.serialize_entry("oldSha", &old_sha.to_hex())?; + map.serialize_entry("newSha", &new_sha.to_hex())?; + } + } + map.end() + } +} + +#[derive(Debug, Clone, Serialize)] +pub struct GitRefUpdate { + #[serde(rename = "$type")] + record_type: &'static str, + #[serde(rename = "changedFiles", skip_serializing_if = "Vec::is_empty")] + changed_files: Vec, + #[serde(rename = "committerDid")] + committer_did: AccountDid, + meta: Option, + #[serde(rename = "ownerDid", skip_serializing_if = "Option::is_none")] + owner_did: Option, + #[serde(rename = "pushOptions", skip_serializing_if = "PushOptions::is_empty")] + push_options: PushOptions, + #[serde(flatten)] + change: RefChange, + repo: RepoDid, +} + +impl GitRefUpdate { + pub fn new(repo: RepoDid, owner: Option, committer: AccountDid) -> Self { + Self { + record_type: Self::NSID, + changed_files: Vec::new(), + committer_did: committer, + meta: None, + owner_did: owner, + push_options: PushOptions::default(), + change: RefChange::Absent, + repo, + } + } + + pub fn on_ref( + mut self, + ref_name: RefName, + transition: RefTransition, + format: ObjectFormat, + ) -> Self { + self.change = RefChange::Applied { + ref_name, + old_sha: transition.old_oid().unwrap_or_else(|| format.null_oid()), + new_sha: transition.new_oid().unwrap_or_else(|| format.null_oid()), + }; + self + } + + pub fn with_changed_files(mut self, changed: ChangedFiles) -> Self { + self.changed_files = changed.into_paths(); + self + } + + pub fn with_push_options(mut self, options: &PushOptions) -> Self { + self.push_options = options.clone(); + self + } + + pub fn with_meta(mut self, meta: RefUpdateMeta) -> Self { + self.meta = Some(meta); + self + } +} + +impl Publish for GitRefUpdate { + const NSID: &'static str = "sh.tangled.git.refUpdate"; +} + +#[derive(Debug, Clone, Serialize)] +pub struct RefUpdateMeta { + #[serde(rename = "isDefaultRef")] + is_default_ref: bool, + #[serde(rename = "commitCount")] + commit_count: CommitCountBreakdown, + #[serde(rename = "langBreakdown", skip_serializing_if = "Option::is_none")] + lang_breakdown: Option, +} + +impl RefUpdateMeta { + pub fn new( + is_default_ref: bool, + by_email: Vec, + languages: Vec, + ) -> Self { + Self { + is_default_ref, + commit_count: CommitCountBreakdown { + by_email: (!by_email.is_empty()).then_some(by_email), + }, + lang_breakdown: (!languages.is_empty()).then_some(LangBreakdown { + inputs: Some(languages), + }), + } + } +} + +#[derive(Debug, Clone, Serialize)] +struct CommitCountBreakdown { + #[serde(rename = "byEmail", skip_serializing_if = "Option::is_none")] + by_email: Option>, +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)] +#[serde(transparent)] +pub struct CommitCount(u64); + +impl CommitCount { + pub const fn new(count: u64) -> Self { + Self(count) + } + + pub const fn get(self) -> u64 { + self.0 + } + + pub const fn succ(self) -> Self { + Self(self.0.saturating_add(1)) + } +} + +#[derive(Debug, Clone, Serialize)] +pub struct EmailCommitCount { + email: Email, + count: CommitCount, +} + +impl EmailCommitCount { + pub fn new(email: Email, count: CommitCount) -> Self { + Self { email, count } + } +} + +#[derive(Debug, Clone, Serialize)] +struct LangBreakdown { + #[serde(skip_serializing_if = "Option::is_none")] + inputs: Option>, +} + +#[derive(Debug, Clone, Serialize)] +pub struct LanguageSize { + lang: LanguageName, + size: LanguageBytes, +} + +impl LanguageSize { + pub fn new(lang: LanguageName, size: LanguageBytes) -> Self { + Self { lang, size } + } +} + +#[derive(Debug, Clone, Copy, Serialize)] +#[serde(rename_all = "lowercase")] +enum AclOp { + Add, + Remove, +} + +#[derive(Debug, Clone, Serialize)] +pub struct KnotMemberUpdate { + op: AclOp, + subject: AccountDid, +} + +impl KnotMemberUpdate { + pub fn added(subject: AccountDid) -> Self { + Self { + op: AclOp::Add, + subject, + } + } + + pub fn removed(subject: AccountDid) -> Self { + Self { + op: AclOp::Remove, + subject, + } + } +} + +impl Publish for KnotMemberUpdate { + const NSID: &'static str = "sh.tangled.knot.memberUpdate"; +} + +#[derive(Debug, Clone, Serialize)] +pub struct RepoCollaboratorUpdate { + op: AclOp, + subject: AccountDid, + repo: RepoDid, +} + +impl RepoCollaboratorUpdate { + pub fn added(subject: AccountDid, repo: RepoDid) -> Self { + Self { + op: AclOp::Add, + subject, + repo, + } + } + + pub fn removed(subject: AccountDid, repo: RepoDid) -> Self { + Self { + op: AclOp::Remove, + subject, + repo, + } + } +} + +impl Publish for RepoCollaboratorUpdate { + const NSID: &'static str = "sh.tangled.repo.collaboratorUpdate"; +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ReplayEvents(std::num::NonZeroUsize); + +impl ReplayEvents { + pub fn new(value: usize) -> Option { + std::num::NonZeroUsize::new(value).map(Self) + } + + pub fn get(self) -> usize { + self.0.get() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ReplayBytes(std::num::NonZeroUsize); + +impl ReplayBytes { + pub fn new(value: usize) -> Option { + std::num::NonZeroUsize::new(value).map(Self) + } + + pub fn get(self) -> usize { + self.0.get() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ReplayBounds { + events: ReplayEvents, + bytes: ReplayBytes, +} + +impl ReplayBounds { + pub fn new(events: ReplayEvents, bytes: ReplayBytes) -> Self { + Self { events, bytes } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BatchEnd { + CaughtUp, + Bounded, +} + +pub struct Replayed { + pub events: Vec>, + pub end: BatchEnd, +} + +struct Entry { + event: Arc, + bytes: usize, +} + +struct Ring { + entries: VecDeque, + bytes: usize, + last_micros: UnixMicros, + pending: BTreeSet, +} + +struct Inner { + bounds: ReplayBounds, + ring: Mutex, + head: watch::Sender, +} + +impl Inner { + fn lock(&self) -> std::sync::MutexGuard<'_, Ring> { + self.ring + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + // leave that guiness be, boy! it needs to + fn settle(&self, ring: std::sync::MutexGuard<'_, Ring>) { + let head = stable_head(&ring); + drop(ring); + self.head.send_if_modified(|current| { + let changed = *current != head; + *current = head; + changed + }); + } +} + +fn stable_head(ring: &Ring) -> EventCursor { + let stable = match ring.pending.iter().next().copied() { + Some(horizon) => ring + .entries + .partition_point(|entry| entry.event.created < horizon), + None => ring.entries.len(), + }; + stable + .checked_sub(1) + .and_then(|index| ring.entries.get(index)) + .map(|entry| entry.event.created) + .unwrap_or(EventCursor::START) +} + +fn value_bytes(value: &Value) -> usize { + let node = std::mem::size_of::(); + match value { + Value::Null | Value::Bool(_) | Value::Number(_) => node, + Value::String(text) => node + text.len(), + Value::Array(items) => node + items.iter().map(value_bytes).sum::(), + Value::Object(fields) => { + node + fields + .iter() + .map(|(key, field)| key.len() + value_bytes(field)) + .sum::() + } + } +} + +fn insert_sorted(ring: &mut Ring, event: Event, bounds: ReplayBounds) { + let bytes = std::mem::size_of::() + value_bytes(&event.payload); + let position = ring + .entries + .partition_point(|existing| existing.event.created < event.created); + ring.entries.insert( + position, + Entry { + event: Arc::new(event), + bytes, + }, + ); + ring.bytes += bytes; + evict_oldest(ring, bounds); +} + +fn evict_oldest(ring: &mut Ring, bounds: ReplayBounds) { + let over = ring.entries.len() > bounds.events.get() + || (ring.bytes > bounds.bytes.get() && ring.entries.len() > 1); + if let Some(evicted) = over.then(|| ring.entries.pop_front()).flatten() { + ring.bytes -= evicted.bytes; + evict_oldest(ring, bounds); + } +} + +pub struct EventLog { + clock: C, + inner: Arc, +} + +impl EventLog { + pub fn new(clock: C, bounds: ReplayBounds) -> Self { + Self { + clock, + inner: Arc::new(Inner { + bounds, + ring: Mutex::new(Ring { + entries: VecDeque::new(), + bytes: 0, + last_micros: UnixMicros::new(0), + pending: BTreeSet::new(), + }), + head: watch::Sender::new(EventCursor::START), + }), + } + } + + fn next_cursor(&self, ring: &mut Ring) -> (UnixMicros, EventCursor) { + let micros = self.clock.now_unix_micros().max(ring.last_micros.next()); + ring.last_micros = micros; + (micros, EventCursor::from_unix_micros(micros)) + } + + pub fn publish(&self, payload: &P) -> EventCursor { + let payload = serde_json::to_value(payload).expect("event payload serializes to JSON"); + let mut ring = self.inner.lock(); + let (micros, created) = self.next_cursor(&mut ring); + insert_sorted( + &mut ring, + Event { + rkey: Tid::from_time(micros.get(), 0), + nsid: P::NSID, + payload, + created, + }, + self.inner.bounds, + ); + self.inner.settle(ring); + created + } + + pub fn reserve(&self) -> Reservation { + let mut ring = self.inner.lock(); + let (micros, cursor) = self.next_cursor(&mut ring); + ring.pending.insert(cursor); + drop(ring); + Reservation { + inner: Arc::clone(&self.inner), + cursor, + micros, + fulfilled: false, + } + } + + pub fn replay(&self, after: EventCursor, bounds: ReplayBounds) -> Replayed { + let ring = self.inner.lock(); + // The corresponding read side guarantee of `reserve`. + let horizon = ring.pending.iter().next().copied(); + let visible = |entry: &&Entry| { + entry.event.created > after + && horizon.is_none_or(|horizon| entry.event.created < horizon) + }; + let events: Vec> = ring + .entries + .iter() + .filter(visible) + .take(bounds.events.get()) + // Why the first event gets to ignore the byte bound? + // Imagine a consumer whose next event is by itself wider + // than the entire bound, right - + // every batch it requests would come back empty, + // its cursor would never advance, + // it would ask again, repeat. + // Sending that one event alone over the bound + // is the only way. + .scan(0usize, |spent, entry| { + let first = *spent == 0; + *spent += entry.bytes; + (first || *spent <= bounds.bytes.get()).then(|| Arc::clone(&entry.event)) + }) + .collect(); + let end = match ring.entries.iter().filter(visible).nth(events.len()) { + Some(_) => BatchEnd::Bounded, + None => BatchEnd::CaughtUp, + }; + Replayed { events, end } + } + + pub fn subscribe(&self) -> watch::Receiver { + self.inner.head.subscribe() + } +} + +pub struct Reservation { + inner: Arc, + cursor: EventCursor, + micros: UnixMicros, + fulfilled: bool, +} + +impl Reservation { + pub fn cursor(&self) -> EventCursor { + self.cursor + } + + pub fn fulfill(mut self, payload: &P) { + let payload = serde_json::to_value(payload).expect("event payload serializes to JSON"); + let mut ring = self.inner.lock(); + insert_sorted( + &mut ring, + Event { + rkey: Tid::from_time(self.micros.get(), 0), + nsid: P::NSID, + payload, + created: self.cursor, + }, + self.inner.bounds, + ); + ring.pending.remove(&self.cursor); + self.inner.settle(ring); + self.fulfilled = true; + } +} + +impl Drop for Reservation { + fn drop(&mut self) { + if self.fulfilled { + return; + } + let mut ring = self.inner.lock(); + ring.pending.remove(&self.cursor); + self.inner.settle(ring); + } +} + +knot_types::scalar_newtype! { + pub struct GlobalSubscriberLimit(usize); + pub struct PerPeerSubscriberLimit(usize); +} + +pub struct SubscriberGate { + global: Arc, + per_peer_max: usize, + peers: Mutex>, +} + +impl SubscriberGate { + pub fn new(global_max: GlobalSubscriberLimit, per_peer_max: PerPeerSubscriberLimit) -> Self { + Self { + global: Arc::new(Semaphore::new(global_max.get().max(1))), + per_peer_max: per_peer_max.get().max(1), + peers: Mutex::new(HashMap::new()), + } + } + + pub fn try_admit(self: &Arc, peer: IpAddr) -> Option { + let global = Arc::clone(&self.global).try_acquire_owned().ok()?; + let mut peers = self + .peers + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let current = peers.get(&peer).copied().unwrap_or(0); + if current >= self.per_peer_max { + return None; + } + peers.insert(peer, current + 1); + Some(SubscriberPermit { + _global: global, + gate: Arc::clone(self), + peer, + }) + } +} + +pub struct SubscriberPermit { + _global: OwnedSemaphorePermit, + gate: Arc, + peer: IpAddr, +} + +impl Drop for SubscriberPermit { + fn drop(&mut self) { + let mut peers = self + .gate + .peers + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if let Some(count) = peers.get_mut(&self.peer) { + *count -= 1; + if *count == 0 { + peers.remove(&self.peer); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use knot_runtime::{ManualClock, UnixMicros}; + + fn bounds(events: usize, bytes: usize) -> ReplayBounds { + ReplayBounds::new( + ReplayEvents::new(events).unwrap(), + ReplayBytes::new(bytes).unwrap(), + ) + } + + fn log(capacity: usize) -> EventLog { + EventLog::new( + ManualClock::new(UnixMicros::new(1_700_000_000_000_000)), + bounds(capacity, 1 << 20), + ) + } + + fn replay(log: &EventLog, after: EventCursor, limit: usize) -> Vec> { + log.replay(after, bounds(limit, 1 << 30)).events + } + + fn update() -> GitRefUpdate { + GitRefUpdate::new( + RepoDid::new("did:plc:limpet").unwrap(), + Some(OwnerDid::new("did:web:olaren.dev").unwrap()), + AccountDid::new("did:plc:nel").unwrap(), + ) + } + + fn wire(log: &EventLog, payload: &P) -> serde_json::Value { + log.publish(payload); + serde_json::to_value(&*replay(log, EventCursor::START, 1).remove(0)).unwrap() + } + + #[test] + fn publish_nsids_are_valid_type_names() { + [ + GitRefUpdate::NSID, + KnotMemberUpdate::NSID, + RepoCollaboratorUpdate::NSID, + ] + .iter() + .for_each(|nsid| { + assert!(knot_types::TypeName::new(*nsid).is_ok(), "{nsid}"); + }); + } + + #[test] + fn a_frozen_clock_still_yields_strictly_increasing_cursors_and_distinct_rkeys() { + let log = log(8); + let cursors: Vec<_> = (0..3).map(|_| log.publish(&update())).collect(); + assert!(cursors.windows(2).all(|pair| pair[0] < pair[1])); + let events = replay(&log, EventCursor::START, 8); + let rkeys: std::collections::BTreeSet<_> = events + .iter() + .map(|event| event.rkey.as_str().to_string()) + .collect(); + assert_eq!(rkeys.len(), 3); + } + + #[test] + fn the_ring_evicts_the_oldest_event_past_capacity() { + let log = log(2); + let first = log.publish(&update()); + log.publish(&update()); + log.publish(&update()); + let replayed = replay(&log, EventCursor::START, 8); + assert_eq!(replayed.len(), 2); + assert!(replayed.iter().all(|event| event.created > first)); + } + + #[test] + fn a_wide_event_evicts_by_bytes_long_before_the_ring_fills() { + let wide = |count: usize| { + update().with_changed_files(fill_changed( + (0..count).map(|index| format!("crates/knot-events/src/f{index}.rs")), + )) + }; + let log = EventLog::new( + ManualClock::new(UnixMicros::new(1_700_000_000_000_000)), + bounds(1_024, 64 * 1_024), + ); + (0..16).for_each(|_| { + log.publish(&wide(512)); + }); + let replayed = replay(&log, EventCursor::START, 1_024); + assert!( + (1..16).contains(&replayed.len()), + "the byte maximum evicts before the event maximum does: {}", + replayed.len() + ); + + let one = EventLog::new( + ManualClock::new(UnixMicros::new(1_700_000_000_000_000)), + bounds(1_024, 1), + ); + let only = one.publish(&wide(512)); + assert_eq!( + replay(&one, EventCursor::START, 8) + .iter() + .map(|event| event.created) + .collect::>(), + vec![only], + "the ring keeps the one event wider than the whole byte maximum" + ); + } + + fn fill_changed(paths: impl Iterator) -> ChangedFiles { + let mut budget = knot_types::ChangedFilesBudget::new(); + let _ = paths.into_iter().try_for_each(|path| { + budget.admit(knot_types::RepoPath::new(path).expect("test path is well-formed")) + }); + budget.finish() + } + + #[test] + fn replay_honors_the_cursor_and_the_limit() { + let log = log(8); + let cursors: Vec<_> = (0..4).map(|_| log.publish(&update())).collect(); + let after_second = replay(&log, cursors[1], 8); + assert_eq!( + after_second + .iter() + .map(|event| event.created) + .collect::>(), + cursors[2..].to_vec() + ); + assert_eq!(replay(&log, EventCursor::START, 2).len(), 2); + assert!(replay(&log, cursors[3], 8).is_empty()); + } + + #[test] + fn a_replay_batch_stops_at_the_byte_maximum_and_reports_whether_more_remains() { + let log = EventLog::new( + ManualClock::new(UnixMicros::new(1_700_000_000_000_000)), + bounds(1_024, 1 << 20), + ); + assert_eq!( + log.replay(EventCursor::START, bounds(8, 1 << 20)).end, + BatchEnd::CaughtUp, + "an empty ring has nothing left to send" + ); + let wide = update().with_changed_files(fill_changed( + (0..512).map(|index| format!("crates/knot-events/src/f{index}.rs")), + )); + let cursors: Vec = (0..8).map(|_| log.publish(&wide)).collect(); + + let batch = log.replay(EventCursor::START, bounds(1_024, 16 * 1_024)); + assert!( + (1..8).contains(&batch.events.len()), + "the byte maximum stops the batch before the event maximum does: {}", + batch.events.len() + ); + assert_eq!(batch.end, BatchEnd::Bounded); + + let rest = log.replay( + batch.events.last().expect("the batch is nonempty").created, + bounds(1_024, 1 << 30), + ); + assert_eq!(rest.end, BatchEnd::CaughtUp); + assert_eq!( + batch.events.len() + rest.events.len(), + cursors.len(), + "the two batches together are every event, with none repeated or skipped" + ); + let head = log.replay(cursors[7], bounds(8, 1 << 20)); + assert!(head.events.is_empty() && head.end == BatchEnd::CaughtUp); + + let single = log.replay(EventCursor::START, bounds(1_024, 1)); + assert_eq!( + single.events.len(), + 1, + "an event wider than the whole batch maximum is sent alone" + ); + assert_eq!(single.end, BatchEnd::Bounded); + } + + #[test] + fn a_subscriber_observes_the_head_advance() { + let log = log(8); + let mut head = log.subscribe(); + assert_eq!(*head.borrow_and_update(), EventCursor::START); + let created = log.publish(&update()); + assert!(head.has_changed().unwrap()); + assert_eq!(*head.borrow_and_update(), created); + } + + #[test] + fn the_wire_event_matches_the_eventstream_shape() { + let wire = wire(&log(8), &update()); + assert_eq!(wire["nsid"], "sh.tangled.git.refUpdate"); + assert_eq!(wire["created"].as_i64().unwrap() % 1_000, 0); + assert_eq!(wire["rkey"].as_str().unwrap().len(), 13); + let payload = &wire["event"]; + assert_eq!(payload["$type"], "sh.tangled.git.refUpdate"); + assert_eq!(payload["committerDid"], "did:plc:nel"); + assert_eq!(payload["ownerDid"], "did:web:olaren.dev"); + assert_eq!(payload["repo"], "did:plc:limpet"); + assert_eq!(payload["meta"], serde_json::Value::Null); + assert_eq!(payload["ref"], ""); + assert_eq!( + payload["oldSha"], "", + "a record about no ref sends the empty sha" + ); + assert_eq!(payload["newSha"], ""); + } + + #[test] + fn an_absent_sha_of_a_transition_is_the_null_oid_of_the_repo_object_format() { + let new = Oid::from_hex(&"cd".repeat(32)).unwrap(); + let created = &wire( + &log(8), + &GitRefUpdate::new( + RepoDid::new("did:plc:limpet").unwrap(), + None, + AccountDid::new("did:plc:nel").unwrap(), + ) + .on_ref( + RefName::new("refs/heads/fresh").unwrap(), + RefTransition::Create { new }, + ObjectFormat::SHA256, + ), + )["event"]; + assert_eq!(created["oldSha"], "0".repeat(64)); + assert_eq!(created["newSha"], new.to_hex()); + + let old = Oid::from_hex(&"ab".repeat(20)).unwrap(); + let rebuilt = update() + .on_ref( + RefName::new("refs/heads/fresh").unwrap(), + RefTransition::Create { + new: Oid::from_hex(&"cd".repeat(20)).unwrap(), + }, + ObjectFormat::SHA1, + ) + .on_ref( + RefName::new("refs/heads/gone").unwrap(), + RefTransition::Delete { old }, + ObjectFormat::SHA1, + ); + let payload = &wire(&log(8), &rebuilt)["event"]; + assert_eq!( + payload["ref"], "refs/heads/gone", + "a later transition replaces the earlier one whole" + ); + assert_eq!(payload["oldSha"], old.to_hex()); + assert_eq!(payload["newSha"], "0".repeat(40)); + } + + fn peer(last: u8) -> IpAddr { + IpAddr::V4(std::net::Ipv4Addr::new(127, 0, 0, last)) + } + + #[test] + fn gate_enforces_limits_and_prunes() { + let global = Arc::new(SubscriberGate::new( + GlobalSubscriberLimit::new(2), + PerPeerSubscriberLimit::new(8), + )); + let first = global + .try_admit(peer(1)) + .expect("first subscriber is admitted"); + let _second = global + .try_admit(peer(2)) + .expect("second subscriber is admitted"); + assert!( + global.try_admit(peer(3)).is_none(), + "third subscriber is refused once the global limit is reached" + ); + drop(first); + assert!( + global.try_admit(peer(3)).is_some(), + "freeing global slot admits waiting subscriber" + ); + + let per_peer = Arc::new(SubscriberGate::new( + GlobalSubscriberLimit::new(16), + PerPeerSubscriberLimit::new(2), + )); + let _socket = per_peer + .try_admit(peer(1)) + .expect("first socket is admitted"); + let second = per_peer + .try_admit(peer(1)) + .expect("second socket is admitted"); + assert!( + per_peer.try_admit(peer(1)).is_none(), + "third socket from same peer is refused at the per-peer limit" + ); + assert!( + per_peer.try_admit(peer(2)).is_some(), + "different peer keeps its own budget" + ); + drop(second); + assert!( + per_peer.try_admit(peer(1)).is_some(), + "freed per-peer slot is reusable" + ); + + let prune = Arc::new(SubscriberGate::new( + GlobalSubscriberLimit::new(16), + PerPeerSubscriberLimit::new(2), + )); + let permit = prune.try_admit(peer(1)).expect("admitted"); + drop(permit); + assert!( + prune + .peers + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .is_empty(), + "peer map prunes peer once its last socket closes" + ); + } + + #[test] + fn a_ref_update_includes_its_computed_meta_on_the_wire() { + let wire = wire( + &log(8), + &update().with_meta(RefUpdateMeta::new( + true, + vec![EmailCommitCount::new( + Email::new("nel@oyster.cafe"), + CommitCount::new(3), + )], + vec![LanguageSize::new( + LanguageName::new("Rust"), + LanguageBytes::new(1234), + )], + )), + ); + let meta = &wire["event"]["meta"]; + assert_eq!(meta["isDefaultRef"], true); + assert_eq!( + meta["commitCount"]["byEmail"][0]["email"], + "nel@oyster.cafe" + ); + assert_eq!(meta["commitCount"]["byEmail"][0]["count"], 3); + assert_eq!(meta["langBreakdown"]["inputs"][0]["lang"], "Rust"); + assert_eq!(meta["langBreakdown"]["inputs"][0]["size"], 1234); + } + + #[test] + fn an_empty_breakdown_omits_the_optional_meta_arrays() { + let wire = wire( + &log(8), + &update().with_meta(RefUpdateMeta::new(false, Vec::new(), Vec::new())), + ); + let meta = &wire["event"]["meta"]; + assert_eq!(meta["isDefaultRef"], false); + assert!(meta["commitCount"].get("byEmail").is_none()); + assert!(meta.get("langBreakdown").is_none()); + } + + #[test] + fn acl_updates_match_eventstream_shape() { + let log = log(8); + log.publish(&KnotMemberUpdate::added( + AccountDid::new("did:plc:nel").unwrap(), + )); + log.publish(&KnotMemberUpdate::removed( + AccountDid::new("did:plc:olaren").unwrap(), + )); + log.publish(&RepoCollaboratorUpdate::added( + AccountDid::new("did:plc:nel").unwrap(), + RepoDid::new("did:plc:limpet").unwrap(), + )); + log.publish(&RepoCollaboratorUpdate::removed( + AccountDid::new("did:plc:nel").unwrap(), + RepoDid::new("did:plc:limpet").unwrap(), + )); + let events = replay(&log, EventCursor::START, 8); + + let member_added = serde_json::to_value(&*events[0]).unwrap(); + assert_eq!(member_added["nsid"], "sh.tangled.knot.memberUpdate"); + assert_eq!(member_added["event"]["op"], "add"); + assert_eq!(member_added["event"]["subject"], "did:plc:nel"); + assert!(member_added["event"].get("$type").is_none()); + let member_removed = serde_json::to_value(&*events[1]).unwrap(); + assert_eq!(member_removed["event"]["op"], "remove"); + assert_eq!(member_removed["event"]["subject"], "did:plc:olaren"); + + let collab_added = serde_json::to_value(&*events[2]).unwrap(); + assert_eq!(collab_added["nsid"], "sh.tangled.repo.collaboratorUpdate"); + assert_eq!(collab_added["event"]["op"], "add"); + assert_eq!(collab_added["event"]["subject"], "did:plc:nel"); + assert_eq!(collab_added["event"]["repo"], "did:plc:limpet"); + assert!(collab_added["event"].get("$type").is_none()); + let collab_removed = serde_json::to_value(&*events[3]).unwrap(); + assert_eq!(collab_removed["event"]["op"], "remove"); + assert_eq!(collab_removed["event"]["repo"], "did:plc:limpet"); + } + + fn cursors(log: &EventLog) -> Vec { + replay(log, EventCursor::START, 64) + .iter() + .map(|event| event.created) + .collect() + } + + #[test] + fn a_reservation_holds_back_later_events_until_it_is_fulfilled() { + let log = log(8); + let early = log.publish(&update()); + let reservation = log.reserve(); + let later = log.publish(&update()); + assert!(reservation.cursor() > early && reservation.cursor() < later); + assert_eq!( + cursors(&log), + vec![early], + "event published after reservation waits behind it" + ); + let mid = reservation.cursor(); + reservation.fulfill(&update()); + assert_eq!( + cursors(&log), + vec![early, mid, later], + "fulfilling reservation releases it and event queued behind it, in cursor order" + ); + } + + #[test] + fn out_of_order_fulfillment_still_replays_in_cursor_order() { + let log = log(8); + let first = log.reserve(); + let second = log.reserve(); + let (c1, c2) = (first.cursor(), second.cursor()); + assert!(c1 < c2); + second.fulfill(&update()); + assert!( + cursors(&log).is_empty(), + "later reservation stays hidden while earlier one is outstanding" + ); + first.fulfill(&update()); + assert_eq!( + cursors(&log), + vec![c1, c2], + "both surface in cursor order regardless of fulfillment order" + ); + } + + #[test] + fn a_dropped_reservation_unblocks_the_horizon_without_an_event() { + let log = log(8); + let reservation = log.reserve(); + let later = log.publish(&update()); + assert!( + cursors(&log).is_empty(), + "later event waits behind unfulfilled reservation" + ); + drop(reservation); + assert_eq!( + cursors(&log), + vec![later], + "dropping reservation surfaces queued event and leaves no gap" + ); + } + + #[test] + fn the_head_holds_at_the_last_stable_event_until_a_reservation_is_fulfilled() { + let log = log(8); + let mut head = log.subscribe(); + let early = log.publish(&update()); + assert_eq!(*head.borrow_and_update(), early); + let reservation = log.reserve(); + let later = log.publish(&update()); + assert_eq!( + *head.borrow_and_update(), + early, + "head holds while lower-cursor reservation is pending" + ); + reservation.fulfill(&update()); + assert_eq!( + *head.borrow_and_update(), + later, + "fulfilling reservation advances head past released events" + ); + } + + #[test] + fn an_anonymous_owner_is_omitted_from_the_wire() { + let wire = wire( + &log(8), + &GitRefUpdate::new( + RepoDid::new("did:plc:limpet").unwrap(), + None, + AccountDid::new("did:plc:nel").unwrap(), + ), + ); + assert!(wire["event"].get("ownerDid").is_none()); + } +} diff --git a/knot2/crates/knot-fixtures/src/lib.rs b/knot2/crates/knot-fixtures/src/lib.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-fixtures/src/lib.rs @@ -0,0 +1,127 @@ +use std::io::Write; +use std::path::Path; +use std::process::{Command, Output, Stdio}; + +pub const AUTHOR_NAME: &str = "nel"; +pub const AUTHOR_EMAIL: &str = "nel@oyster.cafe"; +pub const PINNED_DATE: &str = "2026-01-01T00:00:00 +0000"; + +pub fn command(cwd: &Path) -> Command { + let mut command = Command::new("git"); + command + .current_dir(cwd) + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env("GIT_CONFIG_SYSTEM", "/dev/null") + .env("GIT_TERMINAL_PROMPT", "0") + .env("GIT_ASKPASS", "true") + .env("GIT_AUTHOR_NAME", AUTHOR_NAME) + .env("GIT_AUTHOR_EMAIL", AUTHOR_EMAIL) + .env("GIT_COMMITTER_NAME", AUTHOR_NAME) + .env("GIT_COMMITTER_EMAIL", AUTHOR_EMAIL); + command +} + +pub fn command_at(cwd: &Path, stamp: &str) -> Command { + let mut command = command(cwd); + command + .env("GIT_AUTHOR_DATE", stamp) + .env("GIT_COMMITTER_DATE", stamp); + command +} + +pub fn available() -> bool { + Command::new("git") + .arg("--version") + .output() + .map(|out| out.status.success()) + .unwrap_or(false) +} + +fn combined(out: &Output) -> String { + format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ) +} + +pub fn run(cwd: &Path, args: &[&str]) -> (bool, String) { + let out = command_at(cwd, PINNED_DATE) + .args(args) + .output() + .expect("git is available"); + (out.status.success(), combined(&out)) +} + +pub fn must(cwd: &Path, args: &[&str]) -> String { + let out = command_at(cwd, PINNED_DATE) + .args(args) + .output() + .expect("git is available"); + assert!(out.status.success(), "git {args:?}:\n{}", combined(&out)); + String::from_utf8_lossy(&out.stdout).trim().to_string() +} + +pub fn feed(cwd: &Path, args: &[&str], stdin: &[u8]) -> (bool, String) { + let mut child = command_at(cwd, PINNED_DATE) + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("git is available"); + child + .stdin + .take() + .expect("stdin was piped") + .write_all(stdin) + .expect("the write to git's stdin succeeds"); + let out = child.wait_with_output().expect("git exits"); + (out.status.success(), combined(&out)) +} + +pub fn fsck(bare: &Path) -> Result<(), String> { + match run( + bare, + &["fsck", "--no-dangling", "--no-reflogs", "--no-progress"], + ) { + (true, _) => Ok(()), + (false, report) => Err(report), + } +} + +pub fn commit(work: &Path, file: &str, contents: &str, message: &str) { + std::fs::write(work.join(file), contents).expect("fixture file is writable"); + must(work, &["add", "-A"]); + must(work, &["commit", "-q", "-m", message]); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_helper_pins_its_dates_so_the_same_sequence_yields_the_same_oid() { + if !available() { + return; + } + let oids = || { + let dir = tempfile::tempdir().unwrap(); + must(dir.path(), &["init", "-q", "-b", "main"]); + commit(dir.path(), "README.md", "kelp\n", "initial"); + let tree = must(dir.path(), &["hash-object", "-t", "tree", "-w", "--stdin"]); + let (ok, from_stdin) = feed(dir.path(), &["commit-tree", &tree, "-F", "-"], b"empty\n"); + assert!(ok, "{from_stdin}"); + ( + must(dir.path(), &["rev-parse", "HEAD"]), + from_stdin.trim().to_string(), + ) + }; + assert_eq!( + oids(), + oids(), + "an unpinned committer date would make every differential run disagree, \ + so a fixture writing through stdin pins the same dates as one that doesn't" + ); + } +} diff --git a/knot2/crates/knot-git/fuzz/.gitignore b/knot2/crates/knot-git/fuzz/.gitignore new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-git/fuzz/.gitignore @@ -0,0 +1,4 @@ +target +corpus +artifacts +coverage diff --git a/knot2/crates/knot-git/fuzz/Cargo.lock b/knot2/crates/knot-git/fuzz/Cargo.lock new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-git/fuzz/Cargo.lock @@ -0,0 +1,5247 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + +[[package]] +name = "arc-swap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +dependencies = [ + "rustversion", +] + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "async-compression" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "atomic-polyfill" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" +dependencies = [ + "critical-section", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base-x" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" + +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + +[[package]] +name = "base256emoji" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e9430d9a245a77c92176e649af6e275f20839a48389859d1661e9a128d077c" +dependencies = [ + "const-str", + "match-lookup", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bon" +version = "3.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2f04f6fef12d70d42a77b1433c9e0f065238479a6cefc4f5bab105e9873a3c3" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d0bd4c2f75335ad98052a37efb54f428b492f64340257143b3429c8a508fa7b" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + +[[package]] +name = "borrow-or-share" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" + +[[package]] +name = "borsh" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfd1e3f8955a5d7de9fab72fc8373fade9fb8a703968cb200ae3dc6cf08e185a" +dependencies = [ + "bytes", + "cfg_aliases", +] + +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "bstr" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +dependencies = [ + "serde", +] + +[[package]] +name = "bytesize" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd91ee7b2422bcb158d90ef4d14f75ef67f340943fc4149891dcce8f8b972a3" + +[[package]] +name = "cbor4ii" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544cf8c89359205f4f990d0e6f3828db42df85b5dac95d09157a250eb0749c4" +dependencies = [ + "serde", +] + +[[package]] +name = "cc" +version = "1.2.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "cid" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a304f95f84d169a6f31c4d0a30d784643aaa0bbc9c1e449a2c23e963ec4971" +dependencies = [ + "multibase", + "multihash", + "serde", + "serde_bytes", + "unsigned-varint 0.8.0", +] + +[[package]] +name = "clru" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "197fd99cb113a8d5d9b6376f3aa817f32c1078f2343b714fff7d2ca44fdf67d5" +dependencies = [ + "hashbrown 0.16.1", +] + +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.18", +] + +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "brotli", + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-str" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f421161cb492475f1661ddc9815a745a1c894592070661180fdec3d4872e9c3" + +[[package]] +name = "cordyceps" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "688d7fbb8092b8de775ef2536f36c8c31f2bc4006ece2e8d8ad2d17d00ce0a2a" +dependencies = [ + "loom", + "tracing", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "data-encoding-macro" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3259c913752a86488b501ed8680446a5ed2d5aeac6e596cb23ba3800768ea32c" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccc2776f0c61eca1ca32528f85548abd1a4be8fb53d1b21c013e4f18da1e7090" +dependencies = [ + "data-encoding", + "syn", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "derive_more" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "diatomic-waker" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab03c107fafeb3ee9f5925686dbb7a73bc76e3932abb0d2b365cb64b169cf04c" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "const-oid", + "crypto-common 0.1.6", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest 0.10.7", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "rand_core 0.6.4", + "serde", + "sha2 0.10.9", + "subtle", + "zeroize", +] + +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "pem-rfc7468", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "faster-hex" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7223ae2d2f179b803433d9c830478527e92b8117eab39460edae7f1614d9fb73" +dependencies = [ + "heapless 0.8.0", + "serde", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "fluent-uri" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-buffered" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4421cb78ee172b6b06080093479d3c50f058e7c81b7d577bbb8d118d551d4cd5" +dependencies = [ + "cordyceps", + "diatomic-waker", + "futures-core", + "pin-project-lite", + "spin 0.10.0", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generator" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + +[[package]] +name = "generic-array" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +dependencies = [ + "typenum", + "version_check", + "zeroize", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + +[[package]] +name = "gix" +version = "0.84.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae54ae0ebd1a5a3c3f8d95dd3b5ca6e63f4fed9bfd585e13801a97d7bde8f9ce" +dependencies = [ + "gix-actor", + "gix-archive", + "gix-attributes", + "gix-blame", + "gix-command", + "gix-commitgraph", + "gix-config", + "gix-credentials", + "gix-date", + "gix-diff", + "gix-dir", + "gix-discover", + "gix-error", + "gix-features", + "gix-filter", + "gix-fs", + "gix-glob", + "gix-hash", + "gix-hashtable", + "gix-ignore", + "gix-index", + "gix-lock", + "gix-mailmap", + "gix-negotiate", + "gix-object", + "gix-odb", + "gix-pack", + "gix-path", + "gix-pathspec", + "gix-prompt", + "gix-protocol", + "gix-ref", + "gix-refspec", + "gix-revision", + "gix-revwalk", + "gix-sec", + "gix-shallow", + "gix-status", + "gix-submodule", + "gix-tempfile", + "gix-trace", + "gix-traverse", + "gix-url", + "gix-utils", + "gix-validate", + "gix-worktree", + "gix-worktree-state", + "gix-worktree-stream", + "nonempty", + "parking_lot", + "regex", + "signal-hook", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-actor" +version = "0.41.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bc998b8f746dda8565450d08a63b792ced9165d8c27a1ed3f02799ec6a7820f" +dependencies = [ + "bstr", + "gix-date", + "gix-error", +] + +[[package]] +name = "gix-archive" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16909cacc78936ab96f6c3be08379d0a2e88bfa3a7527972d2ed75c7517ef31e" +dependencies = [ + "bstr", + "flate2", + "gix-date", + "gix-error", + "gix-object", + "gix-path", + "gix-worktree-stream", + "rawzip", + "tar", +] + +[[package]] +name = "gix-attributes" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d43f12e246d3bf7ec624c8fc15ac4a4b62b7c4c6f586cb82be6c90bf84c9d02" +dependencies = [ + "bstr", + "gix-glob", + "gix-path", + "gix-quote", + "gix-trace", + "kstring", + "smallvec", + "thiserror 2.0.18", + "unicode-bom", +] + +[[package]] +name = "gix-bitmap" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ebef0c26ad305747649e727bbcd56a7b7910754eb7cea88f6dff6f93c51283" +dependencies = [ + "gix-error", +] + +[[package]] +name = "gix-blame" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d39a0c14af94c2edaa5eefe06d5ef2cdea55316ae9a9321314288e3f55fa4c0" +dependencies = [ + "gix-commitgraph", + "gix-date", + "gix-diff", + "gix-error", + "gix-hash", + "gix-object", + "gix-revwalk", + "gix-trace", + "gix-traverse", + "gix-worktree", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-chunk" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9faee47943b638e58ddd5e275a4906ad3e4b6c8584f1d41bd18ab9032ec52afb" +dependencies = [ + "gix-error", +] + +[[package]] +name = "gix-command" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00706d4fef135ef4b01680d5218c6ee40cda8baf697b864296cbc887d19118f6" +dependencies = [ + "bstr", + "gix-path", + "gix-quote", + "gix-trace", + "shell-words", +] + +[[package]] +name = "gix-commitgraph" +version = "0.37.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f675d0df484a7f6a47e64bd6f311af489d947c0323b0564f36d14f3d7762abb" +dependencies = [ + "bstr", + "gix-chunk", + "gix-error", + "gix-hash", + "memmap2", + "nonempty", +] + +[[package]] +name = "gix-config" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2372d4b49ca28431e7d150cab9d25edc1890f0184bd57eb0e917c7799e63de" +dependencies = [ + "bstr", + "gix-config-value", + "gix-features", + "gix-glob", + "gix-path", + "gix-ref", + "gix-sec", + "smallvec", + "thiserror 2.0.18", + "unicode-bom", +] + +[[package]] +name = "gix-config-value" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed42168329552f6c2e5df09665c104199d45d84bedb53683738a49b57fe1baab" +dependencies = [ + "bitflags", + "bstr", + "gix-path", + "libc", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-credentials" +version = "0.38.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40cd22f0dd71988be12d6e78b1709de2370e1957c5f107ff31e56caeba3745d" +dependencies = [ + "bstr", + "gix-command", + "gix-config-value", + "gix-date", + "gix-path", + "gix-prompt", + "gix-sec", + "gix-trace", + "gix-url", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-date" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3ecab64a98bbac9f8e02990a9ea5e3c974a7d49b95f2bd70ad94ad22fa6b48c" +dependencies = [ + "bstr", + "gix-error", + "itoa", + "jiff", +] + +[[package]] +name = "gix-diff" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b6d9528f32d94cef2edf39a1ac01fe5a0fc44ddbb18d9e44099936047c3302b" +dependencies = [ + "bstr", + "gix-attributes", + "gix-command", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-imara-diff", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-tempfile", + "gix-trace", + "gix-traverse", + "gix-worktree", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-dir" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21bb2a53a6fd917ec499ed0bfb5b6887de7a15bd79197dcea7c987938749a9f1" +dependencies = [ + "bstr", + "gix-discover", + "gix-fs", + "gix-ignore", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-trace", + "gix-utils", + "gix-worktree", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-discover" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77bacdd12b7879d2178a80c58c2f319995e4654e1a7a23e3181e5c8a12b824f7" +dependencies = [ + "bstr", + "dunce", + "gix-fs", + "gix-path", + "gix-ref", + "gix-sec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-error" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57831e199be480af90dcd7e459abed8a174c09ec9a6e2cc8f7ca6c54598b06b" +dependencies = [ + "bstr", +] + +[[package]] +name = "gix-features" +version = "0.48.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1849ae154d38bc403185be14fa871e38e3c93ee606875d94e207fdb9fba52dbc" +dependencies = [ + "bytes", + "bytesize", + "crc32fast", + "crossbeam-channel", + "gix-path", + "gix-trace", + "gix-utils", + "libc", + "once_cell", + "parking_lot", + "prodash", + "thiserror 2.0.18", + "walkdir", + "zlib-rs", +] + +[[package]] +name = "gix-filter" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecf74b7d16f6694ce4a3049074c41be0c7987105743674f1671807bd6dce09fa" +dependencies = [ + "bstr", + "encoding_rs", + "gix-attributes", + "gix-command", + "gix-hash", + "gix-object", + "gix-packetline", + "gix-path", + "gix-quote", + "gix-trace", + "gix-utils", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-fs" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cdff46db8798e47e2f727d84b9379aac5add3dd3d9d0b07bb4d7d5d640771fe" +dependencies = [ + "bstr", + "fastrand", + "gix-features", + "gix-path", + "gix-utils", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-glob" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1fcb8ef5b16bcf874abe9b68d8abb3c0493c876d367ab824151f30a0f3f3756" +dependencies = [ + "bitflags", + "bstr", + "gix-features", + "gix-path", +] + +[[package]] +name = "gix-hash" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0926d3819c837750b4e03c7754901e73f68b8c9b690753a6372a1bed4eedce" +dependencies = [ + "faster-hex", + "gix-features", + "sha1-checked", + "sha2 0.11.0", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-hashtable" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0e30b93eea8718baf7d8153fcb938e2926175bbf18097c09f1c01b6f0be0563" +dependencies = [ + "gix-hash", + "hashbrown 0.17.1", + "parking_lot", +] + +[[package]] +name = "gix-ignore" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d491bab9bf2c9f341dc754f425c31d5d3f63aca615312167b82e1deeaca97d8d" +dependencies = [ + "bstr", + "gix-glob", + "gix-path", + "gix-trace", + "unicode-bom", +] + +[[package]] +name = "gix-imara-diff" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19753d40da53d0ec41604750eeb969097a90fb2d7f7992730d904541c04e2c19" +dependencies = [ + "bstr", + "hashbrown 0.17.1", +] + +[[package]] +name = "gix-index" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e6b28cc592dc753adb58302bb14a64e412ee591a3bec77aa4df87bff74fa80d" +dependencies = [ + "bitflags", + "bstr", + "filetime", + "fnv", + "gix-bitmap", + "gix-features", + "gix-fs", + "gix-hash", + "gix-lock", + "gix-object", + "gix-traverse", + "gix-utils", + "gix-validate", + "hashbrown 0.17.1", + "itoa", + "libc", + "memmap2", + "rustix", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-lock" +version = "23.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9dedd9e90b0d47624d2ed241d394e09294118364e87b9b7e5f1fe755f3c2c" +dependencies = [ + "gix-tempfile", + "gix-utils", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-mailmap" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "195fd20808055824531be2fd0d34136d900e5fbca3ffb0a3c07e8beeefb9c828" +dependencies = [ + "bstr", + "gix-actor", + "gix-date", + "gix-error", +] + +[[package]] +name = "gix-negotiate" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "890c936a215bae25818c076cb881cb2e54d2c66ba947ba58b8dd47cff921bf55" +dependencies = [ + "bitflags", + "gix-commitgraph", + "gix-date", + "gix-hash", + "gix-object", + "gix-revwalk", +] + +[[package]] +name = "gix-object" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5cd857e29429c7213bdef3f5aef83f8cc124774fe8ae0d27b1607d218d6d525" +dependencies = [ + "bstr", + "gix-actor", + "gix-date", + "gix-features", + "gix-hash", + "gix-hashtable", + "gix-utils", + "gix-validate", + "itoa", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-odb" +version = "0.81.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d004c32858b1556f2d7874405edb3c97dc78fc09beaa87d57bb077ee2858a7d" +dependencies = [ + "arc-swap", + "gix-features", + "gix-fs", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-pack", + "gix-path", + "gix-quote", + "memmap2", + "parking_lot", + "tempfile", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-pack" +version = "0.71.0" +dependencies = [ + "clru", + "gix-chunk", + "gix-diff", + "gix-error", + "gix-features", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-path", + "gix-tempfile", + "gix-traverse", + "parking_lot", + "smallvec", + "thiserror 2.0.18", + "uluru", +] + +[[package]] +name = "gix-packetline" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb18337ba2830bb43367d1af43819c8c78f31337f079fc76d0f1f1750a173126" +dependencies = [ + "bstr", + "faster-hex", + "gix-trace", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-path" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afa6ac14cd14939ea94a496ce7460daa6511c09f5b84757e9cfc6f9c8d0f93a6" +dependencies = [ + "bstr", + "gix-trace", + "gix-validate", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-pathspec" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3050783b41ee11511e1e8fb35623df81806194f4030395f14f48ea37c2798c9f" +dependencies = [ + "bitflags", + "bstr", + "gix-attributes", + "gix-config-value", + "gix-glob", + "gix-path", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-prompt" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee604d7746080ae7e1023bf47204bcc2c5f307bfbe2306a3c90b1bfd1a2c6d8" +dependencies = [ + "gix-command", + "gix-config-value", + "parking_lot", + "rustix", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-protocol" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51dea3acb390707ab868f1f9584f18449eb95d869deffae96768e47d303595ee" +dependencies = [ + "bstr", + "gix-date", + "gix-features", + "gix-hash", + "gix-ref", + "gix-shallow", + "gix-transport", + "gix-utils", + "maybe-async", + "nonempty", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-quote" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6e541fc33cc2b783b7979040d445a0c86a2eca747c8faea4ca84230d06ae6ef" +dependencies = [ + "bstr", + "gix-error", + "gix-utils", +] + +[[package]] +name = "gix-ref" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c04f64c37eb7e6feb73c7060f8dc6f381cc5de5d53249bfd450bc48a86b2e8b" +dependencies = [ + "gix-actor", + "gix-features", + "gix-fs", + "gix-hash", + "gix-lock", + "gix-object", + "gix-path", + "gix-tempfile", + "gix-utils", + "gix-validate", + "memmap2", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-refspec" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b216ae06ec74b5f24ad0142026a997fb0a935b7410eaf9c1616fc3f0e6c5a6d3" +dependencies = [ + "bstr", + "gix-error", + "gix-glob", + "gix-hash", + "gix-revision", + "gix-validate", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-revision" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b47c88884dd3c1a19a39da19d10211fcdea2809aadc86869b6e824a1774340f" +dependencies = [ + "bitflags", + "bstr", + "gix-commitgraph", + "gix-date", + "gix-error", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-revwalk", + "gix-trace", + "nonempty", +] + +[[package]] +name = "gix-revwalk" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85f5756abffe0917827aac683b13684ed99875bc398fa1f9b8f479b0681ef9e6" +dependencies = [ + "gix-commitgraph", + "gix-date", + "gix-error", + "gix-hash", + "gix-hashtable", + "gix-object", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-sec" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab8519976e4c7e486270740a5400369f37940779b80bd1377d94cfa1125d01b3" +dependencies = [ + "bitflags", + "gix-path", + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "gix-shallow" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a292fc2fe548c5dfa575479d16b445b0ddf1dd2f56f1fec6aed386f82553cd97" +dependencies = [ + "bstr", + "gix-hash", + "gix-lock", + "nonempty", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-status" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22042e385d28a34275e029d98f4970285045be14b9073658ca897923f2ed8700" +dependencies = [ + "bstr", + "filetime", + "gix-diff", + "gix-dir", + "gix-features", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-worktree", + "portable-atomic", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-submodule" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3059890ef054066c22a94bfc6a3eaba0d806aedcd630a0bc9e5783fd88884781" +dependencies = [ + "bstr", + "gix-config", + "gix-path", + "gix-pathspec", + "gix-refspec", + "gix-url", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-tempfile" +version = "23.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27850097e1ff9515f46a0dad0f5f9c9d020e972727772dabab9450690c4adb22" +dependencies = [ + "dashmap", + "gix-fs", + "libc", + "parking_lot", + "signal-hook", + "signal-hook-registry", + "tempfile", +] + +[[package]] +name = "gix-trace" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44dc45eae785c0eb14173e0f152e6e224dcf4d45b6a6999a3aed22af541ad678" + +[[package]] +name = "gix-transport" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd0e34995b1aab0fa8dff2af8db726a0bfad3e119c89302604463264046e7ff" +dependencies = [ + "bstr", + "gix-command", + "gix-features", + "gix-packetline", + "gix-quote", + "gix-sec", + "gix-url", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-traverse" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8de590ecc86a3b2870665f2288324fa9f7f8672c7fc2d4e020fdd81cd1f7aed" +dependencies = [ + "bitflags", + "gix-commitgraph", + "gix-date", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-revwalk", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-url" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bb01ec69d55e82ccb7a19e264501ead4e6aac38463a8cebfdd81e22bb67ab2" +dependencies = [ + "bstr", + "gix-path", + "percent-encoding", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-utils" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66c50966184123caf580ffa64e28031a878597f1c7fceb8fe19566c38eb1b771" +dependencies = [ + "bstr", + "fastrand", + "unicode-normalization", +] + +[[package]] +name = "gix-validate" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bc6fc771c4063ba7cd2f47b91fb6076251c6a823b64b7fe7b8874b0fe4afae3" +dependencies = [ + "bstr", +] + +[[package]] +name = "gix-worktree" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef414ed275e8407cd5d53d301e83be19700b0dd3f859d2434417b58f454a2d1" +dependencies = [ + "bstr", + "gix-attributes", + "gix-fs", + "gix-glob", + "gix-hash", + "gix-ignore", + "gix-index", + "gix-object", + "gix-path", + "gix-validate", +] + +[[package]] +name = "gix-worktree-state" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bffae8b3ca258fdd50370cd51f06deb4c76a3b43db3868bc28dde45ffa77d69" +dependencies = [ + "bstr", + "gix-features", + "gix-filter", + "gix-fs", + "gix-index", + "gix-object", + "gix-path", + "gix-worktree", + "io-close", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-worktree-stream" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25e9ed30100c63f7590bc581c225e53f731a53e06aa79a245739c07f7dcc557" +dependencies = [ + "gix-attributes", + "gix-error", + "gix-features", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-object", + "gix-path", + "gix-traverse", + "parking_lot", +] + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "h2" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "heapless" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" +dependencies = [ + "atomic-polyfill", + "hash32 0.2.1", + "rustc_version", + "serde", + "spin 0.9.8", + "stable_deref_trait", +] + +[[package]] +name = "heapless" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32 0.3.1", + "stable_deref_trait", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "human_format" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaec953f16e5bcf6b8a3cb3aa959b17e5577dbd2693e94554c462c08be22624b" + +[[package]] +name = "hybrid-array" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "inventory" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +dependencies = [ + "rustversion", +] + +[[package]] +name = "io-close" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cadcf447f06744f8ce713d2d6239bb5bde2c357a452397a9ed90c625da390bc" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "ipld-core" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090f624976d72f0b0bb71b86d58dc16c15e069193067cb3a3a09d655246cbbda" +dependencies = [ + "cid", + "serde", + "serde_bytes", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "iroh-car" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f8cd4cb9aa083fba8b52e921764252d0b4dcb1cd6d120b809dbfe1106e81a" +dependencies = [ + "anyhow", + "cid", + "futures", + "serde", + "serde_ipld_dagcbor", + "thiserror 1.0.69", + "tokio", + "unsigned-varint 0.7.2", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jacquard-api" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5c803a3c097e3ef8aea63747b4fe3fc9e339cd18272dd0366b1d10dd90d5c3f" +dependencies = [ + "jacquard-common", + "jacquard-derive", + "jacquard-lexicon", + "miette", + "serde", + "thiserror 2.0.18", +] + +[[package]] +name = "jacquard-common" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fec8b6661e6fcfb4d8b9eb1132503be71bc3e3c6f6ff15fa5500ec62655f6da7" +dependencies = [ + "base64", + "bon", + "bytes", + "chrono", + "ciborium", + "ciborium-io", + "cid", + "ed25519-dalek", + "fluent-uri", + "futures", + "getrandom 0.2.17", + "getrandom 0.3.4", + "hashbrown 0.15.5", + "http", + "ipld-core", + "k256", + "maitake-sync", + "miette", + "multibase", + "multihash", + "n0-future", + "oxilangtag", + "p256", + "phf", + "postcard", + "rand 0.9.4", + "regex", + "regex-automata", + "regex-lite", + "reqwest", + "rustversion", + "serde", + "serde_bytes", + "serde_html_form", + "serde_ipld_dagcbor", + "serde_json", + "signature", + "smol_str", + "spin 0.10.0", + "thiserror 2.0.18", + "tokio", + "tokio-tungstenite-wasm", + "tokio-util", + "trait-variant", + "unicode-segmentation", +] + +[[package]] +name = "jacquard-derive" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e41b82a9a72c0d2b907a5e0734e0692615e717cda903502c6c034ae6f3ddddf7" +dependencies = [ + "heck", + "jacquard-lexicon", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jacquard-lexicon" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "160049c269e3d7ec56f130736d25d61953fc6216da660c8b5e102f6b052dd3a1" +dependencies = [ + "cid", + "dashmap", + "heck", + "inventory", + "jacquard-common", + "miette", + "multihash", + "prettyplease", + "proc-macro2", + "quote", + "serde", + "serde_ipld_dagcbor", + "serde_json", + "serde_path_to_error", + "serde_repr", + "serde_with", + "sha2 0.10.9", + "syn", + "thiserror 2.0.18", + "unicode-segmentation", +] + +[[package]] +name = "jacquard-repo" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98986367bb78dadaa0f2f07196bab357786c0e3670d8311b350585b91f84d6eb" +dependencies = [ + "bytes", + "cid", + "ed25519-dalek", + "iroh-car", + "jacquard-api", + "jacquard-common", + "jacquard-derive", + "k256", + "miette", + "multihash", + "n0-future", + "p256", + "serde", + "serde_bytes", + "serde_ipld_dagcbor", + "sha2 0.10.9", + "smol_str", + "thiserror 2.0.18", + "tokio", + "trait-variant", +] + +[[package]] +name = "jiff" +version = "0.2.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4603d3033e49e2b0e31229fcab20a5d40089c607d975cd9c80551dc69eed9102" +dependencies = [ + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link", +] + +[[package]] +name = "jiff-static" +version = "0.2.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "782d32378dddf207193ac91cefb848ad41abb58195c95168e1291227a0832b47" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c900ef84826f1338a557697dc8fc601df9ca9af4ac137c7fb61d4c6f2dfd3076" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2025f20d7a4fa7785846e7b63d10a76d3f1cee98ee5cb79ea59703f95e42162" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "k256" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" +dependencies = [ + "cfg-if", + "ecdsa", + "elliptic-curve", + "once_cell", + "sha2 0.10.9", + "signature", +] + +[[package]] +name = "knot-git" +version = "0.1.0" +dependencies = [ + "base64", + "dashmap", + "flate2", + "gix", + "gix-archive", + "gix-bitmap", + "gix-hash", + "gix-pack", + "knot-resource", + "knot-types", + "moka", + "thiserror 2.0.18", + "walkdir", +] + +[[package]] +name = "knot-git-fuzz" +version = "0.0.0" +dependencies = [ + "knot-git", + "libfuzzer-sys", +] + +[[package]] +name = "knot-resource" +version = "0.1.0" +dependencies = [ + "rustix", +] + +[[package]] +name = "knot-types" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "chrono", + "cid", + "gix-hash", + "http", + "jacquard-common", + "jacquard-derive", + "jacquard-lexicon", + "jacquard-repo", + "miette", + "serde", + "serde_json", + "thiserror 2.0.18", + "walkdir", +] + +[[package]] +name = "kstring" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "558bf9508a558512042d3095138b1f7b8fe90c5467d94f9f1da28b3731c5dbd1" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" + +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "maitake-sync" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6816ab14147f80234c675b80ed6dc4f440d8a1cefc158e766067aedb84c0bcd5" +dependencies = [ + "cordyceps", + "loom", + "mycelium-bitfield", + "pin-project", + "portable-atomic", +] + +[[package]] +name = "match-lookup" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "757aee279b8bdbb9f9e676796fd459e4207a1f986e87886700abf589f5abf771" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "maybe-async" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "746873a384ad60adc5db74471dfaba74bd278afbdcfd81db93fafcdfc8b5ca0c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "memchr" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" + +[[package]] +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +dependencies = [ + "libc", +] + +[[package]] +name = "miette" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" +dependencies = [ + "cfg-if", + "miette-derive", + "unicode-width", +] + +[[package]] +name = "miette-derive" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "moka" +version = "0.12.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "event-listener", + "futures-util", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "multibase" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8694bb4835f452b0e3bb06dbebb1d6fc5385b6ca1caf2e55fd165c042390ec77" +dependencies = [ + "base-x", + "base256emoji", + "data-encoding", + "data-encoding-macro", +] + +[[package]] +name = "multihash" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "577c63b00ad74d57e8c9aa870b5fccebf2fd64a308a5aee9f1bb88e4aea19447" +dependencies = [ + "serde", + "unsigned-varint 0.8.0", +] + +[[package]] +name = "mycelium-bitfield" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24e0cc5e2c585acbd15c5ce911dff71e1f4d5313f43345873311c4f5efd741cc" + +[[package]] +name = "n0-future" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb0e5d99e681ab3c938842b96fcb41bf8a7bb4bfdb11ccbd653a7e83e06c794" +dependencies = [ + "cfg_aliases", + "derive_more", + "futures-buffered", + "futures-lite", + "futures-util", + "js-sys", + "pin-project", + "send_wrapper", + "tokio", + "tokio-util", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "nonempty" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9737e026353e5cd0736f98eddae28665118eb6f6600902a7f50db585621fecb6" + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.60.2", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "oxilangtag" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d3b4eb570abd4a1dcb062c31fd37b832264d9dc7292c3e69acfe926c87b063f" +dependencies = [ + "serde", +] + +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2 0.10.9", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand 0.8.6", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "heapless 0.7.17", + "serde", +] + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prodash" +version = "31.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "962200e2d7d551451297d9fdce85138374019ada198e30ea9ede38034e27604c" +dependencies = [ + "bytesize", + "human_format", + "parking_lot", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.4", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rawzip" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d9575f44c8cf85bc843ad666dcdf20d05a7753772bef56eb2a5140282b32150" + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "regex" +version = "1.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "mime", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "send_wrapper" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_html_form" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acf96b1d9364968fce46ebb548f1c0e1d7eceae27bdff73865d42e6c7369d94" +dependencies = [ + "form_urlencoded", + "indexmap", + "itoa", + "serde_core", +] + +[[package]] +name = "serde_ipld_dagcbor" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46182f4f08349a02b45c998ba3215d3f9de826246ba02bb9dddfe9a2a2100778" +dependencies = [ + "cbor4ii", + "ipld-core", + "scopeguard", + "serde", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" +dependencies = [ + "base64", + "bs58", + "chrono", + "hex", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha1-checked" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89f599ac0c323ebb1c6082821a54962b839832b03984598375bff3975b804423" +dependencies = [ + "digest 0.10.7", + "sha1", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shell-words" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a0c28ca5908dbdbcd52e6fdaa00358ab88637f8ab33e1f188dd510eb44b53d" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "smol_str" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa7368fcf4852a4c2dd92df0cace6a71f2091ca0a23391ce7f3a31833f1523" +dependencies = [ + "borsh", + "serde_core", +] + +[[package]] +name = "socket2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.52.0", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" +dependencies = [ + "futures-util", + "log", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tungstenite", +] + +[[package]] +name = "tokio-tungstenite-wasm" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e21a5c399399c3db9f08d8297ac12b500e86bca82e930253fdc62eaf9c0de6ae" +dependencies = [ + "futures-channel", + "futures-util", + "http", + "httparse", + "js-sys", + "rustls", + "thiserror 1.0.69", + "tokio", + "tokio-tungstenite", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "async-compression", + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tokio", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "trait-variant" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.8.6", + "rustls", + "rustls-pki-types", + "sha1", + "thiserror 1.0.69", + "utf-8", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "uluru" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c8a2469e56e6e5095c82ccd3afb98dad95f7af7929aab6d8ba8d6e0f73657da" +dependencies = [ + "arrayvec", +] + +[[package]] +name = "unicode-bom" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eec5d1121208364f6793f7d2e222bf75a915c19557537745b195b253dd64217" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "unsigned-varint" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" + +[[package]] +name = "unsigned-varint" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" +dependencies = [ + "getrandom 0.4.2", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.3+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +dependencies = [ + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a254a4b10c19a76f09a27640e7ffbf9bc30bf67e16a3bf28aaefa4920fe81563" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54568702fabf5d4849ce2b90fadfa64168a097eaf4b351ce9df8b687a0086aaf" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24a40fc75b0ec6f3746ceb10d36f53a93dcd68a93b11b6445983945d79eba0dc" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "908f34bd9b9ce3d4caf07b72dfab63d61504d156856c6bd3cd87fa350cf3985b" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7acbf7616c27b194bbb550bf77ed0c2c3e5b7fd1260a93082b95fb7f47959b92" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "web-sys" +version = "0.3.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e0871acf327f283dc6da28a1696cdc64fb355ba9f935d052021fa77f35cce69" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zlib-rs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/knot2/crates/knot-git/fuzz/Cargo.toml b/knot2/crates/knot-git/fuzz/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-git/fuzz/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "knot-git-fuzz" +version = "0.0.0" +publish = false +edition = "2024" + +[package.metadata] +cargo-fuzz = true + +[workspace] + +[dependencies] +libfuzzer-sys = "0.4" + +[dependencies.knot-git] +path = ".." + +[[bin]] +name = "patch" +path = "fuzz_targets/patch.rs" +test = false +doc = false +bench = false + +[patch.crates-io] +gix-pack = { path = "../../../third_party/gix-pack" } diff --git a/knot2/crates/knot-git/src/archive.rs b/knot2/crates/knot-git/src/archive.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-git/src/archive.rs @@ -0,0 +1,119 @@ +use std::sync::atomic::AtomicBool; + +use gix::bstr::BString; +use knot_types::{Oid, ParseError}; + +use crate::error::{GitError, backend}; +use crate::repo::Repo; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ArchiveFormat { + Tar, + TarGz, + Zip, +} + +impl ArchiveFormat { + fn gix(self) -> gix_archive::Format { + match self { + ArchiveFormat::Tar => gix_archive::Format::Tar, + ArchiveFormat::TarGz => gix_archive::Format::TarGz { + compression_level: None, + }, + ArchiveFormat::Zip => gix_archive::Format::Zip { + compression_level: None, + }, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ArchivePrefix(String); + +impl ArchivePrefix { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + let safe = !value.contains('\0') + && !value.starts_with(['/', '\\']) + && value.split(['/', '\\']).all(|component| component != ".."); + match safe { + true => Ok(Self(value)), + false => Err(ParseError::Invalid { + kind: "archive prefix", + value, + }), + } + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl Repo { + pub fn peel_to_tree(&self, oid: Oid) -> Result { + self.git() + .find_object(oid.object_id()) + .map_err(backend)? + .peel_to_tree() + .map(|tree| Oid::from(tree.id)) + .map_err(backend) + } + + pub fn write_archive( + &self, + tree: Oid, + format: ArchiveFormat, + prefix: Option<&ArchivePrefix>, + mut out: impl std::io::Write + std::io::Seek, + ) -> Result<(), GitError> { + let (stream, _index) = self + .git() + .worktree_stream(tree.object_id()) + .map_err(backend)?; + let interrupt = AtomicBool::new(false); + self.git() + .worktree_archive( + stream, + &mut out, + gix::progress::Discard, + &interrupt, + gix_archive::Options { + format: format.gix(), + tree_prefix: prefix.map(|prefix| BString::from(prefix.as_str())), + modification_time: 0, + }, + ) + .map_err(backend) + } +} + +#[cfg(test)] +mod tests { + use super::ArchivePrefix; + + #[test] + fn a_plain_nested_prefix_is_accepted() { + assert!(ArchivePrefix::new("squid-main").is_ok()); + assert!(ArchivePrefix::new("nested/path").is_ok()); + } + + #[test] + fn traversal_is_rejected_across_both_separators() { + assert!(ArchivePrefix::new("../escape").is_err()); + assert!(ArchivePrefix::new("nested/../escape").is_err()); + assert!(ArchivePrefix::new("..\\escape").is_err()); + assert!(ArchivePrefix::new("nested\\..\\escape").is_err()); + assert!( + ArchivePrefix::new("dotted-..-name/").is_ok(), + "a component that merely contains dot-dot is not a traversal" + ); + } + + #[test] + fn absolute_and_null_bearing_prefixes_are_rejected() { + assert!(ArchivePrefix::new("/etc").is_err()); + assert!(ArchivePrefix::new("\\windows").is_err()); + assert!(ArchivePrefix::new("good\0bad").is_err()); + } +} diff --git a/knot2/crates/knot-git/src/error.rs b/knot2/crates/knot-git/src/error.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-git/src/error.rs @@ -0,0 +1,86 @@ +use std::path::PathBuf; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SelectionLimit { + Objects, + Time, +} + +impl std::fmt::Display for SelectionLimit { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + SelectionLimit::Objects => "object-set limit", + SelectionLimit::Time => "wall-clock budget", + }) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum GitError { + #[error("repository already exists at {0}")] + AlreadyExists(PathBuf), + #[error("open repository at {path}: {message}")] + Open { path: PathBuf, message: String }, + #[error("create repository at {path}: {message}")] + Create { path: PathBuf, message: String }, + #[error("remove repository at {path}: {message}")] + Remove { path: PathBuf, message: String }, + #[error("reference {name}: {message}")] + Reference { name: String, message: String }, + #[error("atomic ref transaction: {0}")] + AtomicRefs(String), + #[error("fsync {path}: {message}")] + Fsync { path: PathBuf, message: String }, + #[error("atomic write to {path}: {message}")] + Write { path: PathBuf, message: String }, + #[error("repo DID {0} maps to unsafe on-disk path component")] + UnsafeRepoDid(String), + #[error("repo DID {0} is reserved for knot meta-repo and is never served as user repo")] + ReservedDid(String), + #[error("{0} exceeds maximum supported depth")] + DepthExceeded(&'static str), + #[error("revision walk: {0}")] + RevWalk(String), + #[error("upload-pack selection exceeded its {0}")] + Selection(SelectionLimit), + #[error("object not found: {0}")] + ObjectNotFound(knot_types::Oid), + #[error("remove loose object {oid}: {message}")] + RemoveObject { + oid: knot_types::Oid, + message: String, + }, + #[error("object {oid} is corrupt: {message}")] + Corrupt { + oid: knot_types::Oid, + message: String, + }, + #[error("object {oid} isn't {expected}")] + ObjectType { + oid: knot_types::Oid, + expected: &'static str, + }, + #[error("decode object: {0}")] + Decode(String), + #[error("git backend: {0}")] + Backend(String), + #[error("object staging: {0}")] + Staging(String), + #[error("repository config at {path}: {message}")] + Config { path: PathBuf, message: String }, + #[error("repository maintenance: {0}")] + Maintenance(String), +} + +impl From for GitError { + fn from(error: knot_resource::FsError) -> Self { + GitError::Write { + path: error.path, + message: error.source.to_string(), + } + } +} + +pub(crate) fn backend(error: impl std::fmt::Display) -> GitError { + GitError::Backend(error.to_string()) +} diff --git a/knot2/crates/knot-git/src/instrument.rs b/knot2/crates/knot-git/src/instrument.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-git/src/instrument.rs @@ -0,0 +1,1 @@ +knot_types::read_counter!(OdbReads, record_read, odb_reads, reset_odb_reads, measure); diff --git a/knot2/crates/knot-git/src/lib.rs b/knot2/crates/knot-git/src/lib.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-git/src/lib.rs @@ -0,0 +1,57 @@ +mod archive; +mod bitmap; +mod error; +#[cfg(feature = "instrument")] +pub mod instrument; +mod maintenance; +mod objects; +mod patch; +mod patch_apply; +mod patch_parse; +mod reads; +mod repo; +mod staging; + +pub use archive::{ArchiveFormat, ArchivePrefix}; +pub use bitmap::{reachable_via_bitmap, verbatim_clone_pack, write_bitmap, write_midx_bitmap}; +pub use error::{GitError, SelectionLimit}; +pub use maintenance::{PackRefsReport, ReflogReport}; +pub use objects::{ + BlobReader, Commit, CommitChangeId, CommitDepth, CommitRange, Comparison, Deepen, EntryKind, + FileChange, Filter, Haves, Identity, MAX_TREE_DEPTH, PackBudget, PackSelection, ShallowCommits, + ShallowPlan, Tree, TreeDepth, TreeEntry, Wants, +}; +pub use patch::{ + FilePatch, Hunk, HunkLine, LineCount, LineNumber, LineOp, MAX_DIFF_BLOB_BYTES, PatchRange, + PatchStatus, +}; +pub use patch_apply::{ + ApplyError, ApplyOutcome, Conflict, ConflictReason, NewCommit, PatchApplier, StagedAction, + StagedChange, +}; +pub use patch_parse::{ + FileIntent, MailPatch, ParsedFile, PatchParseError, PatchPayload, is_format_patch, + parse_mailbox, parse_mailbox_bounded, parse_patch, parse_patch_bounded, +}; +pub use reads::{ + AnnotatedTag, BranchInfo, BranchTip, LastCommit, LogLimit, LogSkip, PathEntry, SizedEntry, + Submodule, TagInfo, +}; +pub use repo::{ + AdvertScope, HeadRef, Layout, PackHash, PackfileUri, PackfileUrl, RefRecord, RefTxn, RefUpdate, + ReflogUpdate, Repo, is_branch, is_public_ref, is_reserved, knot_shard, repo_shard, + screens_reserved, +}; +pub use staging::{INCOMING_PREFIX, Staging}; + +#[doc(hidden)] +pub mod fuzz { + pub fn patch(data: &[u8]) { + let text = String::from_utf8_lossy(data); + let _ = crate::is_format_patch(&text); + let _ = crate::parse_patch(&text); + let _ = crate::parse_mailbox(&text); + let mid = data.len() / 2; + let _ = crate::patch_apply::apply_delta(&data[..mid], &data[mid..]); + } +} diff --git a/knot2/crates/knot-git/src/maintenance.rs b/knot2/crates/knot-git/src/maintenance.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-git/src/maintenance.rs @@ -0,0 +1,355 @@ +use std::path::Path; + +use gix::lock::acquire::Fail; +use gix::refs::transaction::{Change, LogChange, PreviousValue, RefEdit, RefLog}; +use gix::refs::{FullName, Target, file::transaction::PackedRefs}; +use knot_types::UnixSeconds; + +use crate::error::{GitError, backend}; +use crate::repo::{Repo, fsync_if_present}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PackRefsReport { + pub packed: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ReflogReport { + pub files: usize, + pub dropped: usize, +} + +impl Repo { + pub fn pack_refs(&self) -> Result { + self.locked(|| self.pack_refs_locked()) + } + + fn pack_refs_locked(&self) -> Result { + clear_stale_ref_locks(self.git().git_dir()); + let references = self.git().references().map_err(backend)?; + let edits: Vec = references + .all() + .map_err(backend)? + .filter_map(Result::ok) + .filter_map(|reference| { + let oid = reference.target().try_id()?.to_owned(); + let name: FullName = reference.name().to_owned(); + Some(RefEdit { + change: Change::Update { + log: LogChange { + mode: RefLog::AndReference, + force_create_reflog: false, + message: "knot pack-refs".into(), + }, + expected: PreviousValue::Any, + new: Target::Object(oid), + }, + name, + deref: false, + }) + }) + .collect(); + let packed = edits.len(); + if packed == 0 { + return Ok(PackRefsReport { packed }); + } + let committer: Option> = None; + self.git() + .refs + .transaction() + .packed_refs( + PackedRefs::DeletionsAndNonSymbolicUpdatesRemoveLooseSourceReference(Box::new( + &self.git().objects, + )), + ) + .prepare(edits, Fail::Immediately, Fail::Immediately) + .map_err(backend)? + .commit(committer) + .map_err(backend)?; + let git_dir = self.git().git_dir(); + fsync_if_present(&git_dir.join("packed-refs"))?; + fsync_if_present(&git_dir.join("refs"))?; + fsync_if_present(git_dir)?; + Ok(PackRefsReport { packed }) + } + + pub fn expire_reflogs(&self, floor_seconds: UnixSeconds) -> Result { + self.with_ref_lock(|| self.expire_reflogs_locked(floor_seconds)) + } + + fn expire_reflogs_locked(&self, floor_seconds: UnixSeconds) -> Result { + let logs_dir = self.git().git_dir().join("logs"); + if !logs_dir.exists() { + return Ok(ReflogReport { + files: 0, + dropped: 0, + }); + } + let touched_dirs = walkdir::WalkDir::new(&logs_dir) + .into_iter() + .filter_map(Result::ok) + .filter(|entry| entry.file_type().is_file()) + .map(|entry| entry.into_path()) + .filter(|path| { + if is_maintenance_temp(path) { + let _ = std::fs::remove_file(path); + false + } else { + true + } + }) + .try_fold( + (0usize, 0usize, std::collections::BTreeSet::new()), + |(files, dropped, mut dirs), path| { + let removed = expire_reflog_file(&path, floor_seconds)?; + if let Some(parent) = path.parent() { + dirs.insert(parent.to_path_buf()); + } + Ok::<_, GitError>((files + 1, dropped + removed, dirs)) + }, + )?; + let (files, dropped, dirs) = touched_dirs; + dirs.iter().try_for_each(|dir| fsync_if_present(dir))?; + Ok(ReflogReport { files, dropped }) + } +} + +fn clear_stale_ref_locks(git_dir: &Path) { + let _ = std::fs::remove_file(git_dir.join("packed-refs.lock")); + walkdir::WalkDir::new(git_dir.join("refs")) + .into_iter() + .filter_map(Result::ok) + .filter(|entry| entry.file_type().is_file()) + .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "lock")) + .for_each(|entry| { + let _ = std::fs::remove_file(entry.path()); + }); +} + +fn is_maintenance_temp(path: &Path) -> bool { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.contains(".knot-tmp.")) +} + +fn expire_reflog_file(path: &Path, floor_seconds: UnixSeconds) -> Result { + let raw = std::fs::read(path).map_err(|error| GitError::Maintenance(error.to_string()))?; + if raw.is_empty() { + return Ok(0); + } + let lines: Vec<&[u8]> = split_keep_lines(&raw); + let total = lines.len(); + let last_index = total - 1; + let kept: Vec<&[u8]> = lines + .iter() + .enumerate() + .filter(|(index, line)| { + *index == last_index + || reflog_line_seconds(line).is_none_or(|secs| secs >= floor_seconds) + }) + .map(|(_, line)| *line) + .collect(); + let dropped = total - kept.len(); + if dropped == 0 { + return Ok(0); + } + let rewritten: Vec = kept.concat(); + rewrite_atomic(path, &rewritten)?; + Ok(dropped) +} + +fn split_keep_lines(raw: &[u8]) -> Vec<&[u8]> { + let mut out = Vec::new(); + let mut start = 0usize; + raw.iter().enumerate().for_each(|(index, byte)| { + if *byte == b'\n' { + out.push(&raw[start..=index]); + start = index + 1; + } + }); + if start < raw.len() { + out.push(&raw[start..]); + } + out +} + +fn reflog_line_seconds(line: &[u8]) -> Option { + let tab = line.iter().position(|byte| *byte == b'\t')?; + let before = &line[..tab]; + let text = std::str::from_utf8(before).ok()?; + let mut tokens = text.split_whitespace().rev(); + let _tz = tokens.next()?; + tokens.next()?.parse::().ok().map(UnixSeconds::new) +} + +fn rewrite_atomic(path: &Path, contents: &[u8]) -> Result<(), GitError> { + knot_resource::atomic_write_bytes(path, contents, knot_resource::FileMode::Inherited)?; + fsync_if_present(path) +} + +#[cfg(test)] +mod tests { + use knot_types::{AuthorName, BranchName, Email, Oid, RefName, RepoDid, UnixSeconds}; + + use crate::{EntryKind, Identity, Layout, NewCommit, RefUpdate, StagedAction, StagedChange}; + + // ah yes, of course, little johnny 4b + const EMPTY_TREE: &str = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"; + + fn identity() -> Identity { + Identity { + name: AuthorName::new("nel"), + email: Email::new("nel@oyster.cafe"), + time: UnixSeconds::new(1_700_000_000), + offset_seconds: 0, + } + } + + fn commit_on(repo: &crate::Repo, body: u8, parent: Option) -> Oid { + let tree = repo + .write_staged_tree( + Oid::from_hex(EMPTY_TREE).unwrap(), + &[StagedChange { + path: knot_types::RepoPath::new(format!("file{body}.txt")).unwrap(), + action: StagedAction::Put { + content: vec![body], + kind: EntryKind::Blob, + }, + }], + ) + .unwrap(); + repo.write_commit(&NewCommit { + tree, + parents: parent.into_iter().collect(), + author: identity(), + committer: identity(), + message: format!("commit {body}"), + extra_headers: Vec::new(), + }) + .unwrap() + } + + #[test] + fn pack_refs_moves_loose_refs_into_packed_refs_and_resolves() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()).with_default_branch(BranchName::new("main").unwrap()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let repo = layout.create(&did).unwrap(); + let main = RefName::new("refs/heads/main").unwrap(); + let side = RefName::new("refs/heads/side").unwrap(); + + let base = commit_on(&repo, 0, None); + let tip = commit_on(&repo, 1, Some(base)); + repo.update_ref(&RefUpdate::Create { + name: main.clone(), + new: tip, + }) + .unwrap(); + repo.update_ref(&RefUpdate::Create { + name: side.clone(), + new: base, + }) + .unwrap(); + + let git_dir = repo.git().git_dir().to_path_buf(); + assert!(git_dir.join("refs/heads/main").exists()); + + let report = repo.pack_refs().unwrap(); + assert!(report.packed >= 2, "both branches are packed"); + assert!( + git_dir.join("packed-refs").exists(), + "packed-refs file is written" + ); + assert!( + !git_dir.join("refs/heads/main").exists(), + "loose ref file is removed once packed" + ); + assert_eq!(repo.find_ref(&main).unwrap(), Some(tip)); + assert_eq!(repo.find_ref(&side).unwrap(), Some(base)); + } + + #[test] + fn pack_refs_clears_stale_lock_files_left_by_a_crash() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()).with_default_branch(BranchName::new("main").unwrap()); + let did = RepoDid::new("did:plc:barnacle").unwrap(); + let repo = layout.create(&did).unwrap(); + let main = RefName::new("refs/heads/main").unwrap(); + + let tip = commit_on(&repo, 0, None); + repo.update_ref(&RefUpdate::Create { + name: main.clone(), + new: tip, + }) + .unwrap(); + + let git_dir = repo.git().git_dir().to_path_buf(); + std::fs::write(git_dir.join("packed-refs.lock"), b"").unwrap(); + std::fs::write(git_dir.join("refs/heads/main.lock"), b"").unwrap(); + + let report = repo + .pack_refs() + .expect("crashed prior run's stale locks mustn't wedge next pack-refs"); + assert!(report.packed >= 1); + assert!( + !git_dir.join("packed-refs.lock").exists(), + "stale packed-refs lock is cleared" + ); + assert!( + !git_dir.join("refs/heads/main.lock").exists(), + "stale per-ref lock is cleared" + ); + assert_eq!(repo.find_ref(&main).unwrap(), Some(tip)); + } + + #[test] + fn expire_reflogs_drops_old_entries_but_keeps_the_newest() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()).with_default_branch(BranchName::new("main").unwrap()); + let did = RepoDid::new("did:plc:limpet").unwrap(); + let repo = layout.create(&did).unwrap(); + let main = RefName::new("refs/heads/main").unwrap(); + + let c0 = commit_on(&repo, 0, None); + let c1 = commit_on(&repo, 1, Some(c0)); + let c2 = commit_on(&repo, 2, Some(c1)); + repo.update_ref(&RefUpdate::Create { + name: main.clone(), + new: c0, + }) + .unwrap(); + repo.update_ref(&RefUpdate::Update { + name: main.clone(), + old: c0, + new: c1, + }) + .unwrap(); + repo.update_ref(&RefUpdate::Update { + name: main.clone(), + old: c1, + new: c2, + }) + .unwrap(); + + let log_path = repo.git().git_dir().join("logs/refs/heads/main"); + let before = std::fs::read(&log_path).unwrap(); + let line_count = before.iter().filter(|byte| **byte == b'\n').count(); + assert_eq!(line_count, 3, "three ref updates leave three reflog lines"); + + let report = repo.expire_reflogs(UnixSeconds::new(i64::MAX / 2)).unwrap(); + assert!( + report.files >= 2, + "main branch reflog and HEAD reflog are both rewritten" + ); + assert_eq!( + report.dropped, 4, + "future floor drops all but newest line of each of two reflogs" + ); + let after = std::fs::read(&log_path).unwrap(); + assert_eq!(after.iter().filter(|byte| **byte == b'\n').count(), 1); + assert_eq!(repo.find_ref(&main).unwrap(), Some(c2)); + + let untouched = repo.expire_reflogs(UnixSeconds::new(0)).unwrap(); + assert_eq!(untouched.dropped, 0, "zero floor keeps everything"); + } +} diff --git a/knot2/crates/knot-git/src/objects.rs b/knot2/crates/knot-git/src/objects.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-git/src/objects.rs @@ -0,0 +1,1555 @@ +use std::collections::{HashMap, HashSet, VecDeque}; +use std::io::{BufRead, Read}; +use std::ops::ControlFlow; +use std::path::PathBuf; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; + +use knot_types::{AuthorName, Email, ObjectCount, Oid, ParseError, RepoPath, UnixSeconds}; + +use crate::error::{GitError, SelectionLimit}; +use crate::repo::Repo; + +// why? idk. should we let this be deeper +pub const MAX_TREE_DEPTH: usize = 1024; +const MAX_TAG_DEPTH: usize = 32; + +#[derive(Debug, Clone, Copy)] +pub struct Wants<'a>(&'a [Oid]); + +#[derive(Debug, Clone, Copy)] +pub struct Haves<'a>(&'a [Oid]); + +#[derive(Debug, Clone, Copy)] +pub struct ShallowCommits<'a>(&'a [Oid]); + +impl<'a> Wants<'a> { + pub fn new(oids: &'a [Oid]) -> Self { + Self(oids) + } + + pub fn as_slice(self) -> &'a [Oid] { + self.0 + } +} + +impl<'a> Haves<'a> { + pub fn new(oids: &'a [Oid]) -> Self { + Self(oids) + } + + pub fn as_slice(self) -> &'a [Oid] { + self.0 + } +} + +impl<'a> ShallowCommits<'a> { + pub fn new(oids: &'a [Oid]) -> Self { + Self(oids) + } + + pub fn as_slice(self) -> &'a [Oid] { + self.0 + } +} + +enum Peeled { + Commit(gix::ObjectId), + Direct(gix::ObjectId), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Identity { + pub name: AuthorName, + pub email: Email, + pub time: UnixSeconds, + pub offset_seconds: i32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Commit { + pub id: Oid, + pub tree: Oid, + pub parents: Vec, + pub author: Identity, + pub committer: Identity, + pub message: String, + pub pgp_signature: Option, + pub merge_tag: Option, + pub extra_headers: Vec<(String, Vec)>, +} + +impl Commit { + pub fn change_id(&self) -> Option { + self.extra_headers + .iter() + .find(|(name, _)| name == "change-id") + .and_then(|(_, value)| std::str::from_utf8(value).ok()) + .and_then(|value| CommitChangeId::new(value).ok()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CommitChangeId(String); + +impl CommitChangeId { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + let valid = + !value.is_empty() && value.len() <= 100 && value.chars().all(|c| c.is_ascii_graphic()); + match valid { + true => Ok(Self(value)), + false => Err(ParseError::Invalid { + kind: "commit change-id", + value, + }), + } + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl std::fmt::Display for CommitChangeId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.pad(&self.0) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EntryKind { + Tree, + Blob, + BlobExecutable, + Link, + Commit, +} + +impl EntryKind { + pub fn mode_octal(self) -> &'static str { + match self { + EntryKind::Tree => "0040000", + EntryKind::Blob => "0100644", + EntryKind::BlobExecutable => "0100755", + EntryKind::Link => "0120000", + EntryKind::Commit => "0160000", + } + } + + pub fn is_file(self) -> bool { + matches!(self, EntryKind::Blob | EntryKind::BlobExecutable) + } + + pub fn from_git_mode(mode: &str) -> Option { + match mode.trim() { + "100644" | "100664" => Some(EntryKind::Blob), + "100755" => Some(EntryKind::BlobExecutable), + "120000" => Some(EntryKind::Link), + "160000" => Some(EntryKind::Commit), + "040000" | "40000" => Some(EntryKind::Tree), + _ => None, + } + } +} + +impl From for gix::objs::tree::EntryKind { + fn from(kind: EntryKind) -> Self { + match kind { + EntryKind::Tree => Self::Tree, + EntryKind::Blob => Self::Blob, + EntryKind::BlobExecutable => Self::BlobExecutable, + EntryKind::Link => Self::Link, + EntryKind::Commit => Self::Commit, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TreeEntry { + pub name: String, + pub oid: Oid, + pub kind: EntryKind, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Tree { + pub entries: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum FileChange { + Added { + path: RepoPath, + oid: Oid, + }, + Deleted { + path: RepoPath, + oid: Oid, + }, + Modified { + path: RepoPath, + old: Oid, + new: Oid, + }, + Renamed { + from: RepoPath, + to: RepoPath, + old: Oid, + new: Oid, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Comparison { + pub commits: Vec, + pub changes: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct CommitRange { + pub base: Oid, + pub head: Oid, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct TreeDepth(u32); + +impl TreeDepth { + pub const fn new(depth: u32) -> Self { + Self(depth) + } + + pub const fn deeper(self) -> Self { + Self(self.0.saturating_add(1)) + } + + const fn is_exhausted(self) -> bool { + self.0 == 0 + } + + const fn shallower(self) -> Self { + Self(self.0.saturating_sub(1)) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Filter { + None, + BlobNone, + BlobLimit(u64), + TreeDepth(TreeDepth), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct CommitDepth(u32); + +impl CommitDepth { + pub const fn new(depth: u32) -> Self { + Self(depth) + } + + const fn deeper(self) -> Self { + Self(self.0.saturating_add(1)) + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Deepen { + pub depth: Option, + pub since: Option, + pub not: Vec, + pub relative: bool, +} + +impl Deepen { + pub fn is_shallow_request(&self) -> bool { + self.depth.is_some() || self.since.is_some() || !self.not.is_empty() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ShallowPlan { + pub commits: Vec, + pub shallow: Vec, + pub unshallow: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PackSelection { + pub send: Vec, + pub client_has: HashSet, +} + +#[derive(Debug, Clone, Copy)] +pub struct PackBudget { + max_objects: ObjectCount, + stall: Option, +} + +impl PackBudget { + pub fn new(max_objects: ObjectCount, stall: Duration) -> Self { + Self { + max_objects, + stall: Some(stall), + } + } + + pub fn unbounded() -> Self { + Self { + max_objects: ObjectCount::new(usize::MAX), + stall: None, + } + } +} + +#[derive(Clone, Copy)] +pub(crate) struct Walked { + budget: PackBudget, + count: usize, + deadline: Option, +} + +impl Walked { + pub(crate) fn new(budget: PackBudget) -> Self { + Self { + budget, + count: 0, + deadline: budget.stall.map(|stall| Instant::now() + stall), + } + } + + pub(crate) fn tick(&mut self) -> Result<(), GitError> { + self.count += 1; + if self.count > self.budget.max_objects.get() { + return Err(GitError::Selection(SelectionLimit::Objects)); + } + advance_stall(&mut self.deadline, self.budget.stall) + } +} + +fn advance_stall(deadline: &mut Option, stall: Option) -> Result<(), GitError> { + if let (Some(deadline), Some(stall)) = (deadline.as_mut(), stall) { + let now = Instant::now(); + if now >= *deadline { + return Err(GitError::Selection(SelectionLimit::Time)); + } + *deadline = now + stall; + } + Ok(()) +} + +const MAX_LOOSE_HEADER: usize = 64; + +const PARALLEL_SELECT_MIN: usize = 4096; + +struct SharedWalk<'a> { + counter: &'a AtomicUsize, + max_objects: usize, + stall: Option, + deadline: Option, +} + +impl<'a> SharedWalk<'a> { + fn new(counter: &'a AtomicUsize, budget: &PackBudget) -> Self { + Self { + counter, + max_objects: budget.max_objects.get(), + stall: budget.stall, + deadline: budget.stall.map(|stall| Instant::now() + stall), + } + } + + fn tick(&mut self) -> Result<(), GitError> { + let count = self.counter.fetch_add(1, Ordering::Relaxed) + 1; + if count > self.max_objects { + return Err(GitError::Selection(SelectionLimit::Objects)); + } + advance_stall(&mut self.deadline, self.stall) + } +} + +pub enum BlobReader { + Loose(std::io::BufReader>), + Packed(std::io::Cursor>), +} + +impl Read for BlobReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + match self { + BlobReader::Loose(reader) => reader.read(buf), + BlobReader::Packed(reader) => reader.read(buf), + } + } +} + +fn skip_loose_header(reader: &mut impl std::io::BufRead, oid: Oid) -> Result<(), GitError> { + let corrupt = |message: String| GitError::Corrupt { oid, message }; + let mut header = Vec::with_capacity(MAX_LOOSE_HEADER); + reader + .take(MAX_LOOSE_HEADER as u64) + .read_until(0, &mut header) + .map_err(|error| corrupt(error.to_string()))?; + if header.last() != Some(&0) || !header.starts_with(b"blob ") { + return Err(corrupt("loose object header isn't blob".to_string())); + } + Ok(()) +} + +pub(crate) fn identity(signature: gix::actor::SignatureRef<'_>) -> Result { + let time = signature + .time() + .map_err(|error| GitError::Decode(error.to_string()))?; + Ok(Identity { + name: AuthorName::new(signature.name.to_string()), + email: Email::new(signature.email.to_string()), + time: UnixSeconds::new(time.seconds), + offset_seconds: time.offset, + }) +} + +pub(crate) fn signature(identity: &Identity) -> gix::actor::Signature { + let clean = |raw: &str| raw.replace(['<', '>'], "").trim().to_string(); + gix::actor::Signature { + name: clean(identity.name.as_str()).into(), + email: clean(identity.email.as_str()).into(), + time: gix::date::Time { + seconds: identity.time.get(), + offset: identity.offset_seconds, + }, + } +} + +pub(crate) fn map_kind(kind: gix::objs::tree::EntryKind) -> EntryKind { + use gix::objs::tree::EntryKind as Source; + match kind { + Source::Tree => EntryKind::Tree, + Source::Blob => EntryKind::Blob, + Source::BlobExecutable => EntryKind::BlobExecutable, + Source::Link => EntryKind::Link, + Source::Commit => EntryKind::Commit, + } +} + +impl Repo { + fn load_object(&self, oid: Oid) -> Result, GitError> { + #[cfg(feature = "instrument")] + crate::instrument::record_read(); + match self.git().try_find_object(oid.object_id()) { + Ok(Some(object)) => Ok(object), + Ok(None) => Err(GitError::ObjectNotFound(oid)), + Err(error) => Err(GitError::Corrupt { + oid, + message: error.to_string(), + }), + } + } + + pub fn find_commit(&self, oid: Oid) -> Result { + let object = self.load_object(oid)?; + let commit = object.try_into_commit().map_err(|_| GitError::ObjectType { + oid, + expected: "commit", + })?; + let tree = Oid::from( + commit + .tree_id() + .map_err(|error| GitError::Decode(error.to_string()))? + .detach(), + ); + let parents = commit + .parent_ids() + .map(|id| Oid::from(id.detach())) + .collect(); + let author = identity( + commit + .author() + .map_err(|error| GitError::Decode(error.to_string()))?, + )?; + let committer = identity( + commit + .committer() + .map_err(|error| GitError::Decode(error.to_string()))?, + )?; + let message = commit + .message_raw() + .map_err(|error| GitError::Decode(error.to_string()))? + .to_string(); + let decoded = commit + .decode() + .map_err(|error| GitError::Decode(error.to_string()))?; + let (mut pgp_signature, mut merge_tag) = (None, None); + let extra_headers = decoded + .extra_headers + .iter() + .filter_map(|(name, value)| match name.to_string().as_str() { + "gpgsig" => { + pgp_signature = Some(value.to_string()); + None + } + "mergetag" => { + merge_tag = Some(value.to_string()); + None + } + other => Some((other.to_string(), value.to_vec())), + }) + .collect(); + Ok(Commit { + id: oid, + tree, + parents, + author, + committer, + message, + pgp_signature, + merge_tag, + extra_headers, + }) + } + + pub fn find_tree(&self, oid: Oid) -> Result { + let object = self.load_object(oid)?; + let tree = object.try_into_tree().map_err(|_| GitError::ObjectType { + oid, + expected: "tree", + })?; + let decoded = tree + .decode() + .map_err(|error| GitError::Decode(error.to_string()))?; + let entries = decoded + .entries + .iter() + .map(|entry| TreeEntry { + name: entry.filename.to_string(), + oid: Oid::from(entry.oid.to_owned()), + kind: map_kind(entry.mode.kind()), + }) + .collect(); + Ok(Tree { entries }) + } + + pub fn blob_size(&self, oid: Oid) -> Result { + match self.git().try_find_header(oid.object_id()) { + Ok(Some(header)) if header.kind() == gix::object::Kind::Blob => Ok(header.size()), + Ok(Some(_)) => Err(GitError::ObjectType { + oid, + expected: "blob", + }), + Ok(None) => Err(GitError::ObjectNotFound(oid)), + Err(error) => Err(GitError::Corrupt { + oid, + message: error.to_string(), + }), + } + } + + pub fn read_blob(&self, oid: Oid) -> Result, GitError> { + let object = self.load_object(oid)?; + let mut blob = object.try_into_blob().map_err(|_| GitError::ObjectType { + oid, + expected: "blob", + })?; + Ok(blob.take_data()) + } + + fn loose_object_path(&self, oid: Oid) -> PathBuf { + let hex = oid.to_hex(); + let (shard, rest) = hex.split_at(2); + self.git().git_dir().join("objects").join(shard).join(rest) + } + + pub fn remove_loose_object(&self, oid: Oid) -> Result<(), GitError> { + match std::fs::remove_file(self.loose_object_path(oid)) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(GitError::RemoveObject { + oid, + message: error.to_string(), + }), + } + } + + pub fn open_blob(&self, oid: Oid) -> Result<(u64, BlobReader), GitError> { + let header = match self.git().try_find_header(oid.object_id()) { + Ok(Some(header)) => header, + Ok(None) => return Err(GitError::ObjectNotFound(oid)), + Err(error) => { + return Err(GitError::Corrupt { + oid, + message: error.to_string(), + }); + } + }; + if header.kind() != gix::object::Kind::Blob { + return Err(GitError::ObjectType { + oid, + expected: "blob", + }); + } + let size = header.size(); + match std::fs::File::open(self.loose_object_path(oid)) { + Ok(file) => { + let mut reader = std::io::BufReader::new(flate2::read::ZlibDecoder::new(file)); + skip_loose_header(&mut reader, oid)?; + Ok((size, BlobReader::Loose(reader))) + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(( + size, + BlobReader::Packed(std::io::Cursor::new(self.read_blob(oid)?)), + )), + Err(error) => Err(GitError::Corrupt { + oid, + message: error.to_string(), + }), + } + } + + fn graph_tree_and_parents(&self, commit: Oid) -> Option<(gix::ObjectId, Vec)> { + let graph = self.commit_graph()?; + let node = graph.commit_by_id(commit.object_id())?; + let tree = node.root_tree_id().to_owned(); + let parents = node + .iter_parents() + .filter_map(Result::ok) + .map(|position| graph.commit_at(position).id().to_owned()) + .collect(); + Some((tree, parents)) + } + + pub(crate) fn commit_tree(&self, commit: Oid) -> Result { + if let Some(node) = self + .commit_graph() + .and_then(|graph| graph.commit_by_id(commit.object_id())) + { + return Ok(node.root_tree_id().to_owned()); + } + let object = self.load_object(commit)?; + let commit_object = object.try_into_commit().map_err(|_| GitError::ObjectType { + oid: commit, + expected: "commit", + })?; + Ok(commit_object + .tree_id() + .map_err(|error| GitError::Decode(error.to_string()))? + .detach()) + } + + pub fn diff(&self, range: CommitRange) -> Result, GitError> { + let old_tree_oid = self.commit_tree(range.base)?; + let new_tree_oid = self.commit_tree(range.head)?; + let old_tree = self + .load_object(Oid::from(old_tree_oid))? + .try_into_tree() + .map_err(|error| GitError::Backend(error.to_string()))?; + let new_tree = self + .load_object(Oid::from(new_tree_oid))? + .try_into_tree() + .map_err(|error| GitError::Backend(error.to_string()))?; + + let mut changes = Vec::new(); + old_tree + .changes() + .map_err(|error| GitError::Backend(error.to_string()))? + .for_each_to_obtain_tree(&new_tree, |change| { + use gix::object::tree::diff::Change; + let tree_path = |location: &gix::bstr::BStr| { + RepoPath::new(location.to_string()) + .map_err(|error| GitError::Decode(error.to_string())) + }; + let mapped = match change { + Change::Addition { location, id, .. } => FileChange::Added { + path: tree_path(location)?, + oid: Oid::from(id.detach()), + }, + Change::Deletion { location, id, .. } => FileChange::Deleted { + path: tree_path(location)?, + oid: Oid::from(id.detach()), + }, + Change::Modification { + location, + previous_id, + id, + .. + } => FileChange::Modified { + path: tree_path(location)?, + old: Oid::from(previous_id.detach()), + new: Oid::from(id.detach()), + }, + Change::Rewrite { + source_location, + location, + source_id, + id, + .. + } => FileChange::Renamed { + from: tree_path(source_location)?, + to: tree_path(location)?, + old: Oid::from(source_id.detach()), + new: Oid::from(id.detach()), + }, + }; + changes.push(mapped); + Ok::<_, GitError>(ControlFlow::Continue(())) + }) + .map_err(|error| GitError::Backend(error.to_string()))?; + Ok(changes) + } + + pub fn compare(&self, range: CommitRange) -> Result { + let commits = self.rev_walk(Wants::new(&[range.head]), Haves::new(&[range.base]))?; + let changes = self.diff(range)?; + Ok(Comparison { commits, changes }) + } + + fn peel( + &self, + oid: gix::ObjectId, + tags: &mut Vec, + depth: usize, + ) -> Result { + if depth == 0 { + return Err(GitError::DepthExceeded("annotated tag chain")); + } + let object = self.load_object(Oid::from(oid))?; + match object.kind { + gix::object::Kind::Commit => Ok(Peeled::Commit(oid)), + gix::object::Kind::Tree | gix::object::Kind::Blob => Ok(Peeled::Direct(oid)), + gix::object::Kind::Tag => { + tags.push(Oid::from(oid)); + let target = object + .try_into_tag() + .map_err(|error| GitError::Decode(error.to_string()))? + .target_id() + .map_err(|error| GitError::Decode(error.to_string()))? + .detach(); + self.peel(target, tags, depth - 1) + } + } + } + + pub fn peeled_target(&self, oid: Oid) -> Result, GitError> { + let object = self.load_object(oid)?; + match object.kind { + gix::object::Kind::Tag => { + let mut tags = Vec::new(); + let peeled = match self.peel(oid.object_id(), &mut tags, MAX_TAG_DEPTH)? { + Peeled::Commit(id) | Peeled::Direct(id) => Oid::from(id), + }; + Ok(Some(peeled)) + } + _ => Ok(None), + } + } + + pub(crate) fn commit_tree_and_parents( + &self, + commit: Oid, + ) -> Result<(gix::ObjectId, Vec), GitError> { + if let Some(found) = self.graph_tree_and_parents(commit) { + return Ok(found); + } + let object = self.load_object(commit)?; + let commit_object = object.try_into_commit().map_err(|_| GitError::ObjectType { + oid: commit, + expected: "commit", + })?; + let tree = commit_object + .tree_id() + .map_err(|error| GitError::Decode(error.to_string()))? + .detach(); + let parents = commit_object.parent_ids().map(|id| id.detach()).collect(); + Ok((tree, parents)) + } + + fn blob_kept(&self, oid: gix::ObjectId, filter: Filter) -> Result { + match filter { + Filter::None | Filter::TreeDepth(_) => Ok(true), + Filter::BlobNone => Ok(false), + Filter::BlobLimit(limit) => { + let header = self + .git() + .try_find_header(oid) + .map_err(|error| GitError::Corrupt { + oid: Oid::from(oid), + message: error.to_string(), + })? + .ok_or(GitError::ObjectNotFound(Oid::from(oid)))?; + Ok(header.size() < limit) + } + } + } + + fn walk_tree( + &self, + tree: gix::ObjectId, + seen: &mut HashSet, + visit: &mut dyn FnMut(Oid), + filter: Filter, + nesting: usize, + walked: &mut Walked, + ) -> Result<(), GitError> { + if nesting == 0 { + return Err(GitError::DepthExceeded("tree nesting")); + } + if tree == gix::ObjectId::empty_tree(self.git().object_hash()) { + return Ok(()); + } + if !seen.insert(tree) { + return Ok(()); + } + visit(Oid::from(tree)); + walked.tick()?; + let object = self.load_object(Oid::from(tree))?; + let decoded = object + .try_into_tree() + .map_err(|error| GitError::Decode(error.to_string()))?; + let decoded = decoded + .decode() + .map_err(|error| GitError::Decode(error.to_string()))?; + decoded.entries.iter().try_for_each(|entry| { + let oid = entry.oid.to_owned(); + match entry.mode.kind() { + gix::objs::tree::EntryKind::Tree => { + self.walk_tree(oid, seen, visit, filter, nesting - 1, walked) + } + gix::objs::tree::EntryKind::Commit => Ok(()), + _ => { + if !seen.contains(&oid) && self.blob_kept(oid, filter)? { + seen.insert(oid); + visit(Oid::from(oid)); + walked.tick()?; + } + Ok(()) + } + } + }) + } + + #[allow(clippy::too_many_arguments)] + fn walk_tree_depth( + &self, + tree: gix::ObjectId, + remaining: TreeDepth, + seen: &mut HashSet, + expanded: &mut HashMap, + visit: &mut dyn FnMut(Oid), + nesting: usize, + walked: &mut Walked, + ) -> Result<(), GitError> { + if nesting == 0 { + return Err(GitError::DepthExceeded("tree nesting")); + } + if remaining.is_exhausted() || tree == gix::ObjectId::empty_tree(self.git().object_hash()) { + return Ok(()); + } + if expanded + .get(&tree) + .is_some_and(|deepest| *deepest >= remaining) + { + return Ok(()); + } + expanded.insert(tree, remaining); + if seen.insert(tree) { + visit(Oid::from(tree)); + walked.tick()?; + } + let object = self.load_object(Oid::from(tree))?; + let decoded = object + .try_into_tree() + .map_err(|error| GitError::Decode(error.to_string()))?; + let decoded = decoded + .decode() + .map_err(|error| GitError::Decode(error.to_string()))?; + let next = remaining.shallower(); + decoded.entries.iter().try_for_each(|entry| { + let oid = entry.oid.to_owned(); + match entry.mode.kind() { + gix::objs::tree::EntryKind::Tree => { + self.walk_tree_depth(oid, next, seen, expanded, visit, nesting - 1, walked) + } + gix::objs::tree::EntryKind::Commit => Ok(()), + _ => { + if !next.is_exhausted() && seen.insert(oid) { + visit(Oid::from(oid)); + walked.tick()?; + } + Ok(()) + } + } + }) + } + + fn walk_root_tree( + &self, + tree: gix::ObjectId, + seen: &mut HashSet, + expanded: &mut HashMap, + visit: &mut dyn FnMut(Oid), + filter: Filter, + walked: &mut Walked, + ) -> Result<(), GitError> { + match filter { + Filter::TreeDepth(max) => { + self.walk_tree_depth(tree, max, seen, expanded, visit, MAX_TREE_DEPTH, walked) + } + _ => self.walk_tree(tree, seen, visit, filter, MAX_TREE_DEPTH, walked), + } + } + + fn walk_tree_shared( + &self, + tree: gix::ObjectId, + seen: &scc::HashSet, + out: &mut Vec, + nesting: usize, + walk: &mut SharedWalk, + ) -> Result<(), GitError> { + if nesting == 0 { + return Err(GitError::DepthExceeded("tree nesting")); + } + if tree == gix::ObjectId::empty_tree(self.git().object_hash()) { + return Ok(()); + } + if seen.insert_sync(tree).is_err() { + return Ok(()); + } + out.push(Oid::from(tree)); + walk.tick()?; + let object = self.load_object(Oid::from(tree))?; + let decoded = object + .try_into_tree() + .map_err(|error| GitError::Decode(error.to_string()))?; + let decoded = decoded + .decode() + .map_err(|error| GitError::Decode(error.to_string()))?; + decoded.entries.iter().try_for_each(|entry| { + let oid = entry.oid.to_owned(); + match entry.mode.kind() { + gix::objs::tree::EntryKind::Tree => { + self.walk_tree_shared(oid, seen, out, nesting - 1, walk) + } + gix::objs::tree::EntryKind::Commit => Ok(()), + _ => { + if seen.insert_sync(oid).is_ok() { + out.push(Oid::from(oid)); + walk.tick()?; + } + Ok(()) + } + } + }) + } + + fn walk_send_trees( + &self, + send: &[(Oid, gix::ObjectId, Vec)], + seen: HashSet, + mut out: Vec, + budget: PackBudget, + ) -> Result, GitError> { + let seen: scc::HashSet = seen.into_iter().collect(); + let counter = AtomicUsize::new(out.len()); + let path = self.path().to_owned(); + let walk = + |batch: &[(Oid, gix::ObjectId, Vec)]| -> Result, GitError> { + let local = Repo::open(&path)?; + let mut walk = SharedWalk::new(&counter, &budget); + batch.iter().try_fold( + Vec::new(), + |mut acc, (commit, tree, _)| -> Result, GitError> { + if seen.insert_sync(commit.object_id()).is_ok() { + acc.push(*commit); + } + local.walk_tree_shared( + *tree, + &seen, + &mut acc, + MAX_TREE_DEPTH, + &mut walk, + )?; + Ok(acc) + }, + ) + }; + out.extend(knot_resource::map_chunks(send, walk)?); + Ok(out) + } + + fn collect_direct( + &self, + oid: Oid, + seen: &mut HashSet, + expanded: &mut HashMap, + visit: &mut dyn FnMut(Oid), + filter: Filter, + walked: &mut Walked, + ) -> Result<(), GitError> { + let object = self.load_object(oid)?; + match (object.kind, filter) { + (gix::object::Kind::Tree, Filter::TreeDepth(max)) => self.walk_tree_depth( + oid.object_id(), + max.deeper(), + seen, + expanded, + visit, + MAX_TREE_DEPTH, + walked, + ), + (gix::object::Kind::Tree, _) => { + self.walk_tree(oid.object_id(), seen, visit, filter, MAX_TREE_DEPTH, walked) + } + _ => { + if seen.insert(oid.object_id()) { + visit(oid); + walked.tick()?; + } + Ok(()) + } + } + } + + fn commit_time(&self, commit: gix::ObjectId) -> Result { + let object = self.load_object(Oid::from(commit))?; + let commit = object.try_into_commit().map_err(|_| GitError::ObjectType { + oid: Oid::from(commit), + expected: "commit", + })?; + let time = commit + .committer() + .map_err(|error| GitError::Decode(error.to_string()))? + .time() + .map_err(|error| GitError::Decode(error.to_string()))?; + Ok(UnixSeconds::new(time.seconds)) + } + + pub fn shallow_walk( + &self, + wants: Wants<'_>, + deepen: &Deepen, + client_shallow: ShallowCommits<'_>, + ) -> Result { + let want_commits: Vec = wants + .as_slice() + .iter() + .map(|want| self.peel(want.object_id(), &mut Vec::new(), MAX_TAG_DEPTH)) + .collect::, _>>()? + .into_iter() + .filter_map(|peeled| match peeled { + Peeled::Commit(commit) => Some(commit), + Peeled::Direct(_) => None, + }) + .collect(); + + let excluded: HashSet = if deepen.not.is_empty() { + HashSet::new() + } else { + self.rev_walk(Wants::new(&deepen.not), Haves::new(&[]))? + .into_iter() + .map(Oid::object_id) + .collect() + }; + + let drop = |oid: gix::ObjectId, depth: CommitDepth| -> Result { + if excluded.contains(&oid) { + return Ok(true); + } + if let Some(max) = deepen.depth + && depth > max + { + return Ok(true); + } + if let Some(since) = deepen.since + && self.commit_time(oid)? < since + { + return Ok(true); + } + Ok(false) + }; + + let grafts = self.shallow_grafts()?; + let mut min_depth: HashMap = HashMap::new(); + let mut parents_of: HashMap> = HashMap::new(); + let mut queue: VecDeque<(gix::ObjectId, CommitDepth)> = want_commits + .iter() + .map(|commit| (*commit, CommitDepth::new(1))) + .collect(); + if deepen.relative { + client_shallow + .as_slice() + .iter() + .for_each(|oid| queue.push_back((oid.object_id(), CommitDepth::new(0)))); + } + while let Some((commit, depth)) = queue.pop_front() { + if drop(commit, depth)? { + continue; + } + if min_depth.get(&commit).is_some_and(|seen| *seen <= depth) { + continue; + } + min_depth.insert(commit, depth); + let (_, parents) = self.commit_tree_and_parents(Oid::from(commit))?; + if !grafts.contains(&commit) { + parents + .iter() + .for_each(|parent| queue.push_back((*parent, depth.deeper()))); + } + parents_of.insert(commit, parents); + } + + let included: HashSet = min_depth.keys().copied().collect(); + let boundary: HashSet = included + .iter() + .filter(|commit| { + parents_of + .get(*commit) + .is_some_and(|parents| parents.iter().any(|parent| !included.contains(parent))) + }) + .copied() + .collect(); + + let commits: Vec = min_depth.keys().map(|oid| Oid::from(*oid)).collect(); + let shallow: Vec = boundary.iter().map(|oid| Oid::from(*oid)).collect(); + let unshallow: Vec = client_shallow + .as_slice() + .iter() + .filter(|oid| { + min_depth.contains_key(&oid.object_id()) && !boundary.contains(&oid.object_id()) + }) + .copied() + .collect(); + Ok(ShallowPlan { + commits, + shallow, + unshallow, + }) + } + + pub fn select_shallow_objects( + &self, + wants: Wants, + commits: ShallowCommits, + haves: Haves, + filter: Filter, + budget: PackBudget, + ) -> Result { + let wants = wants.as_slice(); + let commits = commits.as_slice(); + let haves = haves.as_slice(); + let mut walked = Walked::new(budget); + let mut seen: HashSet = HashSet::new(); + let mut expanded: HashMap = HashMap::new(); + let mut have_commits: Vec = Vec::new(); + haves + .iter() + .filter(|have| self.contains(**have)) + .try_for_each(|have| -> Result<(), GitError> { + match self.peel(have.object_id(), &mut Vec::new(), MAX_TAG_DEPTH)? { + Peeled::Commit(commit) => { + have_commits.push(commit); + let tree = self.commit_tree(Oid::from(commit))?; + self.walk_tree( + tree, + &mut seen, + &mut |_| {}, + Filter::None, + MAX_TREE_DEPTH, + &mut walked, + ) + } + Peeled::Direct(direct) => self.collect_direct( + Oid::from(direct), + &mut seen, + &mut expanded, + &mut |_| {}, + Filter::None, + &mut walked, + ), + } + })?; + let client_has: HashSet = seen + .iter() + .copied() + .chain(have_commits) + .map(Oid::from) + .collect(); + + let mut want_tags = Vec::new(); + wants.iter().try_for_each(|want| -> Result<(), GitError> { + self.peel(want.object_id(), &mut want_tags, MAX_TAG_DEPTH) + .map(|_| ()) + })?; + + let mut out: Vec = Vec::new(); + want_tags + .iter() + .try_for_each(|tag| -> Result<(), GitError> { + if seen.insert(tag.object_id()) { + out.push(*tag); + walked.tick()?; + } + Ok(()) + })?; + commits + .iter() + .try_for_each(|commit| -> Result<(), GitError> { + if seen.insert(commit.object_id()) { + out.push(*commit); + walked.tick()?; + } + let tree = self.commit_tree(*commit)?; + self.walk_root_tree( + tree, + &mut seen, + &mut expanded, + &mut |oid| out.push(oid), + filter, + &mut walked, + ) + })?; + Ok(PackSelection { + send: out, + client_has, + }) + } + + pub fn select_pack_objects(&self, wants: Wants, haves: Haves) -> Result, GitError> { + self.select_pack_objects_filtered(wants, haves, Filter::None, PackBudget::unbounded()) + .map(|selection| selection.send) + } + + pub fn clone_roots(&self, wants: &[Oid], budget: PackBudget) -> Result, GitError> { + let mut walked = Walked::new(budget); + let mut want_tags = Vec::new(); + let mut want_commits = Vec::new(); + let mut want_direct = Vec::new(); + wants.iter().try_for_each(|want| -> Result<(), GitError> { + match self.peel(want.object_id(), &mut want_tags, MAX_TAG_DEPTH)? { + Peeled::Commit(commit) => want_commits.push(Oid::from(commit)), + Peeled::Direct(direct) => want_direct.push(Oid::from(direct)), + } + Ok(()) + })?; + let commits = + self.rev_walk_each(Wants::new(&want_commits), Haves::new(&[]), &mut walked)?; + Ok(want_tags + .into_iter() + .chain(commits) + .chain(want_direct) + .collect()) + } + + pub fn reachable_commits( + &self, + tips: &[Oid], + budget: PackBudget, + ) -> Result, GitError> { + let mut walked = Walked::new(budget); + let commits: Vec = self + .peel_to_commits(tips.iter().copied())? + .into_iter() + .map(Oid::from) + .collect(); + Ok(self + .rev_walk_each(Wants::new(&commits), Haves::new(&[]), &mut walked)? + .into_iter() + .collect()) + } + + fn peel_to_commits( + &self, + oids: impl Iterator, + ) -> Result, GitError> { + oids.map(|oid| self.peel(oid.object_id(), &mut Vec::new(), MAX_TAG_DEPTH)) + .filter_map(|peeled| match peeled { + Ok(Peeled::Commit(commit)) => Some(Ok(commit)), + Ok(Peeled::Direct(_)) => None, + Err(error) => Some(Err(error)), + }) + .collect() + } + + pub fn wants_satisfied_by(&self, wants: Wants, haves: Haves) -> Result { + let wants = wants.as_slice(); + let haves = haves.as_slice(); + let commons: HashSet = self + .peel_to_commits(haves.iter().copied().filter(|have| self.contains(*have)))? + .into_iter() + .collect(); + if commons.is_empty() { + return Ok(false); + } + let oldest = commons + .iter() + .map(|commit| self.commit_time(*commit)) + .collect::, _>>()? + .into_iter() + .min() + .unwrap_or(UnixSeconds::new(i64::MIN)); + let grafts = self.shallow_grafts()?; + + let reaches_a_common = |want: gix::ObjectId| -> Result { + let mut seen: HashSet = HashSet::new(); + let mut queue: VecDeque = VecDeque::from([want]); + while let Some(commit) = queue.pop_front() { + if commons.contains(&commit) { + return Ok(true); + } + if !seen.insert(commit) || grafts.contains(&commit) { + continue; + } + if self.commit_time(commit)? < oldest { + continue; + } + let (_, parents) = self.commit_tree_and_parents(Oid::from(commit))?; + queue.extend(parents); + } + Ok(false) + }; + + self.peel_to_commits(wants.iter().copied())? + .into_iter() + .try_fold(true, |all, want| Ok(all && reaches_a_common(want)?)) + } + + pub fn select_pack_objects_filtered( + &self, + wants: Wants, + haves: Haves, + filter: Filter, + budget: PackBudget, + ) -> Result { + let wants = wants.as_slice(); + let haves = haves.as_slice(); + let mut walked = Walked::new(budget); + let mut expanded: HashMap = HashMap::new(); + let mut want_tags = Vec::new(); + let mut want_commits = Vec::new(); + let mut want_direct = Vec::new(); + wants.iter().try_for_each(|want| -> Result<(), GitError> { + match self.peel(want.object_id(), &mut want_tags, MAX_TAG_DEPTH)? { + Peeled::Commit(commit) => want_commits.push(Oid::from(commit)), + Peeled::Direct(direct) => want_direct.push(Oid::from(direct)), + } + Ok(()) + })?; + + let mut have_tags = Vec::new(); + let mut have_commits = Vec::new(); + let mut have_direct = Vec::new(); + haves + .iter() + .filter(|have| self.contains(**have)) + .try_for_each(|have| -> Result<(), GitError> { + match self.peel(have.object_id(), &mut have_tags, MAX_TAG_DEPTH)? { + Peeled::Commit(commit) => have_commits.push(Oid::from(commit)), + Peeled::Direct(direct) => have_direct.push(Oid::from(direct)), + } + Ok(()) + })?; + + let grafts = self.shallow_grafts()?; + let send: Vec<(Oid, gix::ObjectId, Vec)> = self + .rev_walk_each( + Wants::new(&want_commits), + Haves::new(&have_commits), + &mut walked, + )? + .into_iter() + .map(|commit| { + let (tree, parents) = self.commit_tree_and_parents(commit)?; + let parents = match grafts.contains(&commit.object_id()) { + true => Vec::new(), + false => parents, + }; + Ok::<_, GitError>((commit, tree, parents)) + }) + .collect::>()?; + let send_set: HashSet = send + .iter() + .map(|(commit, _, _)| commit.object_id()) + .collect(); + + let mut uninteresting: HashSet = + have_tags.iter().map(|tag| tag.object_id()).collect(); + let boundary_commits: Vec = have_commits + .iter() + .map(|commit| commit.object_id()) + .chain( + send.iter() + .flat_map(|(_, _, parents)| parents.iter().copied()) + .filter(|parent| !send_set.contains(parent)), + ) + .collect(); + boundary_commits + .iter() + .try_for_each(|commit| -> Result<(), GitError> { + let tree = self.commit_tree(Oid::from(*commit))?; + self.walk_tree( + tree, + &mut uninteresting, + &mut |_| {}, + Filter::None, + MAX_TREE_DEPTH, + &mut walked, + ) + })?; + have_direct.iter().try_for_each(|direct| { + self.collect_direct( + *direct, + &mut uninteresting, + &mut expanded, + &mut |_| {}, + Filter::None, + &mut walked, + ) + })?; + let client_has: HashSet = uninteresting + .iter() + .copied() + .chain(boundary_commits) + .map(Oid::from) + .collect(); + + let mut seen = uninteresting; + let mut out: Vec = Vec::new(); + want_tags + .iter() + .try_for_each(|tag| -> Result<(), GitError> { + if seen.insert(tag.object_id()) { + out.push(*tag); + walked.tick()?; + } + Ok(()) + })?; + if matches!(filter, Filter::None) + && want_direct.is_empty() + && send.len() >= PARALLEL_SELECT_MIN + { + out = self.walk_send_trees(&send, seen, out, walked.budget)?; + } else { + send.iter() + .try_for_each(|(commit, tree, _)| -> Result<(), GitError> { + if seen.insert(commit.object_id()) { + out.push(*commit); + } + self.walk_root_tree( + *tree, + &mut seen, + &mut expanded, + &mut |oid| out.push(oid), + filter, + &mut walked, + ) + })?; + want_direct.iter().try_for_each(|direct| { + self.collect_direct( + *direct, + &mut seen, + &mut expanded, + &mut |oid| out.push(oid), + filter, + &mut walked, + ) + })?; + } + Ok(PackSelection { + send: out, + client_has, + }) + } +} + +#[cfg(test)] +mod tests { + use std::os::unix::fs::PermissionsExt; + + use knot_types::RepoDid; + + use super::*; + use crate::Layout; + + fn seeded() -> (tempfile::TempDir, Layout, RepoDid) { + let dir = tempfile::tempdir().unwrap(); + let layout = Layout::new(dir.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + layout.create(&did).unwrap(); + (dir, layout, did) + } + + #[test] + fn open_blob_streams_a_loose_blob() { + let (_dir, layout, did) = seeded(); + let content: Vec = (0..8192u32).map(|byte| byte as u8).collect(); + let repo = layout.open(&did).unwrap(); + let oid = Oid::from(repo.git().write_blob(&content).unwrap().detach()); + + let reread = layout.open(&did).unwrap(); + let (size, mut reader) = reread.open_blob(oid).unwrap(); + assert_eq!(size, content.len() as u64); + assert!(matches!(reader, BlobReader::Loose(_))); + let mut buf = Vec::new(); + reader.read_to_end(&mut buf).unwrap(); + assert_eq!(buf, content); + } + + #[test] + fn open_blob_rejects_a_non_blob() { + let (_dir, layout, did) = seeded(); + let repo = layout.open(&did).unwrap(); + let tree = repo + .git() + .write_object(gix::objs::Tree { + entries: Vec::new(), + }) + .unwrap() + .detach(); + assert!(matches!( + repo.open_blob(Oid::from(tree)), + Err(GitError::ObjectType { + expected: "blob", + .. + }) + )); + } + + #[test] + fn corrupt_loose_object_is_a_typed_error_not_a_panic() { + let (_dir, layout, did) = seeded(); + let repo = layout.open(&did).unwrap(); + let oid = Oid::from( + repo.git() + .write_blob(b"hello streaming world\n") + .unwrap() + .detach(), + ); + let loose = repo.loose_object_path(oid); + + std::fs::set_permissions(&loose, std::fs::Permissions::from_mode(0o644)).unwrap(); + std::fs::write(&loose, b"this isn't a valid zlib object").unwrap(); + + let reopened = layout.open(&did).unwrap(); + assert!(matches!( + reopened.read_blob(oid), + Err(GitError::Corrupt { .. }) + )); + assert!(matches!( + reopened.open_blob(oid), + Err(GitError::Corrupt { .. }) + )); + } + + #[test] + fn missing_object_is_not_found() { + let (_dir, layout, did) = seeded(); + let repo = layout.open(&did).unwrap(); + let absent = Oid::from_hex("dead00000000000000000000000000000000beef").unwrap(); + assert!(matches!( + repo.read_blob(absent), + Err(GitError::ObjectNotFound(_)) + )); + assert!(matches!( + repo.open_blob(absent), + Err(GitError::ObjectNotFound(_)) + )); + } +} diff --git a/knot2/crates/knot-git/src/patch.rs b/knot2/crates/knot-git/src/patch.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-git/src/patch.rs @@ -0,0 +1,390 @@ +use std::convert::Infallible; +use std::ops::ControlFlow; + +use gix::diff::blob::unified_diff::{ConsumeHunk, ContextSize, DiffLineKind, HunkHeader}; +use gix::diff::blob::{Algorithm, Diff, InternedInput, UnifiedDiff}; +use knot_types::{ChangedFiles, ChangedFilesBudget, Listing, Oid, RepoPath}; + +use crate::error::{GitError, backend}; +use crate::objects::EntryKind; +use crate::repo::Repo; + +const BINARY_SNIFF_BYTES: usize = 8000; +pub const MAX_DIFF_BLOB_BYTES: u64 = 25 * 1024 * 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LineOp { + Context, + Delete, + Add, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HunkLine { + pub op: LineOp, + pub text: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct PatchRange { + pub base: Option, + pub head: Oid, +} + +// Just making sure a count in a start slot doesn't even compile. +knot_types::scalar_newtype! { + pub struct LineNumber(u32); + pub struct LineCount(u32); +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Hunk { + pub old_start: LineNumber, + pub old_lines: LineCount, + pub new_start: LineNumber, + pub new_lines: LineCount, + pub lines: Vec, +} + +impl Hunk { + pub fn added(&self) -> LineCount { + self.count(LineOp::Add) + } + + pub fn deleted(&self) -> LineCount { + self.count(LineOp::Delete) + } + + fn count(&self, op: LineOp) -> LineCount { + LineCount::new( + self.lines + .iter() + .filter(|line| line.op == op) + .count() + .try_into() + .unwrap_or(u32::MAX), + ) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PatchStatus { + Added, + Deleted, + Modified, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FilePatch { + pub status: PatchStatus, + pub path: RepoPath, + pub old_oid: Oid, + pub new_oid: Oid, + pub old_kind: Option, + pub new_kind: Option, + pub is_binary: bool, + pub hunks: Vec, +} + +fn is_binary(content: &[u8]) -> bool { + content[..content.len().min(BINARY_SNIFF_BYTES)].contains(&0) +} + +struct CollectHunks { + hunks: Vec, +} + +impl ConsumeHunk for CollectHunks { + type Out = Vec; + + fn consume_hunk( + &mut self, + header: HunkHeader, + lines: &[(DiffLineKind, &[u8])], + ) -> std::io::Result<()> { + let map_op = |kind: DiffLineKind| match kind { + DiffLineKind::Context => LineOp::Context, + DiffLineKind::Remove => LineOp::Delete, + DiffLineKind::Add => LineOp::Add, + }; + let adjust = |start: u32, len: u32| { + if len == 0 { + start.saturating_sub(1) + } else { + start + } + }; + self.hunks.push(Hunk { + old_start: LineNumber::new(adjust(header.before_hunk_start, header.before_hunk_len)), + old_lines: LineCount::new(header.before_hunk_len), + new_start: LineNumber::new(adjust(header.after_hunk_start, header.after_hunk_len)), + new_lines: LineCount::new(header.after_hunk_len), + lines: lines + .iter() + .map(|(kind, text)| HunkLine { + op: map_op(*kind), + text: text.to_vec(), + }) + .collect(), + }); + Ok(()) + } + + fn finish(self) -> Self::Out { + self.hunks + } +} + +fn text_hunks(old: &[u8], new: &[u8]) -> Result, GitError> { + let input = InternedInput::new(old, new); + let diff = Diff::compute(Algorithm::Histogram, &input); + UnifiedDiff::new( + &diff, + &input, + CollectHunks { hunks: Vec::new() }, + ContextSize::symmetrical(3), + ) + .consume() + .map_err(backend) +} + +enum Side { + Absent, + Present { oid: Oid, kind: EntryKind }, +} + +impl Side { + fn oid(&self, absent: Oid) -> Oid { + match self { + Side::Absent => absent, + Side::Present { oid, .. } => *oid, + } + } + + fn kind(&self) -> Option { + match self { + Side::Absent => None, + Side::Present { kind, .. } => Some(*kind), + } + } +} + +impl Repo { + fn patch_content(&self, side: &Side) -> Result, GitError> { + match side { + Side::Absent => Ok(Vec::new()), + Side::Present { oid, kind } => match kind { + EntryKind::Commit => { + Ok(format!("Subproject commit {}\n", oid.to_hex()).into_bytes()) + } + EntryKind::Tree => Ok(Vec::new()), + _ => self.read_blob(*oid), + }, + } + } + + fn side_within_diff_budget(&self, side: &Side) -> Result { + match side { + Side::Present { + oid, + kind: EntryKind::Blob | EntryKind::BlobExecutable | EntryKind::Link, + } => Ok(self.blob_size(*oid)? <= MAX_DIFF_BLOB_BYTES), + _ => Ok(true), + } + } + + fn file_patch( + &self, + status: PatchStatus, + path: RepoPath, + old: Side, + new: Side, + ) -> Result { + let within_budget = + self.side_within_diff_budget(&old)? && self.side_within_diff_budget(&new)?; + let (binary, hunks) = match within_budget { + false => (true, Vec::new()), + true => { + let old_content = self.patch_content(&old)?; + let new_content = self.patch_content(&new)?; + let binary = is_binary(&old_content) || is_binary(&new_content); + let hunks = match binary { + true => Vec::new(), + false => text_hunks(&old_content, &new_content)?, + }; + (binary, hunks) + } + }; + Ok(FilePatch { + status, + path, + old_oid: old.oid(self.object_format().null_oid()), + new_oid: new.oid(self.object_format().null_oid()), + old_kind: old.kind(), + new_kind: new.kind(), + is_binary: binary, + hunks, + }) + } + + fn diff_trees(&self, range: PatchRange) -> Result<(gix::Tree<'_>, gix::Tree<'_>), GitError> { + let PatchRange { + base: old_commit, + head: new_commit, + } = range; + let new_tree = self.root_tree(self.peel_to_commit(new_commit)?)?; + let old_tree = match old_commit { + Some(commit) => self.root_tree(self.peel_to_commit(commit)?)?, + None => self.git().empty_tree(), + }; + Ok((old_tree, new_tree)) + } + + pub fn changed_paths(&self, range: PatchRange) -> Result { + let (old_tree, new_tree) = self.diff_trees(range)?; + let mut budget = ChangedFilesBudget::new(); + let walked = old_tree + .changes() + .map_err(backend)? + .options(|options| { + options.track_rewrites(None); + }) + .for_each_to_obtain_tree(&new_tree, |change| -> Result, Infallible> { + use gix::object::tree::diff::Change; + let (location, is_tree) = match change { + Change::Addition { + location, + entry_mode, + .. + } + | Change::Deletion { + location, + entry_mode, + .. + } => (location, entry_mode.is_tree()), + Change::Modification { + location, + previous_entry_mode, + entry_mode, + .. + } => ( + location, + previous_entry_mode.is_tree() || entry_mode.is_tree(), + ), + Change::Rewrite { .. } => return Ok(ControlFlow::Continue(())), + }; + match (is_tree, RepoPath::new(location.to_string())) { + (true, _) => Ok(ControlFlow::Continue(())), + (false, Ok(path)) => Ok(budget.admit(path)), + (false, Err(_)) => Ok(budget.truncate()), + } + }); + let changed = budget.finish(); + // When the above gives us `Break`, + // gix doesn't return partial-success + // but instead `Error::Cancelled`. + // So if the listing comes out truncated, + // the "error" in `walked` is our own stop-sign given + // back at us and we ignore it on purpose. + // If the listing is complete, + // nothing ever asked to stop + // and when `walked` errors out it's actually + // from the diff itself that we should believe. + match changed.listing() { + Listing::Truncated => Ok(changed), + Listing::Complete => walked.map(|_| changed).map_err(backend), + } + } + + pub fn commit_patches(&self, range: PatchRange) -> Result, GitError> { + let (old_tree, new_tree) = self.diff_trees(range)?; + let mut sides: Vec<(PatchStatus, String, Side, Side)> = Vec::new(); + old_tree + .changes() + .map_err(backend)? + .options(|options| { + options.track_rewrites(None); + }) + .for_each_to_obtain_tree(&new_tree, |change| { + use gix::object::tree::diff::Change; + match change { + Change::Addition { + location, + id, + entry_mode, + .. + } => sides.push(( + PatchStatus::Added, + location.to_string(), + Side::Absent, + Side::Present { + oid: Oid::from(id.detach()), + kind: crate::objects::map_kind(entry_mode.kind()), + }, + )), + Change::Deletion { + location, + id, + entry_mode, + .. + } => sides.push(( + PatchStatus::Deleted, + location.to_string(), + Side::Present { + oid: Oid::from(id.detach()), + kind: crate::objects::map_kind(entry_mode.kind()), + }, + Side::Absent, + )), + Change::Modification { + location, + previous_id, + id, + previous_entry_mode, + entry_mode, + } => sides.push(( + PatchStatus::Modified, + location.to_string(), + Side::Present { + oid: Oid::from(previous_id.detach()), + kind: crate::objects::map_kind(previous_entry_mode.kind()), + }, + Side::Present { + oid: Oid::from(id.detach()), + kind: crate::objects::map_kind(entry_mode.kind()), + }, + )), + Change::Rewrite { .. } => {} + } + Ok::<_, std::convert::Infallible>(std::ops::ControlFlow::Continue(())) + }) + .map_err(backend)?; + + sides + .into_iter() + .filter(|(_, _, old, new)| { + !matches!( + (old, new), + ( + Side::Present { + kind: EntryKind::Tree, + .. + }, + _ + ) | ( + _, + Side::Present { + kind: EntryKind::Tree, + .. + } + ) + ) + }) + .map(|(status, path, old, new)| { + let path = + RepoPath::new(path).map_err(|error| GitError::Decode(error.to_string()))?; + self.file_patch(status, path, old, new) + }) + .collect() + } +} diff --git a/knot2/crates/knot-git/src/patch_apply.rs b/knot2/crates/knot-git/src/patch_apply.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-git/src/patch_apply.rs @@ -0,0 +1,733 @@ +use std::collections::BTreeMap; + +use knot_types::{Oid, RepoPath}; + +use crate::error::{GitError, backend}; +use crate::objects::{EntryKind, Identity, signature}; +use crate::patch::{Hunk, LineOp, MAX_DIFF_BLOB_BYTES}; +use crate::patch_parse::{FileIntent, ParsedFile, PatchPayload}; +use crate::repo::Repo; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ConflictReason { + AlreadyExists, + DoesNotExist, + DoesNotApply, +} + +impl ConflictReason { + pub fn as_str(self) -> &'static str { + match self { + ConflictReason::AlreadyExists => "file already exists", + ConflictReason::DoesNotExist => "file doesn't exist", + ConflictReason::DoesNotApply => "patch doesn't apply", + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Conflict { + pub path: String, + pub reason: ConflictReason, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum StagedAction { + Put { content: Vec, kind: EntryKind }, + PutGitlink { oid: Oid }, + Remove, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StagedChange { + pub path: RepoPath, + pub action: StagedAction, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum ApplyOutcome { + Clean(Vec), + Conflicted(Vec), +} + +#[derive(Debug, thiserror::Error)] +pub enum ApplyError { + #[error(transparent)] + Git(#[from] GitError), + #[error("file touched by patch exceeds {MAX_DIFF_BLOB_BYTES}-byte limit")] + TooLarge, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NewCommit { + pub tree: Oid, + pub parents: Vec, + pub author: Identity, + pub committer: Identity, + pub message: String, + pub extra_headers: Vec<(String, Vec)>, +} + +fn patch_path(raw: &str) -> Option { + RepoPath::new(raw).ok().filter(|path| !path.names_dot_git()) +} + +fn split_lines(content: &[u8]) -> Vec<&[u8]> { + content.split_inclusive(|&byte| byte == b'\n').collect() +} + +fn images(hunk: &Hunk) -> (Vec<&[u8]>, Vec<&[u8]>) { + let pick = |keep: fn(LineOp) -> bool| { + hunk.lines + .iter() + .filter(move |line| keep(line.op)) + .map(|line| line.text.as_slice()) + .collect() + }; + ( + pick(|op| matches!(op, LineOp::Context | LineOp::Delete)), + pick(|op| matches!(op, LineOp::Context | LineOp::Add)), + ) +} + +fn find_match(lines: &[&[u8]], pre: &[&[u8]], cursor: usize, expected: usize) -> Option { + let last = lines.len().checked_sub(pre.len())?; + if last < cursor { + return None; + } + let anchor = expected.clamp(cursor, last); + let matches_at = |at: usize| { + lines[at..at + pre.len()] + .iter() + .zip(pre) + .all(|(a, b)| a == b) + }; + (0..=last - cursor) + .flat_map(|distance| [anchor.checked_add(distance), anchor.checked_sub(distance)]) + .flatten() + .filter(|&at| at >= cursor && at <= last) + .find(|&at| matches_at(at)) +} + +pub(crate) fn apply_hunks(old: &[u8], hunks: &[Hunk]) -> Option> { + let lines = split_lines(old); + let (out, cursor) = + hunks + .iter() + .try_fold((Vec::::new(), 0usize), |(mut out, cursor), hunk| { + let (pre, post) = images(hunk); + let has_context = hunk.lines.iter().any(|line| line.op == LineOp::Context); + let expected = match hunk.old_lines.get() { + 0 => hunk.old_start.get() as usize, + _ => (hunk.old_start.get() as usize).saturating_sub(1), + }; + let at = match pre.is_empty() { + true => expected.clamp(cursor, lines.len()), + false => find_match(&lines, &pre, cursor, expected)?, + }; + if !has_context && !pre.is_empty() && at + pre.len() != lines.len() { + return None; + } + lines + .get(cursor..at)? + .iter() + .for_each(|line| out.extend_from_slice(line)); + post.iter().for_each(|line| out.extend_from_slice(line)); + Some((out, at + pre.len())) + })?; + Some(lines.get(cursor..)?.iter().fold(out, |mut out, line| { + out.extend_from_slice(line); + out + })) +} + +pub(crate) fn apply_delta(base: &[u8], delta: &[u8]) -> Option> { + let mut pos = 0usize; + let declared_base = read_size(delta, &mut pos)?; + let declared_target = read_size(delta, &mut pos)?; + if declared_base != base.len() as u64 || declared_target > MAX_DIFF_BLOB_BYTES { + return None; + } + let mut out: Vec = Vec::with_capacity(declared_target as usize); + std::iter::from_fn(|| { + let opcode = *delta.get(pos)?; + pos += 1; + Some(match opcode { + 0 => None, + literal if literal & 0x80 == 0 => { + let take = literal as usize; + delta.get(pos..pos + take).map(|bytes| { + pos += take; + out.extend_from_slice(bytes); + }) + } + copy => { + let mut field = |bit: u8| -> u64 { + match copy & bit { + 0 => 0, + _ => { + let byte = delta.get(pos).copied().unwrap_or(0); + pos += 1; + u64::from(byte) + } + } + }; + let offset = field(0x01) | field(0x02) << 8 | field(0x04) << 16 | field(0x08) << 24; + let size = match field(0x10) | field(0x20) << 8 | field(0x40) << 16 { + 0 => 0x10000, + size => size, + }; + base.get(offset as usize..(offset + size) as usize) + .map(|bytes| out.extend_from_slice(bytes)) + } + }) + }) + .try_for_each(|step| step.map(|_| ()))?; + (pos == delta.len() && out.len() as u64 == declared_target).then_some(out) +} + +fn read_size(delta: &[u8], pos: &mut usize) -> Option { + let mut shift = 0u32; + let mut acc = 0u64; + std::iter::from_fn(|| { + let byte = *delta.get(*pos)?; + *pos += 1; + acc |= u64::from(byte & 0x7f) << shift; + shift += 7; + Some(byte & 0x80 != 0) + }) + .take(10) + .find(|more| !more) + .map(|_| acc) +} + +enum OverlayEntry { + Put { content: Vec, kind: EntryKind }, + Gitlink { oid: Oid }, + Removed, +} + +fn subproject_content(oid: Oid) -> Vec { + format!("Subproject commit {}\n", oid.to_hex()).into_bytes() +} + +fn parse_subproject(content: &[u8]) -> Option { + let text = std::str::from_utf8(content).ok()?; + Oid::from_hex(text.strip_prefix("Subproject commit ")?.trim()).ok() +} + +struct FoundFile { + content: Vec, + kind: EntryKind, + oid: Option, +} + +struct StepView<'r, 'a> { + repo: &'r Repo, + base: Oid, + accumulated: &'a BTreeMap, + step: BTreeMap, +} + +impl StepView<'_, '_> { + fn overlaid(&self, path: &RepoPath) -> Option<&OverlayEntry> { + self.step.get(path).or_else(|| self.accumulated.get(path)) + } + + fn current(&self, path: &RepoPath) -> Result, ApplyError> { + match self.overlaid(path) { + Some(OverlayEntry::Removed) => Ok(None), + Some(OverlayEntry::Put { content, kind }) => Ok(Some(FoundFile { + content: content.clone(), + kind: *kind, + oid: Some(overlay_oid(self.repo, content)?), + })), + Some(OverlayEntry::Gitlink { oid }) => Ok(Some(FoundFile { + content: subproject_content(*oid), + kind: EntryKind::Commit, + oid: Some(*oid), + })), + None => match self.repo.entry_at(self.base, path)? { + Some(entry) + if matches!( + entry.kind, + EntryKind::Blob | EntryKind::BlobExecutable | EntryKind::Link + ) => + { + if self.repo.blob_size(entry.oid)? > MAX_DIFF_BLOB_BYTES { + return Err(ApplyError::TooLarge); + } + Ok(Some(FoundFile { + content: self.repo.read_blob(entry.oid)?, + kind: entry.kind, + oid: Some(entry.oid), + })) + } + Some(entry) if entry.kind == EntryKind::Commit => Ok(Some(FoundFile { + content: subproject_content(entry.oid), + kind: EntryKind::Commit, + oid: Some(entry.oid), + })), + _ => Ok(None), + }, + } + } + + fn occupied(&self, path: &RepoPath) -> Result { + match self.overlaid(path) { + Some(OverlayEntry::Removed) => Ok(false), + Some(OverlayEntry::Put { .. } | OverlayEntry::Gitlink { .. }) => Ok(true), + None => Ok(self.repo.entry_at(self.base, path)?.is_some()), + } + } + + fn prefix_is_file(&self, prefix: &RepoPath) -> Result { + match self.overlaid(prefix) { + Some(OverlayEntry::Removed) => Ok(false), + Some(OverlayEntry::Put { kind, .. }) => Ok(!matches!(kind, EntryKind::Tree)), + Some(OverlayEntry::Gitlink { .. }) => Ok(true), + None => Ok(matches!( + self.repo.entry_at(self.base, prefix)?, + Some(entry) if !matches!(entry.kind, EntryKind::Tree) + )), + } + } + + fn ancestor_is_file(&self, path: &RepoPath) -> Result { + let parts: Vec<&str> = path.as_str().split('/').collect(); + (1..parts.len()) + .map(|end| parts[..end].join("/")) + .try_fold(false, |blocked, prefix| { + let prefix = + RepoPath::new(prefix).expect("prefix of a valid repo path is well-formed"); + Ok(blocked || self.prefix_is_file(&prefix)?) + }) + } + + fn put(&mut self, path: &RepoPath, content: Vec, kind: EntryKind) { + self.step + .insert(path.clone(), OverlayEntry::Put { content, kind }); + } + + fn put_gitlink(&mut self, path: &RepoPath, oid: Oid) { + self.step + .insert(path.clone(), OverlayEntry::Gitlink { oid }); + } + + fn remove(&mut self, path: &RepoPath) { + self.step.insert(path.clone(), OverlayEntry::Removed); + } +} + +fn index_matches(actual: Option, declared: Option) -> bool { + matches!((actual, declared), (Some(actual), Some(declared)) if actual == declared) +} + +fn overlay_oid(repo: &Repo, content: &[u8]) -> Result { + gix::objs::compute_hash(repo.git().object_hash(), gix::objs::Kind::Blob, content) + .map(Oid::from) + .map_err(|error| ApplyError::Git(backend(error))) +} + +fn transform( + payload: &PatchPayload, + old: &[u8], + old_oid: Option, + declared_old: Option, +) -> Option> { + match payload { + PatchPayload::Text(hunks) => apply_hunks(old, hunks), + PatchPayload::BinaryLiteral(data) => { + index_matches(old_oid, declared_old).then(|| data.clone()) + } + PatchPayload::BinaryDelta(delta) => index_matches(old_oid, declared_old) + .then(|| apply_delta(old, delta)) + .flatten(), + PatchPayload::BinaryOpaque => None, + } +} + +fn fresh_content(payload: &PatchPayload) -> Option> { + match payload { + PatchPayload::Text(hunks) => apply_hunks(&[], hunks), + PatchPayload::BinaryLiteral(data) => Some(data.clone()), + PatchPayload::BinaryDelta(_) | PatchPayload::BinaryOpaque => None, + } +} + +fn file_kind(kind: Option, fallback: EntryKind) -> Option { + match kind.unwrap_or(fallback) { + EntryKind::Tree => None, + usable => Some(usable), + } +} + +fn stage_content( + overlay: &mut StepView<'_, '_>, + path: &RepoPath, + kind: EntryKind, + content: Vec, +) -> Option { + match kind { + EntryKind::Commit => match parse_subproject(&content) { + Some(oid) => { + overlay.put_gitlink(path, oid); + None + } + None => Some(Conflict { + path: path.to_string(), + reason: ConflictReason::DoesNotApply, + }), + }, + _ => { + overlay.put(path, content, kind); + None + } + } +} + +fn apply_file( + overlay: &mut StepView<'_, '_>, + file: &ParsedFile, +) -> Result, ApplyError> { + let conflict = |path: &str, reason: ConflictReason| { + Ok(Some(Conflict { + path: path.to_string(), + reason, + })) + }; + let Some(path) = patch_path(&file.path) else { + return conflict(&file.path, ConflictReason::DoesNotApply); + }; + match &file.intent { + FileIntent::Create => { + let Some(kind) = file_kind(file.new_kind, EntryKind::Blob) else { + return conflict(&file.path, ConflictReason::DoesNotApply); + }; + if overlay.occupied(&path)? { + return conflict(&file.path, ConflictReason::AlreadyExists); + } + if overlay.ancestor_is_file(&path)? { + return conflict(&file.path, ConflictReason::DoesNotApply); + } + match fresh_content(&file.payload) { + Some(content) => Ok(stage_content(overlay, &path, kind, content)), + None => conflict(&file.path, ConflictReason::DoesNotApply), + } + } + FileIntent::Delete => match overlay.current(&path)? { + None => conflict(&file.path, ConflictReason::DoesNotExist), + Some(found) => { + let emptied = match &file.payload { + PatchPayload::BinaryOpaque => { + index_matches(found.oid, file.old_index).then(Vec::new) + } + payload => transform(payload, &found.content, found.oid, file.old_index), + }; + match emptied { + Some(rest) if rest.is_empty() => { + overlay.remove(&path); + Ok(None) + } + _ => conflict(&file.path, ConflictReason::DoesNotApply), + } + } + }, + FileIntent::Modify => match overlay.current(&path)? { + None => conflict(&file.path, ConflictReason::DoesNotExist), + Some(found) => { + let Some(kind) = file_kind(file.new_kind, found.kind) else { + return conflict(&file.path, ConflictReason::DoesNotApply); + }; + match evolved(&file.payload, &found, file.old_index) { + Some(next) => Ok(stage_content(overlay, &path, kind, next)), + None => conflict(&file.path, ConflictReason::DoesNotApply), + } + } + }, + FileIntent::Rename { from } | FileIntent::Copy { from } => { + let Some(source) = patch_path(from) else { + return conflict(from, ConflictReason::DoesNotApply); + }; + if overlay.occupied(&path)? { + return conflict(&file.path, ConflictReason::AlreadyExists); + } + if overlay.ancestor_is_file(&path)? { + return conflict(&file.path, ConflictReason::DoesNotApply); + } + match overlay.current(&source)? { + None => conflict(from, ConflictReason::DoesNotExist), + Some(found) => { + let Some(kind) = file_kind(file.new_kind, found.kind) else { + return conflict(&file.path, ConflictReason::DoesNotApply); + }; + match evolved(&file.payload, &found, file.old_index) { + Some(next) => { + if matches!(&file.intent, FileIntent::Rename { .. }) { + overlay.remove(&source); + } + Ok(stage_content(overlay, &path, kind, next)) + } + None => conflict(&file.path, ConflictReason::DoesNotApply), + } + } + } + } + } +} + +fn evolved( + payload: &PatchPayload, + found: &FoundFile, + declared_old: Option, +) -> Option> { + match payload { + PatchPayload::Text(hunks) if hunks.is_empty() => Some(found.content.clone()), + payload => transform(payload, &found.content, found.oid, declared_old), + } +} + +pub struct PatchApplier<'r> { + repo: &'r Repo, + base: Oid, + accumulated: BTreeMap, +} + +impl<'r> PatchApplier<'r> { + pub fn new(repo: &'r Repo, base_commit: Oid) -> Self { + Self { + repo, + base: base_commit, + accumulated: BTreeMap::new(), + } + } + + pub fn step(&mut self, files: &[ParsedFile]) -> Result { + let mut view = StepView { + repo: self.repo, + base: self.base, + accumulated: &self.accumulated, + step: BTreeMap::new(), + }; + let conflicts: Vec = files + .iter() + .map(|file| apply_file(&mut view, file)) + .collect::, ApplyError>>()? + .into_iter() + .flatten() + .collect(); + if !conflicts.is_empty() { + return Ok(ApplyOutcome::Conflicted(conflicts)); + } + let step = view.step; + let staged: Vec = step + .iter() + .map(|(path, entry)| StagedChange { + path: path.clone(), + action: match entry { + OverlayEntry::Put { content, kind } => StagedAction::Put { + content: content.clone(), + kind: *kind, + }, + OverlayEntry::Gitlink { oid } => StagedAction::PutGitlink { oid: *oid }, + OverlayEntry::Removed => StagedAction::Remove, + }, + }) + .collect(); + self.accumulated.extend(step); + Ok(ApplyOutcome::Clean(staged)) + } +} + +impl Repo { + pub fn write_staged_tree( + &self, + base_tree: Oid, + staged: &[StagedChange], + ) -> Result { + let mut editor = self + .git() + .edit_tree(base_tree.object_id()) + .map_err(backend)?; + staged + .iter() + .try_for_each(|change| -> Result<(), GitError> { + match &change.action { + StagedAction::Put { content, kind } => { + let blob = self.git().write_blob(content).map_err(backend)?.detach(); + editor + .upsert(change.path.as_str(), (*kind).into(), blob) + .map_err(backend)?; + } + StagedAction::PutGitlink { oid } => { + editor + .upsert( + change.path.as_str(), + EntryKind::Commit.into(), + oid.object_id(), + ) + .map_err(backend)?; + } + StagedAction::Remove => { + editor.remove(change.path.as_str()).map_err(backend)?; + } + } + Ok(()) + })?; + Ok(Oid::from(editor.write().map_err(backend)?.detach())) + } + + pub fn write_commit(&self, new: &NewCommit) -> Result { + let message = match new.message.ends_with('\n') { + true => new.message.clone(), + false => format!("{}\n", new.message), + }; + let commit = gix::objs::Commit { + tree: new.tree.object_id(), + parents: new + .parents + .iter() + .map(|parent| parent.object_id()) + .collect(), + author: signature(&new.author), + committer: signature(&new.committer), + encoding: None, + message: message.into(), + extra_headers: new + .extra_headers + .iter() + .map(|(name, value)| { + ( + name.as_str().into(), + gix::bstr::BString::from(value.clone()), + ) + }) + .collect(), + }; + Ok(Oid::from( + self.git().write_object(commit).map_err(backend)?.detach(), + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::patch::{HunkLine, LineCount, LineNumber}; + + type ApplyCase<'a> = (&'a [u8], Vec, Option<&'a [u8]>); + + fn hunk(old_start: u32, old_lines: u32, new_start: u32, new_lines: u32, spec: &str) -> Hunk { + let lines = spec + .split('\n') + .filter(|line| !line.is_empty()) + .map(|line| { + let (op, text) = match line.as_bytes()[0] { + b'-' => (LineOp::Delete, &line[1..]), + b'+' => (LineOp::Add, &line[1..]), + _ => (LineOp::Context, &line[1..]), + }; + HunkLine { + op, + text: format!("{text}\n").into_bytes(), + } + }) + .collect(); + Hunk { + old_start: LineNumber::new(old_start), + old_lines: LineCount::new(old_lines), + new_start: LineNumber::new(new_start), + new_lines: LineCount::new(new_lines), + lines, + } + } + + #[test] + fn apply_hunks_tracks_position_drift_and_boundaries() { + let cases: Vec = vec![ + ( + b"one\ntwo\nthree\n", + vec![hunk(1, 3, 1, 3, " one\n-two\n+TWO\n three\n")], + Some(b"one\nTWO\nthree\n".as_slice()), + ), + ( + b"zero\nzero\none\ntwo\nthree\n", + vec![hunk(1, 3, 1, 3, " one\n-two\n+TWO\n three\n")], + Some(b"zero\nzero\none\nTWO\nthree\n".as_slice()), + ), + ( + b"one\nTWO ALREADY\nthree\n", + vec![hunk(1, 3, 1, 3, " one\n-two\n+TWO\n three\n")], + None, + ), + ( + b"", + vec![hunk(0, 0, 1, 2, "+alpha\n+beta\n")], + Some(b"alpha\nbeta\n".as_slice()), + ), + ( + b"only\n", + vec![hunk(1, 1, 0, 0, "-only\n")], + Some(b"".as_slice()), + ), + ( + b"a\nb\nc\nd\ne\nf\ng\n", + vec![ + hunk(1, 2, 1, 3, " a\n+inserted\n b\n"), + hunk(6, 2, 7, 2, " f\n-g\n+G\n"), + ], + Some(b"a\ninserted\nb\nc\nd\ne\nf\nG\n".as_slice()), + ), + ( + b"one\ntwo\nthree\nfour\n", + vec![hunk(2, 1, 2, 1, "-two\n+TWO\n")], + None, + ), + ( + b"one\ntwo\nthree\nfour\n", + vec![hunk(2, 1, 1, 0, "-two\n")], + None, + ), + ( + b"one\ntwo\nthree\nfour\n", + vec![hunk(4, 1, 4, 1, "-four\n+FOUR\n")], + Some(b"one\ntwo\nthree\nFOUR\n".as_slice()), + ), + ]; + cases.iter().for_each(|(old, hunks, expected)| { + assert_eq!( + apply_hunks(old, hunks).as_deref(), + *expected, + "apply_hunks mismatch for {old:?}" + ); + }); + } + + #[test] + fn delta_application_round_trips_copy_and_insert() { + let base = b"hello world"; + let delta: Vec = vec![11, 9, 0x90, 5, 4, b'-', b'g', b'i', b'x']; + assert_eq!(apply_delta(base, &delta), Some(b"hello-gix".to_vec())); + assert_eq!(apply_delta(b"wrong size base", &delta), None); + assert_eq!(apply_delta(base, &delta[..5]), None); + } + + #[test] + fn unsafe_paths_are_rejected() { + assert!(patch_path("../escape").is_none()); + assert!(patch_path("/absolute").is_none()); + assert!(patch_path("nested/../escape").is_none()); + assert!(patch_path(".git/hooks/pre-receive").is_none()); + assert!(patch_path("dir/.GIT/config").is_none()); + assert!(patch_path("").is_none()); + assert!(patch_path("src/lib.rs").is_some()); + assert!(patch_path("a b/c.txt").is_some()); + } + + #[test] + fn subproject_text_round_trips_through_a_commit_oid() { + let oid = Oid::from_hex("0123456789abcdef0123456789abcdef01234567").unwrap(); + assert_eq!(parse_subproject(&subproject_content(oid)), Some(oid)); + assert_eq!(parse_subproject(b"not a subproject line\n"), None); + } +} diff --git a/knot2/crates/knot-git/src/patch_parse.rs b/knot2/crates/knot-git/src/patch_parse.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-git/src/patch_parse.rs @@ -0,0 +1,1082 @@ +use std::io::Read; +use std::sync::LazyLock; + +use base64::Engine; +use knot_types::{AuthorName, Email, Oid}; + +use crate::objects::{CommitChangeId, EntryKind}; +use crate::patch::{Hunk, HunkLine, LineCount, LineNumber, LineOp, MAX_DIFF_BLOB_BYTES}; + +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum PatchParseError { + #[error("patch is empty")] + Empty, + #[error("patch contains no file changes")] + NoFiles, + #[error("malformed patch: {0}")] + Malformed(String), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum FileIntent { + Create, + Delete, + Modify, + Rename { from: String }, + Copy { from: String }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PatchPayload { + Text(Vec), + BinaryLiteral(Vec), + BinaryDelta(Vec), + BinaryOpaque, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParsedFile { + pub path: String, + pub intent: FileIntent, + pub old_kind: Option, + pub new_kind: Option, + pub old_index: Option, + pub payload: PatchPayload, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MailPatch { + pub author_name: AuthorName, + pub author_email: Email, + pub date: String, + pub subject: String, + pub body: String, + pub change_id: Option, + pub files: Vec, +} + +impl MailPatch { + pub fn commit_message(&self) -> String { + match self.body.is_empty() { + true => self.subject.clone(), + false => format!("{}\n\n{}", self.subject, self.body), + } + } +} + +pub fn is_format_patch(patch: &str) -> bool { + let lines: Vec<&str> = patch.split('\n').collect(); + if lines.len() < 2 { + return false; + } + let first = lines[0].trim(); + if first.starts_with("From ") && first.contains(" Mon Sep 17 00:00:00 2001") { + return true; + } + lines + .iter() + .take(10) + .map(|line| line.trim()) + .filter(|line| { + line.starts_with("From: ") + || line.starts_with("Date: ") + || line.starts_with("Subject: ") + || line.starts_with("commit ") + }) + .count() + >= 2 +} + +struct Cursor<'a> { + lines: &'a [&'a str], + pos: usize, +} + +impl<'a> Cursor<'a> { + fn new(lines: &'a [&'a str]) -> Self { + Self { lines, pos: 0 } + } + + fn peek(&self) -> Option<&'a str> { + self.lines.get(self.pos).copied() + } + + fn next(&mut self) -> Option<&'a str> { + let line = self.peek()?; + self.pos += 1; + Some(line) + } + + fn take_prefix(&mut self, prefix: &str) -> Option<&'a str> { + let rest = self.peek()?.strip_prefix(prefix)?; + self.pos += 1; + Some(rest) + } +} + +fn malformed(message: impl Into) -> PatchParseError { + PatchParseError::Malformed(message.into()) +} + +const MAX_TOTAL_PATCH_BYTES: u64 = 128 * 1024 * 1024; + +struct Budget { + remaining: u64, +} + +impl Budget { + fn new(limit: u64) -> Self { + Self { remaining: limit } + } + + fn charge(&mut self, bytes: u64) -> Result<(), PatchParseError> { + self.remaining = self + .remaining + .checked_sub(bytes) + .ok_or_else(|| malformed("patch exceeds total decompressed size budget"))?; + Ok(()) + } +} + +fn unescape_c(bytes: &[u8]) -> Option> { + let mut pos = 0usize; + std::iter::from_fn(move || match bytes.get(pos..) { + None | Some([]) => None, + Some(slice) => { + let decoded: Option = match slice { + [b'\\', b'n', ..] => apply(&mut pos, 2, b'\n'), + [b'\\', b't', ..] => apply(&mut pos, 2, b'\t'), + [b'\\', b'"', ..] => apply(&mut pos, 2, b'"'), + [b'\\', b'\\', ..] => apply(&mut pos, 2, b'\\'), + [b'\\', a @ b'0'..=b'3', b @ b'0'..=b'7', c @ b'0'..=b'7', ..] => { + apply(&mut pos, 4, (a - b'0') * 64 + (b - b'0') * 8 + (c - b'0')) + } + [b'\\', ..] => { + pos += 1; + None + } + [byte, ..] => apply(&mut pos, 1, *byte), + [] => None, + }; + Some(decoded) + } + }) + .collect() +} + +fn apply(pos: &mut usize, width: usize, byte: u8) -> Option { + *pos += width; + Some(byte) +} + +fn quoted_end(bytes: &[u8]) -> Option { + let mut idx = 0usize; + std::iter::from_fn(move || match bytes.get(idx) { + Some(b'"') => Some(Some(idx)), + Some(b'\\') => { + idx += 2; + Some(None) + } + Some(_) => { + idx += 1; + Some(None) + } + None => None, + }) + .flatten() + .next() +} + +fn unquote(raw: &str) -> Result { + match raw.strip_prefix('"') { + None => Ok(raw.to_string()), + Some(inner) => { + let end = + quoted_end(inner.as_bytes()).ok_or_else(|| malformed("unclosed quoted path"))?; + let unescaped = unescape_c(&inner.as_bytes()[..end]) + .ok_or_else(|| malformed("bad escape in quoted path"))?; + String::from_utf8(unescaped).map_err(|_| malformed("quoted path isn't utf-8")) + } + } +} + +fn strip_level(path: &str) -> String { + path.split_once('/') + .map(|(_, rest)| rest.to_string()) + .unwrap_or_else(|| path.to_string()) +} + +fn parse_label(raw: &str) -> Result, PatchParseError> { + let bare = match raw.starts_with('"') { + true => unquote(raw)?, + false => raw.split('\t').next().unwrap_or(raw).trim_end().to_string(), + }; + Ok(match bare.as_str() { + "/dev/null" => None, + _ => Some(strip_level(&bare)), + }) +} + +fn diff_paths(rest: &str) -> Option<(String, String)> { + match rest.contains('"') { + true => { + let (old, after) = take_path_token(rest)?; + let (new, _) = take_path_token(after.strip_prefix(' ')?)?; + Some((strip_level(&old), strip_level(&new))) + } + false => { + let split = rest.rfind(" b/")?; + let old = rest.get(..split)?.strip_prefix("a/")?; + let new = rest.get(split + 3..)?; + Some((old.to_string(), new.to_string())) + } + } +} + +fn take_path_token(rest: &str) -> Option<(String, &str)> { + match rest.strip_prefix('"') { + Some(inner) => { + let end = quoted_end(inner.as_bytes())?; + let token = String::from_utf8(unescape_c(&inner.as_bytes()[..end])?).ok()?; + Some((token, inner.get(end + 1..)?)) + } + None => { + let end = rest.find(' ').unwrap_or(rest.len()); + Some((rest[..end].to_string(), &rest[end..])) + } + } +} + +fn full_oid(hex: &str) -> Option { + (hex.len() == 40).then(|| Oid::from_hex(hex).ok()).flatten() +} + +fn parse_hunk_header(line: &str) -> Option<(LineNumber, LineCount, LineNumber, LineCount)> { + let rest = line.strip_prefix("@@ -")?; + let (old, rest) = rest.split_once(" +")?; + let (new, _) = rest.split_once(" @@")?; + let span = |raw: &str| -> Option<(LineNumber, LineCount)> { + match raw.split_once(',') { + Some((start, lines)) => Some(( + LineNumber::new(start.parse().ok()?), + LineCount::new(lines.parse().ok()?), + )), + None => Some((LineNumber::new(raw.parse().ok()?), LineCount::new(1))), + } + }; + let (old_start, old_lines) = span(old)?; + let (new_start, new_lines) = span(new)?; + Some((old_start, old_lines, new_start, new_lines)) +} + +fn parse_hunk(cursor: &mut Cursor<'_>, budget: &mut Budget) -> Result { + let header = cursor.next().ok_or_else(|| malformed("truncated hunk"))?; + let (old_start, old_lines, new_start, new_lines) = + parse_hunk_header(header).ok_or_else(|| malformed(format!("bad hunk header {header}")))?; + let mut lines: Vec = Vec::new(); + let mut old_left = old_lines.get() as i64; + let mut new_left = new_lines.get() as i64; + std::iter::from_fn(|| { + (old_left > 0 || new_left > 0).then(|| -> Result<(), PatchParseError> { + let line = cursor + .next() + .ok_or_else(|| malformed("hunk ends before its declared length"))?; + budget.charge(line.len() as u64 + 1)?; + let push = |lines: &mut Vec, op: LineOp| { + let mut text = line.get(1..).unwrap_or("").as_bytes().to_vec(); + text.push(b'\n'); + lines.push(HunkLine { op, text }); + }; + match line.as_bytes().first() { + Some(b' ') | None => { + old_left -= 1; + new_left -= 1; + push(&mut lines, LineOp::Context); + Ok(()) + } + Some(b'-') => { + old_left -= 1; + push(&mut lines, LineOp::Delete); + Ok(()) + } + Some(b'+') => { + new_left -= 1; + push(&mut lines, LineOp::Add); + Ok(()) + } + Some(b'\\') => { + strip_last_newline(&mut lines); + Ok(()) + } + _ => Err(malformed(format!("unexpected hunk line {line}"))), + } + }) + }) + .try_for_each(|outcome| outcome)?; + if cursor.peek().is_some_and(|line| line.starts_with('\\')) { + cursor.next(); + strip_last_newline(&mut lines); + } + Ok(Hunk { + old_start, + old_lines, + new_start, + new_lines, + lines, + }) +} + +fn strip_last_newline(lines: &mut [HunkLine]) { + if let Some(last) = lines.last_mut() + && last.text.last() == Some(&b'\n') + { + last.text.pop(); + } +} + +fn parse_hunks(cursor: &mut Cursor<'_>, budget: &mut Budget) -> Result, PatchParseError> { + std::iter::from_fn(|| { + cursor + .peek() + .is_some_and(|line| line.starts_with("@@ -")) + .then(|| parse_hunk(cursor, budget)) + }) + .collect() +} + +static BASE85: LazyLock<[i16; 256]> = LazyLock::new(|| { + const ALPHABET: &[u8] = + b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~"; + std::array::from_fn(|byte| { + ALPHABET + .iter() + .position(|&c| c as usize == byte) + .map(|digit| digit as i16) + .unwrap_or(-1) + }) +}); + +fn decode_base85_line(line: &str, out: &mut Vec) -> Result<(), PatchParseError> { + let bad = || malformed("bad base85 line in binary patch"); + let (len_char, data) = line.as_bytes().split_first().ok_or_else(bad)?; + let line_len = match len_char { + b'A'..=b'Z' => (len_char - b'A' + 1) as usize, + b'a'..=b'z' => (len_char - b'a' + 27) as usize, + _ => return Err(bad()), + }; + if data.len() != line_len.div_ceil(4) * 5 { + return Err(bad()); + } + data.chunks(5) + .enumerate() + .try_for_each(|(group, chunk)| -> Result<(), PatchParseError> { + let acc = chunk + .iter() + .try_fold(0u64, |acc, &c| { + let digit = BASE85[c as usize]; + (digit >= 0).then(|| acc * 85 + digit as u64) + }) + .filter(|&acc| acc <= u32::MAX as u64) + .ok_or_else(bad)?; + let take = (line_len - group * 4).min(4); + out.extend_from_slice(&(acc as u32).to_be_bytes()[..take]); + Ok(()) + }) +} + +fn parse_binary_block( + cursor: &mut Cursor<'_>, + budget: &mut Budget, +) -> Result<(bool, Vec), PatchParseError> { + let header = cursor + .next() + .ok_or_else(|| malformed("truncated binary patch"))?; + let (kind, size) = header + .split_once(' ') + .ok_or_else(|| malformed(format!("bad binary patch header {header}")))?; + let is_delta = match kind { + "literal" => false, + "delta" => true, + _ => return Err(malformed(format!("unknown binary patch kind {kind}"))), + }; + let size: u64 = size + .trim() + .parse() + .map_err(|_| malformed("bad binary patch size"))?; + if size > MAX_DIFF_BLOB_BYTES { + return Err(malformed("binary patch exceeds size limit")); + } + budget.charge(size)?; + let mut packed: Vec = Vec::new(); + std::iter::from_fn(|| { + cursor + .peek() + .is_some_and(|line| !line.is_empty()) + .then(|| cursor.next().expect("peeked line is present")) + }) + .try_for_each(|line| decode_base85_line(line, &mut packed))?; + cursor.next(); + let mut inflated: Vec = Vec::new(); + flate2::read::ZlibDecoder::new(packed.as_slice()) + .take(size + 1) + .read_to_end(&mut inflated) + .map_err(|error| malformed(format!("bad zlib stream in binary patch: {error}")))?; + if inflated.len() as u64 != size { + return Err(malformed("binary patch size doesn't match its header")); + } + Ok((is_delta, inflated)) +} + +fn parse_binary_payload( + cursor: &mut Cursor<'_>, + budget: &mut Budget, +) -> Result { + cursor.next(); + let (is_delta, data) = parse_binary_block(cursor, budget)?; + if cursor + .peek() + .is_some_and(|line| line.starts_with("literal ") || line.starts_with("delta ")) + { + parse_binary_block(cursor, budget)?; + } + Ok(match is_delta { + true => PatchPayload::BinaryDelta(data), + false => PatchPayload::BinaryLiteral(data), + }) +} + +#[derive(Default)] +struct FileHeaders { + diff_old: Option, + diff_new: Option, + old_mode: Option, + new_mode: Option, + created: bool, + deleted: bool, + rename_from: Option, + rename_to: Option, + copy_from: Option, + copy_to: Option, + old_index: Option, + label_old: Option>, + label_new: Option>, +} + +fn parse_extended_headers( + cursor: &mut Cursor<'_>, + headers: &mut FileHeaders, +) -> Result<(), PatchParseError> { + std::iter::from_fn(|| { + let line = cursor.peek()?; + let step: Option> = if let Some(rest) = + line.strip_prefix("old mode ") + { + headers.old_mode = EntryKind::from_git_mode(rest); + Some(Ok(())) + } else if let Some(rest) = line.strip_prefix("new mode ") { + headers.new_mode = EntryKind::from_git_mode(rest); + Some(Ok(())) + } else if let Some(rest) = line.strip_prefix("new file mode ") { + headers.created = true; + headers.new_mode = EntryKind::from_git_mode(rest); + Some(Ok(())) + } else if let Some(rest) = line.strip_prefix("deleted file mode ") { + headers.deleted = true; + headers.old_mode = EntryKind::from_git_mode(rest); + Some(Ok(())) + } else if let Some(rest) = line.strip_prefix("rename from ") { + Some(unquote(rest).map(|path| { + headers.rename_from = Some(path); + })) + } else if let Some(rest) = line.strip_prefix("rename to ") { + Some(unquote(rest).map(|path| { + headers.rename_to = Some(path); + })) + } else if let Some(rest) = line.strip_prefix("copy from ") { + Some(unquote(rest).map(|path| { + headers.copy_from = Some(path); + })) + } else if let Some(rest) = line.strip_prefix("copy to ") { + Some(unquote(rest).map(|path| { + headers.copy_to = Some(path); + })) + } else if line.starts_with("similarity index ") || line.starts_with("dissimilarity index ") + { + Some(Ok(())) + } else if let Some(rest) = line.strip_prefix("index ") { + let (oids, mode) = rest + .split_once(' ') + .map(|(oids, mode)| (oids, Some(mode))) + .unwrap_or((rest, None)); + if let Some((old, _)) = oids.split_once("..") { + headers.old_index = full_oid(old); + } + if let Some(kind) = mode.and_then(EntryKind::from_git_mode) { + headers.old_mode = headers.old_mode.or(Some(kind)); + headers.new_mode = headers.new_mode.or(Some(kind)); + } + Some(Ok(())) + } else { + None + }; + step.inspect(|_| { + cursor.next(); + }) + }) + .try_for_each(|outcome| outcome) +} + +fn parse_labels_and_hunks( + cursor: &mut Cursor<'_>, + headers: &mut FileHeaders, + budget: &mut Budget, +) -> Result { + let old_raw = cursor + .take_prefix("--- ") + .ok_or_else(|| malformed("expected --- label"))?; + headers.label_old = Some(parse_label(old_raw)?); + let new_raw = cursor + .take_prefix("+++ ") + .ok_or_else(|| malformed("expected +++ label"))?; + headers.label_new = Some(parse_label(new_raw)?); + Ok(PatchPayload::Text(parse_hunks(cursor, budget)?)) +} + +fn assemble(headers: FileHeaders, payload: PatchPayload) -> Result { + let FileHeaders { + diff_old, + diff_new, + old_mode, + new_mode, + created, + deleted, + rename_from, + rename_to, + copy_from, + copy_to, + old_index, + label_old, + label_new, + } = headers; + let created = created || matches!(label_old, Some(None)); + let deleted = deleted || matches!(label_new, Some(None)); + let need = |path: Option, what: &str| { + path.ok_or_else(|| malformed(format!("file section is missing its {what} path"))) + }; + let (intent, path) = match (rename_from, rename_to, copy_from, copy_to) { + (Some(from), to, _, _) => (FileIntent::Rename { from }, need(to, "rename target")?), + (_, _, Some(from), to) => (FileIntent::Copy { from }, need(to, "copy target")?), + _ if created => ( + FileIntent::Create, + need(label_new.flatten().or(diff_new), "new")?, + ), + _ if deleted => ( + FileIntent::Delete, + need(label_old.flatten().or(diff_old), "old")?, + ), + _ => ( + FileIntent::Modify, + need(label_new.flatten().or(diff_new), "target")?, + ), + }; + Ok(ParsedFile { + path, + intent, + old_kind: old_mode, + new_kind: new_mode, + old_index, + payload, + }) +} + +fn parse_git_file( + cursor: &mut Cursor<'_>, + budget: &mut Budget, +) -> Result { + let rest = cursor + .take_prefix("diff --git ") + .ok_or_else(|| malformed("expected diff --git header"))?; + let mut headers = FileHeaders::default(); + if let Some((old, new)) = diff_paths(rest) { + headers.diff_old = Some(old); + headers.diff_new = Some(new); + } + parse_extended_headers(cursor, &mut headers)?; + let payload = match cursor.peek() { + Some(line) if line.starts_with("--- ") => { + parse_labels_and_hunks(cursor, &mut headers, budget)? + } + Some("GIT binary patch") => parse_binary_payload(cursor, budget)?, + Some(line) if line.starts_with("Binary files ") => { + cursor.next(); + PatchPayload::BinaryOpaque + } + _ => PatchPayload::Text(Vec::new()), + }; + assemble(headers, payload) +} + +fn parse_traditional_file( + cursor: &mut Cursor<'_>, + budget: &mut Budget, +) -> Result { + let mut headers = FileHeaders::default(); + let payload = parse_labels_and_hunks(cursor, &mut headers, budget)?; + assemble(headers, payload) +} + +fn at_file_start(cursor: &Cursor<'_>) -> bool { + match cursor.peek() { + Some(line) if line.starts_with("diff --git ") => true, + Some(line) if line.starts_with("--- ") => cursor + .lines + .get(cursor.pos + 1) + .is_some_and(|next| next.starts_with("+++ ")), + _ => false, + } +} + +fn skip_to_file_start(cursor: &mut Cursor<'_>) -> bool { + std::iter::from_fn(|| { + (!at_file_start(cursor) && cursor.peek().is_some()).then(|| cursor.next()) + }) + .for_each(|_| ()); + cursor.peek().is_some() +} + +pub fn parse_patch(text: &str) -> Result, PatchParseError> { + parse_patch_bounded(text, MAX_TOTAL_PATCH_BYTES) +} + +pub fn parse_patch_bounded(text: &str, max_bytes: u64) -> Result, PatchParseError> { + parse_patch_budgeted(text, &mut Budget::new(max_bytes)) +} + +fn parse_patch_budgeted( + text: &str, + budget: &mut Budget, +) -> Result, PatchParseError> { + if text.trim().is_empty() { + return Err(PatchParseError::Empty); + } + let lines: Vec<&str> = text.split('\n').collect(); + let mut cursor = Cursor::new(&lines); + let files: Vec = std::iter::from_fn(|| { + skip_to_file_start(&mut cursor).then(|| match cursor.peek() { + Some(line) if line.starts_with("diff --git ") => parse_git_file(&mut cursor, budget), + _ => parse_traditional_file(&mut cursor, budget), + }) + }) + .collect::>()?; + match files.is_empty() { + true => Err(PatchParseError::NoFiles), + false => Ok(files), + } +} + +fn is_mail_divider(line: &str) -> bool { + line.strip_prefix("From ").is_some_and(|rest| { + rest.len() > 40 + && rest.as_bytes()[..40] + .iter() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(byte)) + && rest.as_bytes()[40] == b' ' + }) +} + +fn split_mail(text: &str) -> Vec<&str> { + let starts: Vec = text + .split_inclusive('\n') + .scan(0usize, |offset, line| { + let start = *offset; + *offset += line.len(); + Some((start, line)) + }) + .filter(|(_, line)| is_mail_divider(line.trim_end_matches('\n'))) + .map(|(start, _)| start) + .collect(); + match starts.is_empty() { + true => vec![text], + false => { + let ends = starts + .iter() + .skip(1) + .copied() + .chain(std::iter::once(text.len())); + starts + .iter() + .copied() + .zip(ends) + .map(|(start, end)| &text[start..end]) + .collect() + } + } +} + +fn decode_q(bytes: &[u8]) -> Option> { + let hex = |c: u8| (c as char).to_digit(16).map(|d| d as u8); + let mut pos = 0usize; + std::iter::from_fn(move || match bytes.get(pos..) { + None | Some([]) => None, + Some(slice) => { + let decoded: Option = match slice { + [b'_', ..] => apply(&mut pos, 1, b' '), + [b'=', high, low, ..] => { + let byte = hex(*high).zip(hex(*low)).map(|(high, low)| high * 16 + low); + pos += 3; + byte + } + [b'=', ..] => { + pos += 1; + None + } + [byte, ..] => apply(&mut pos, 1, *byte), + [] => None, + }; + Some(decoded) + } + }) + .collect() +} + +fn decode_rfc2047_word(word: &str) -> Option { + let inner = word.strip_prefix("=?")?.strip_suffix("?=")?; + let (charset, rest) = inner.split_once('?')?; + let (encoding, payload) = rest.split_once('?')?; + if !charset.eq_ignore_ascii_case("utf-8") { + return None; + } + let bytes = match encoding { + "Q" | "q" => decode_q(payload.as_bytes())?, + "B" | "b" => base64::engine::general_purpose::STANDARD + .decode(payload) + .ok()?, + _ => return None, + }; + Some(String::from_utf8_lossy(&bytes).into_owned()) +} + +fn decode_rfc2047(value: &str) -> String { + value + .split(' ') + .filter(|token| !token.is_empty()) + .map(|token| match decode_rfc2047_word(token) { + Some(decoded) => (true, decoded), + None => (false, token.to_string()), + }) + .fold( + (String::new(), false), + |(mut acc, prev_encoded), (encoded, text)| { + if !(acc.is_empty() || prev_encoded && encoded) { + acc.push(' '); + } + acc.push_str(&text); + (acc, encoded) + }, + ) + .0 +} + +fn strip_subject_prefix(subject: &str) -> String { + let stripped = std::iter::successors(Some(subject.trim_start()), |current| { + current + .strip_prefix('[') + .and_then(|rest| rest.split_once(']')) + .map(|(_, tail)| tail.trim_start()) + }) + .last() + .unwrap_or(""); + match stripped.starts_with('[') { + true => stripped.to_string(), + false => stripped.trim_end().to_string(), + } +} + +fn parse_address(raw: &str) -> (AuthorName, Email) { + let decoded = decode_rfc2047(raw.trim()); + match decoded.rsplit_once('<') { + Some((name, rest)) => { + let email = rest.split('>').next().unwrap_or(rest).trim(); + let name = name.trim().trim_matches('"').trim(); + (AuthorName::new(name), Email::new(email)) + } + None => { + let bare = decoded.trim(); + (AuthorName::new(bare), Email::new(bare)) + } + } +} + +fn fold_headers(lines: &[&str]) -> Vec<(String, String)> { + lines.iter().fold(Vec::new(), |mut acc, line| { + match line.strip_prefix(' ').or_else(|| line.strip_prefix('\t')) { + Some(continuation) => { + if let Some(last) = acc.last_mut() { + last.1.push(' '); + last.1.push_str(continuation.trim()); + } + } + None => { + if let Some((name, value)) = line.split_once(':') { + acc.push((name.trim().to_string(), value.trim().to_string())); + } + } + } + acc + }) +} + +fn parse_mail(chunk: &str, budget: &mut Budget) -> Result { + let lines: Vec<&str> = chunk.split('\n').collect(); + let after_divider: &[&str] = match lines.split_first() { + Some((first, rest)) if is_mail_divider(first) => rest, + _ => &lines, + }; + let header_end = after_divider + .iter() + .position(|line| line.trim().is_empty()) + .ok_or_else(|| malformed("mail patch has no header separator"))?; + let headers = fold_headers(&after_divider[..header_end]); + let header = |name: &str| { + headers + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.clone()) + }; + let (author_name, author_email) = header("From") + .map(|raw| parse_address(&raw)) + .ok_or_else(|| malformed("mail patch has no From header"))?; + let rest = &after_divider[header_end + 1..]; + let body_end = rest + .iter() + .position(|line| line.trim_end() == "---" || line.starts_with("diff --git ")) + .unwrap_or(rest.len()); + let body = rest[..body_end].join("\n").trim().to_string(); + let files = parse_patch_budgeted(&rest[body_end..].join("\n"), budget)?; + Ok(MailPatch { + author_name, + author_email, + date: header("Date").unwrap_or_default(), + subject: strip_subject_prefix(&decode_rfc2047(&header("Subject").unwrap_or_default())), + body, + change_id: header("Change-Id").and_then(|raw| CommitChangeId::new(raw).ok()), + files, + }) +} + +pub fn parse_mailbox(text: &str) -> Result, PatchParseError> { + parse_mailbox_bounded(text, MAX_TOTAL_PATCH_BYTES) +} + +pub fn parse_mailbox_bounded( + text: &str, + max_bytes: u64, +) -> Result, PatchParseError> { + if text.trim().is_empty() { + return Err(PatchParseError::Empty); + } + let mut budget = Budget::new(max_bytes); + split_mail(text) + .into_iter() + .map(|chunk| parse_mail(chunk, &mut budget)) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn format_patch_detection_matches_the_mailbox_heuristic() { + assert!(is_format_patch( + "From 0123456789012345678901234567890123456789 Mon Sep 17 00:00:00 2001\nFrom: nel \n" + )); + assert!(is_format_patch( + "From: nel \nSubject: [PATCH] tide pool\n\n" + )); + assert!(!is_format_patch( + "diff --git a/reef.txt b/reef.txt\n--- a/reef.txt\n+++ b/reef.txt\n" + )); + assert!(!is_format_patch("")); + } + + #[test] + fn a_simple_modification_parses() { + let patch = "diff --git a/reef.txt b/reef.txt\nindex 1111111..2222222 100644\n--- a/reef.txt\n+++ b/reef.txt\n@@ -1,2 +1,2 @@\n-old line\n+new line\n context\n"; + let files = parse_patch(patch).unwrap(); + assert_eq!(files.len(), 1); + assert_eq!(files[0].path, "reef.txt"); + assert_eq!(files[0].intent, FileIntent::Modify); + let PatchPayload::Text(hunks) = &files[0].payload else { + panic!("expected text payload"); + }; + assert_eq!(hunks.len(), 1); + assert_eq!(hunks[0].lines.len(), 3); + } + + #[test] + fn creations_deletions_and_renames_parse() { + let patch = concat!( + "diff --git a/new.txt b/new.txt\n", + "new file mode 100644\n", + "index 0000000..2222222\n", + "--- /dev/null\n", + "+++ b/new.txt\n", + "@@ -0,0 +1 @@\n", + "+hello\n", + "diff --git a/gone.txt b/gone.txt\n", + "deleted file mode 100755\n", + "index 2222222..0000000\n", + "--- a/gone.txt\n", + "+++ /dev/null\n", + "@@ -1 +0,0 @@\n", + "-bye\n", + "diff --git a/old.txt b/moved.txt\n", + "similarity index 100%\n", + "rename from old.txt\n", + "rename to moved.txt\n", + ); + let files = parse_patch(patch).unwrap(); + assert_eq!(files.len(), 3); + assert_eq!(files[0].intent, FileIntent::Create); + assert_eq!(files[0].new_kind, Some(EntryKind::Blob)); + assert_eq!(files[1].intent, FileIntent::Delete); + assert_eq!(files[1].old_kind, Some(EntryKind::BlobExecutable)); + assert_eq!( + files[2].intent, + FileIntent::Rename { + from: "old.txt".to_string() + } + ); + assert_eq!(files[2].path, "moved.txt"); + } + + #[test] + fn the_no_newline_marker_strips_the_trailing_newline() { + let patch = "diff --git a/reef.txt b/reef.txt\nindex 1111111..2222222 100644\n--- a/reef.txt\n+++ b/reef.txt\n@@ -1 +1 @@\n-old\n+new\n\\ No newline at end of file\n"; + let files = parse_patch(patch).unwrap(); + let PatchPayload::Text(hunks) = &files[0].payload else { + panic!("expected text payload"); + }; + assert_eq!(hunks[0].lines[0].text, b"old\n".to_vec()); + assert_eq!(hunks[0].lines[1].text, b"new".to_vec()); + } + + #[test] + fn quoted_paths_unescape() { + assert_eq!(unquote("\"a/sp ace.txt\"").unwrap(), "a/sp ace.txt"); + assert_eq!(unquote("\"a/tab\\there\"").unwrap(), "a/tab\there"); + assert_eq!(unquote("\"a/\\303\\251\"").unwrap(), "a/é"); + assert!(unquote("\"a/broken").is_err()); + } + + #[test] + fn a_mailbox_splits_into_individual_patches() { + let mbox = concat!( + "From 1111111111111111111111111111111111111111 Mon Sep 17 00:00:00 2001\n", + "From: nel \n", + "Date: Tue, 5 Sep 2023 12:00:00 +0530\n", + "Subject: [PATCH 1/2] first\n", + "\n", + "body text\n", + "---\n", + " reef.txt | 1 +\n", + " 1 file changed, 1 insertion(+)\n", + "\n", + "diff --git a/reef.txt b/reef.txt\n", + "new file mode 100644\n", + "index 0000000..2222222\n", + "--- /dev/null\n", + "+++ b/reef.txt\n", + "@@ -0,0 +1 @@\n", + "+one\n", + "-- \n2.43.0\n\n", + "From 2222222222222222222222222222222222222222 Mon Sep 17 00:00:00 2001\n", + "From: =?UTF-8?q?t=C3=A9q?= \n", + "Date: Tue, 5 Sep 2023 13:00:00 +0530\n", + "Subject: [PATCH 2/2] second\n", + "Change-Id: I0123456789abcdef\n", + "\n", + "---\n", + "diff --git a/reef.txt b/reef.txt\n", + "index 2222222..3333333 100644\n", + "--- a/reef.txt\n", + "+++ b/reef.txt\n", + "@@ -1 +1 @@\n", + "-one\n", + "+two\n", + ); + let mails = parse_mailbox(mbox).unwrap(); + assert_eq!(mails.len(), 2); + assert_eq!(mails[0].author_name.as_str(), "nel"); + assert_eq!(mails[0].author_email.as_str(), "nel@oyster.cafe"); + assert_eq!(mails[0].subject, "first"); + assert_eq!(mails[0].body, "body text"); + assert_eq!(mails[0].commit_message(), "first\n\nbody text"); + assert_eq!(mails[0].files.len(), 1); + assert_eq!(mails[1].author_name.as_str(), "téq"); + assert_eq!( + mails[1].change_id, + Some(CommitChangeId::new("I0123456789abcdef").unwrap()) + ); + assert_eq!(mails[1].files[0].intent, FileIntent::Modify); + } + + #[test] + fn base85_decodes_lengths_and_rejects_garbage() { + let mut out = Vec::new(); + decode_base85_line("D00000", &mut out).unwrap(); + assert_eq!(out, vec![0, 0, 0, 0]); + let mut out = Vec::new(); + decode_base85_line("B00000", &mut out).unwrap(); + assert_eq!(out, vec![0, 0]); + assert!(decode_base85_line("D0000", &mut Vec::new()).is_err()); + assert!(decode_base85_line("D0\"000", &mut Vec::new()).is_err()); + assert!(decode_base85_line("?00000", &mut Vec::new()).is_err()); + } + + #[test] + fn malformed_input_yields_typed_errors() { + assert_eq!(parse_patch(" \n "), Err(PatchParseError::Empty)); + assert_eq!(parse_patch("hello world\n"), Err(PatchParseError::NoFiles)); + + let patch = "diff --git a/r.txt b/r.txt\n--- a/r.txt\n+++ b/r.txt\n@@ -0,0 +1 @@\n+a line that is wider than four bytes\n"; + let mut tight = Budget { remaining: 4 }; + assert_eq!( + parse_patch_budgeted(patch, &mut tight), + Err(malformed("patch exceeds total decompressed size budget")), + ); + let mut roomy = Budget { remaining: 1_000 }; + assert!(parse_patch_budgeted(patch, &mut roomy).is_ok()); + } + + #[test] + fn pathological_depth_inputs_do_not_overflow_the_stack() { + let huge = "a".repeat(1_000_000); + assert_eq!(unquote(&format!("\"{huge}\"")).unwrap(), huge); + let brackets = "[x]".repeat(500_000); + assert_eq!(strip_subject_prefix(&brackets), ""); + let encoded = format!("=?utf-8?q?{}?=", "=41".repeat(400_000)); + assert_eq!(decode_rfc2047(&encoded), "A".repeat(400_000)); + } + + #[test] + fn a_bare_email_from_header_becomes_both_name_and_email() { + let mbox = concat!( + "From 1111111111111111111111111111111111111111 Mon Sep 17 00:00:00 2001\n", + "From: nel@oyster.cafe\n", + "Date: Tue, 5 Sep 2023 12:00:00 +0000\n", + "Subject: [PATCH] bare\n", + "\n", + "diff --git a/reef.txt b/reef.txt\n", + "new file mode 100644\n", + "--- /dev/null\n", + "+++ b/reef.txt\n", + "@@ -0,0 +1 @@\n", + "+hi\n", + ); + let mails = parse_mailbox(mbox).unwrap(); + assert_eq!(mails[0].author_name.as_str(), "nel@oyster.cafe"); + assert_eq!(mails[0].author_email.as_str(), "nel@oyster.cafe"); + } +} diff --git a/knot2/crates/knot-git/src/reads.rs b/knot2/crates/knot-git/src/reads.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-git/src/reads.rs @@ -0,0 +1,618 @@ +use std::collections::{HashMap, HashSet}; +use std::ops::ControlFlow; +use std::time::Instant; + +use gix::bstr::ByteSlice; +use knot_types::{BranchName, Oid, RepoPath, TagName, UnixSeconds}; + +use crate::error::{GitError, backend}; +use crate::objects::{Commit, CommitRange, EntryKind, Identity, identity, map_kind}; +use crate::repo::Repo; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SizedEntry { + pub name: String, + pub oid: Oid, + pub kind: EntryKind, + pub size: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PathEntry { + pub oid: Oid, + pub kind: EntryKind, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LastCommit { + pub id: Oid, + pub subject: String, + pub time: UnixSeconds, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BranchTip { + Commit(Box), + Opaque { + id: Oid, + message: String, + created_at: UnixSeconds, + }, +} + +impl BranchTip { + pub fn created_at(&self) -> UnixSeconds { + match self { + BranchTip::Commit(commit) => commit.committer.time, + BranchTip::Opaque { created_at, .. } => *created_at, + } + } + + pub fn id(&self) -> Oid { + match self { + BranchTip::Commit(commit) => commit.id, + BranchTip::Opaque { id, .. } => *id, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BranchInfo { + pub name: BranchName, + pub tip: BranchTip, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AnnotatedTag { + pub tagger: Option, + pub pgp_signature: Option, + pub target: Oid, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TagInfo { + pub name: TagName, + pub id: Oid, + pub created_at: UnixSeconds, + pub message: String, + pub annotated: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Submodule { + pub name: String, + pub path: RepoPath, + pub url: String, + pub branch: Option, +} + +knot_types::scalar_newtype! { + pub struct LogSkip(usize); + pub struct LogLimit(usize); +} + +impl Repo { + pub fn resolve_revision(&self, spec: &str) -> Option { + if spec.is_empty() || spec.contains('\0') { + return None; + } + self.git() + .rev_parse_single(spec.as_bytes()) + .ok() + .map(|id| Oid::from(id.detach())) + } + + pub fn peel_to_commit(&self, oid: Oid) -> Result { + let peeled = self + .git() + .find_object(oid.object_id()) + .map_err(backend)? + .peel_tags_to_end() + .map_err(backend)?; + match peeled.kind { + gix::object::Kind::Commit => Ok(Oid::from(peeled.id)), + _ => Err(GitError::ObjectType { + oid, + expected: "commit", + }), + } + } + + fn walk_from( + &self, + start: Oid, + hidden: Option, + ) -> Result> + '_, GitError> { + let hidden = hidden.filter(|oid| self.contains(*oid)).map(Oid::object_id); + Ok(self + .git() + .rev_walk(Some(start.object_id())) + .sorting(gix::revision::walk::Sorting::ByCommitTime( + gix::traverse::commit::simple::CommitTimeOrder::NewestFirst, + )) + .with_hidden(hidden) + .all() + .map_err(|error| GitError::RevWalk(error.to_string()))? + .map(|info| { + info.map(|info| Oid::from(info.id)) + .map_err(|error| GitError::RevWalk(error.to_string())) + })) + } + + pub fn commits_between( + &self, + range: CommitRange, + limit: LogLimit, + ) -> Result, GitError> { + self.walk_from(range.head, Some(range.base))? + .take(limit.get()) + .collect() + } + + pub fn log_window( + &self, + start: Oid, + skip: LogSkip, + limit: LogLimit, + ) -> Result<(Vec, usize), GitError> { + self.walk_from(start, None)?.enumerate().try_fold( + (Vec::new(), 0usize), + |(mut window, _), (index, oid)| { + let oid = oid?; + if index >= skip.get() && window.len() < limit.get() { + window.push(self.find_commit(oid)?); + } + Ok((window, index + 1)) + }, + ) + } + + pub fn merge_base(&self, one: Oid, two: Oid) -> Result, GitError> { + use gix::repository::merge_base::Error; + match self.git().merge_base(one.object_id(), two.object_id()) { + Ok(id) => Ok(Some(Oid::from(id.detach()))), + Err(Error::NotFound { .. }) => Ok(None), + Err(error) => Err(backend(error)), + } + } + + // For security purposes. + // Without this, anyone who knows an oid can read objects from + // a deleted branch/ unreferenced push. + // + // COBs and forky staging refs aren't counted ofc. + // + // Oh btw to that end `advertised_refs()` *isn't* what upload-pack "advertises": + // upload-pack uses + // `advertised_refs_for(AdvertScope::Upload)` which also omits + // refs matching `transfer.hideRefs`/`uploadpack.hideRefs`. + pub fn reachable_from_public(&self, target: Oid) -> Result { + let tips: Vec = self + .advertised_refs()? + .iter() + // skip any broken refs + .filter_map(|record| self.peel_to_commit(record.target).ok()) + .collect(); + if tips.contains(&target) { + return Ok(true); + } + tips.iter().try_fold(false, |found, tip| { + Ok(found + || self + // Traverse graph, but shouldn't be too hard on CPU + // because `commit_graph_if_enabled` isn't directly + // on the object db. + .merge_base(target, *tip)? + .is_some_and(|base| base == target)) + }) + } + + pub fn branch_list(&self) -> Result, GitError> { + self.branches()? + .into_iter() + .filter_map(|record| record.name.branch_name().map(|name| (name, record.target))) + .map(|(name, target)| self.branch_tip(target).map(|tip| BranchInfo { name, tip })) + .collect() + } + + fn branch_tip(&self, target: Oid) -> Result { + let object = self + .git() + .find_object(target.object_id()) + .map_err(backend)?; + match object.kind { + gix::object::Kind::Commit => self + .find_commit(target) + .map(|commit| BranchTip::Commit(Box::new(commit))), + gix::object::Kind::Tag => { + let tag = object.try_into_tag().map_err(backend)?; + let decoded = tag.decode().map_err(backend)?; + let created_at = decoded + .tagger() + .map_err(|error| GitError::Decode(error.to_string()))? + .map(identity) + .transpose()? + .map(|tagger| tagger.time) + .unwrap_or(UnixSeconds::new(0)); + Ok(BranchTip::Opaque { + id: target, + message: decoded.message.to_string(), + created_at, + }) + } + _ => Ok(BranchTip::Opaque { + id: target, + message: String::new(), + created_at: UnixSeconds::new(0), + }), + } + } + + pub fn tag_list(&self) -> Result, GitError> { + self.tags()? + .into_iter() + .filter_map(|record| record.name.tag_name().map(|name| (name, record.target))) + .map(|(name, target)| self.tag_info(name, target)) + .collect() + } + + fn tag_info(&self, name: TagName, target: Oid) -> Result { + let object = self + .git() + .find_object(target.object_id()) + .map_err(backend)?; + match object.kind { + gix::object::Kind::Tag => { + let tag = object.try_into_tag().map_err(backend)?; + let decoded = tag.decode().map_err(backend)?; + let tagger = decoded + .tagger() + .map_err(|error| GitError::Decode(error.to_string()))? + .map(identity) + .transpose()?; + let created_at = tagger + .as_ref() + .map(|tagger| tagger.time) + .unwrap_or(UnixSeconds::new(0)); + Ok(TagInfo { + name, + id: target, + created_at, + message: decoded.message.to_string(), + annotated: Some(AnnotatedTag { + tagger, + pgp_signature: decoded.pgp_signature.map(|signature| signature.to_string()), + target: Oid::from(decoded.target()), + }), + }) + } + gix::object::Kind::Commit => { + let commit = self.find_commit(target)?; + Ok(TagInfo { + name, + id: target, + created_at: commit.committer.time, + message: commit.message, + annotated: None, + }) + } + _ => Ok(TagInfo { + name, + id: target, + created_at: UnixSeconds::new(0), + message: String::new(), + annotated: None, + }), + } + } + + fn dir_tree_id( + &self, + commit: Oid, + dir: Option<&RepoPath>, + ) -> Result, GitError> { + let root = self.commit_tree(commit)?; + let Some(dir) = dir else { + return Ok(Some(root)); + }; + let tree = self.git().find_tree(root).map_err(backend)?; + match tree.lookup_entry_by_path(dir.as_str()).map_err(backend)? { + Some(entry) if entry.mode().is_tree() => Ok(Some(entry.object_id())), + _ => Ok(None), + } + } + + pub(crate) fn root_tree(&self, commit: Oid) -> Result, GitError> { + let root = self.commit_tree(commit)?; + self.git().find_tree(root).map_err(backend) + } + + pub fn entry_at(&self, commit: Oid, path: &RepoPath) -> Result, GitError> { + let tree = self.root_tree(commit)?; + Ok(tree + .lookup_entry_by_path(path.as_str()) + .map_err(backend)? + .map(|entry| PathEntry { + oid: Oid::from(entry.object_id()), + kind: map_kind(entry.mode().kind()), + })) + } + + pub fn tree_entries_at( + &self, + commit: Oid, + path: Option<&RepoPath>, + ) -> Result>, GitError> { + let Some(path) = path else { + let tree = self.root_tree(commit)?; + return self.sized_entries(&tree).map(Some); + }; + match self.entry_at(commit, path)? { + None => Ok(None), + Some(entry) if entry.kind == EntryKind::Tree => self.tree_entries(entry.oid).map(Some), + Some(entry) if entry.kind == EntryKind::Commit => Ok(None), + Some(_) => Ok(Some(Vec::new())), + } + } + + pub fn tree_entries(&self, tree: Oid) -> Result, GitError> { + let tree = self.git().find_tree(tree.object_id()).map_err(backend)?; + self.sized_entries(&tree) + } + + fn sized_entries(&self, tree: &gix::Tree<'_>) -> Result, GitError> { + let decoded = tree + .decode() + .map_err(|error| GitError::Decode(error.to_string()))?; + decoded + .entries + .iter() + .map(|entry| { + let oid = Oid::from(entry.oid.to_owned()); + let kind = map_kind(entry.mode.kind()); + let size = match kind { + EntryKind::Blob | EntryKind::BlobExecutable | EntryKind::Link => self + .git() + .try_find_header(oid.object_id()) + .map_err(|error| GitError::Corrupt { + oid, + message: error.to_string(), + })? + .map(|header| header.size()) + .unwrap_or(0), + EntryKind::Tree | EntryKind::Commit => 0, + }; + Ok(SizedEntry { + name: entry.filename.to_string(), + oid, + kind, + size, + }) + }) + .collect() + } + + fn entry_oids_of_tree(&self, tree: gix::ObjectId) -> Result, GitError> { + let tree = self.git().find_tree(tree).map_err(backend)?; + let decoded = tree + .decode() + .map_err(|error| GitError::Decode(error.to_string()))?; + Ok(decoded + .entries + .iter() + .map(|entry| (entry.filename.to_string(), Oid::from(entry.oid.to_owned()))) + .collect()) + } + + pub fn last_commits( + &self, + start: Oid, + dir: Option<&RepoPath>, + names: &[String], + deadline: Option, + ) -> Result, GitError> { + let mut pending: HashSet<&str> = names.iter().map(String::as_str).collect(); + let mut attributed = HashMap::new(); + let mut dir_trees: HashMap> = HashMap::new(); + let mut dir_tree_of = |commit: Oid| -> Result, GitError> { + match dir_trees.get(&commit) { + Some(known) => Ok(*known), + None => { + let id = self.dir_tree_id(commit, dir)?; + dir_trees.insert(commit, id); + Ok(id) + } + } + }; + + let mut step = |oid: Result| -> Result { + if pending.is_empty() || deadline.is_some_and(|deadline| Instant::now() >= deadline) { + return Ok(false); + } + let oid = oid?; + let commit = self.find_commit(oid)?; + if commit.parents.len() > 1 { + return Ok(true); + } + let here_tree = dir_tree_of(oid)?; + let parent_tree = commit + .parents + .first() + .copied() + .map(&mut dir_tree_of) + .transpose()? + .flatten(); + if here_tree == parent_tree || here_tree.is_none() { + return Ok(true); + } + let here = self.entry_oids_of_tree(here_tree.expect("checked above"))?; + let parent = parent_tree + .map(|tree| self.entry_oids_of_tree(tree)) + .transpose()? + .unwrap_or_default(); + let changed: Vec = pending + .iter() + .filter(|name| here.contains_key(**name) && here.get(**name) != parent.get(**name)) + .map(|name| name.to_string()) + .collect(); + changed.iter().for_each(|name| { + pending.remove(name.as_str()); + }); + changed.into_iter().for_each(|name| { + attributed.insert( + name, + LastCommit { + id: oid, + subject: subject_line(&commit.message), + time: commit.author.time, + }, + ); + }); + Ok(true) + }; + + let flow = + self.walk_from(start, None)? + .try_for_each(|oid| -> ControlFlow> { + match step(oid) { + Ok(true) => ControlFlow::Continue(()), + Ok(false) => ControlFlow::Break(None), + Err(error) => ControlFlow::Break(Some(error)), + } + }); + match flow { + ControlFlow::Break(Some(error)) => Err(error), + _ => Ok(attributed), + } + } + + pub fn submodules(&self, commit: Oid) -> Result, GitError> { + let gitmodules = RepoPath::new(".gitmodules").expect("literal path is well-formed"); + let Some(entry) = self.entry_at(commit, &gitmodules)? else { + return Ok(Vec::new()); + }; + if !entry.kind.is_file() { + return Ok(Vec::new()); + } + let raw = self.read_blob(entry.oid)?; + Ok(parse_gitmodules(raw.as_bstr().to_str_lossy().as_ref())) + } +} + +fn subject_line(message: &str) -> String { + message.lines().next().unwrap_or_default().to_string() +} + +fn strip_config_comment(line: &str) -> String { + let flow = line.chars().try_fold( + (String::new(), false, false), + |(mut out, quoted, escaped), ch| match (escaped, quoted, ch) { + (false, false, '#' | ';') => ControlFlow::Break(out), + (false, _, '"') => { + out.push(ch); + ControlFlow::Continue((out, !quoted, false)) + } + (false, _, '\\') => { + out.push(ch); + ControlFlow::Continue((out, quoted, true)) + } + _ => { + out.push(ch); + ControlFlow::Continue((out, quoted, false)) + } + }, + ); + match flow { + ControlFlow::Continue((out, _, _)) | ControlFlow::Break(out) => out, + } +} + +fn unquote_config_value(raw: &str) -> String { + raw.trim() + .chars() + .fold((String::new(), false), |(mut out, escaped), ch| { + match (escaped, ch) { + (true, 'n') => { + out.push('\n'); + (out, false) + } + (true, 't') => { + out.push('\t'); + (out, false) + } + (true, 'b') => { + out.push('\u{0008}'); + (out, false) + } + (true, other) => { + out.push(other); + (out, false) + } + (false, '\\') => (out, true), + (false, '"') => (out, false), + (false, other) => { + out.push(other); + (out, false) + } + } + }) + .0 +} + +fn parse_gitmodules(content: &str) -> Vec { + struct Partial { + name: String, + path: Option, + url: Option, + branch: Option, + } + let finish = |partial: Partial| -> Option { + Some(Submodule { + name: partial.name, + path: RepoPath::new(partial.path?).ok()?, + url: partial.url?, + branch: partial + .branch + .and_then(|branch| BranchName::new(branch).ok()), + }) + }; + let (mut sections, last) = content.lines().map(strip_config_comment).fold( + (Vec::new(), None::), + |(mut done, current), line| { + let line = line.trim(); + if let Some(rest) = line.strip_prefix("[submodule \"") + && let Some(name) = rest.strip_suffix("\"]") + { + done.extend(current.and_then(&finish)); + return ( + done, + Some(Partial { + name: name.to_string(), + path: None, + url: None, + branch: None, + }), + ); + } + if line.starts_with('[') { + done.extend(current.and_then(&finish)); + return (done, None); + } + let current = current.map(|mut partial| { + if let Some((key, value)) = line.split_once('=') { + let value = unquote_config_value(value); + match key.trim() { + "path" => partial.path = Some(value), + "url" => partial.url = Some(value), + "branch" => partial.branch = Some(value), + _ => {} + } + } + partial + }); + (done, current) + }, + ); + sections.extend(last.and_then(&finish)); + sections +} diff --git a/knot2/crates/knot-git/src/repo.rs b/knot2/crates/knot-git/src/repo.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-git/src/repo.rs @@ -0,0 +1,1637 @@ +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, OnceLock}; + +use gix::refs::transaction::{Change, LogChange, PreviousValue, RefEdit, RefLog}; +use gix::refs::{FullName, Target}; +use knot_cache::{Cache, Moka, Weight}; +use knot_types::{ + BranchName, KnotId, ObjectFormat, Oid, RefName, RefTransition, RepoDid, UnixSeconds, +}; + +use crate::error::GitError; +use crate::objects::{Haves, PackBudget, Walked, Wants}; + +const RESERVED_PREFIX: &str = "refs/cobs/"; +const CHECKPOINT_PREFIX: &str = "refs/cob-checkpoints/"; +const HIDDEN_PREFIX: &str = "refs/hidden/"; +const REFLOG_COMMITTER_NAME: &str = "knot"; +const REFLOG_COMMITTER_EMAIL: &str = "noreply@knot"; +const HEADS_PREFIX: &str = "refs/heads/"; +const TAGS_PREFIX: &str = "refs/tags/"; +const MAX_SYMREF_DEPTH: usize = 5; +const ADVERT_BYTES_PER_REF: u64 = 128; + +fn tuned(mut git: gix::Repository) -> gix::Repository { + git.object_cache_size_if_unset(knot_resource::object_cache_bytes()); + pin_reflog_identity(&mut git); + git +} + +fn assembled(git: gix::Repository, path: PathBuf) -> Repo { + Repo { + git: tuned(git), + path, + commit_graph: OnceLock::new(), + } +} + +fn pin_reflog_identity(git: &mut gix::Repository) { + use gix::config::tree::{Committer, Core}; + let mut config = git.config_snapshot_mut(); + let pinned = config.set_value(&Core::LOG_ALL_REF_UPDATES, "true").is_ok() + && config + .set_value(&Committer::NAME, REFLOG_COMMITTER_NAME) + .is_ok() + && config + .set_value(&Committer::EMAIL, REFLOG_COMMITTER_EMAIL) + .is_ok(); + if pinned { + let _ = config.commit(); + } +} + +knot_types::scalar_newtype! { + struct RefEpoch(u64); + struct RefGeneration(u64); +} + +struct RefState { + lock: Mutex<()>, + generation: AtomicU64, + epoch: RefEpoch, +} + +type RefRegistry = Mutex>>; + +fn ref_registry() -> &'static RefRegistry { + static REGISTRY: OnceLock = OnceLock::new(); + REGISTRY.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn next_epoch() -> RefEpoch { + static EPOCH: AtomicU64 = AtomicU64::new(0); + RefEpoch::new(EPOCH.fetch_add(1, Ordering::Relaxed)) +} + +fn ref_state(git_dir: &Path) -> Arc { + let mut states = ref_registry() + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + Arc::clone(states.entry(git_dir.to_path_buf()).or_insert_with(|| { + Arc::new(RefState { + lock: Mutex::new(()), + generation: AtomicU64::new(0), + epoch: next_epoch(), + }) + })) +} + +fn forget_ref_state(git_dir: &Path) { + ref_registry() + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .remove(git_dir); +} + +type AdvertCache = Moka<(RefEpoch, RefGeneration), Arc>>; + +fn advert_cache() -> &'static Arc { + static CACHE: OnceLock> = OnceLock::new(); + CACHE.get_or_init(|| { + let cache = Arc::new(Moka::by_weight( + Weight::new(knot_resource::advert_cache_bytes()), + |refs: &Arc>| { + Weight::new( + (refs.len() as u64) + .max(1) + .saturating_mul(ADVERT_BYTES_PER_REF), + ) + }, + )); + knot_cache::register(&cache); + cache + }) +} + +fn safe_component(part: &str) -> bool { + !matches!(part, "." | "..") && !part.contains(['/', '\\', '\0']) +} + +pub fn repo_shard(did: &RepoDid) -> Result { + shard_components(did.as_str()) +} + +pub fn knot_shard(knot: &KnotId) -> Result { + shard_components(knot.as_str()) +} + +fn shard_components(did: &str) -> Result { + let mut parts = did.splitn(3, ':'); + parts.next(); + let method = parts.next().unwrap_or("did"); + let msid = parts.next().unwrap_or_default(); + let split = msid + .char_indices() + .nth(2) + .map(|(index, _)| index) + .unwrap_or(msid.len()); + let (shard, remainder) = msid.split_at(split); + if [method, shard, remainder] + .iter() + .any(|part| !safe_component(part)) + { + return Err(GitError::UnsafeRepoDid(did.to_string())); + } + Ok(PathBuf::from(method).join(shard).join(remainder)) +} + +#[derive(Debug, Clone)] +pub struct Layout { + scan_path: PathBuf, + head: RefName, + reserved: Option, + object_format: ObjectFormat, +} + +fn default_head() -> RefName { + RefName::new(format!("{HEADS_PREFIX}main")).expect("refs/heads/main is valid ref name") +} + +impl Layout { + pub fn new(scan_path: impl Into) -> Self { + Self { + scan_path: scan_path.into(), + head: default_head(), + reserved: None, + object_format: ObjectFormat::default(), + } + } + + pub fn with_default_branch(mut self, branch: BranchName) -> Self { + self.head = branch.head_ref(); + self + } + + pub fn with_object_format(mut self, object_format: ObjectFormat) -> Self { + self.object_format = object_format; + self + } + + pub fn reserving_meta(mut self, knot: &KnotId) -> Result { + let reserved = self.meta_path(knot)?; + self.reserved = Some(reserved); + Ok(self) + } + + pub fn repo_path(&self, did: &RepoDid) -> Result { + Ok(self.scan_path.join(shard_components(did.as_str())?)) + } + + pub fn scratch_dir(&self) -> &Path { + &self.scan_path + } + + pub fn meta_path(&self, knot: &KnotId) -> Result { + Ok(self.scan_path.join(shard_components(knot.as_str())?)) + } + + pub fn guarded_path(&self, did: &RepoDid) -> Result { + let path = self.repo_path(did)?; + match &self.reserved { + Some(reserved) if *reserved == path => { + Err(GitError::ReservedDid(did.as_str().to_string())) + } + _ => Ok(path), + } + } + + pub fn open(&self, did: &RepoDid) -> Result { + Repo::open(self.guarded_path(did)?) + } + + pub fn create(&self, did: &RepoDid) -> Result { + self.init_repo(self.guarded_path(did)?) + } + + pub fn remove(&self, did: &RepoDid) -> Result<(), GitError> { + let path = self.guarded_path(did)?; + if let Ok(repo) = Repo::open(&path) { + forget_ref_state(repo.git.git_dir()); + } + match std::fs::remove_dir_all(&path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(GitError::Remove { + path, + message: error.to_string(), + }), + } + } + + pub fn bootstrap_meta(&self, knot: &KnotId) -> Result { + init_bare_idempotent(self.meta_path(knot)?) + } + + fn init_repo(&self, path: PathBuf) -> Result { + let repo = Repo::create_with_format(path, self.object_format)?; + repo.set_head(&self.head)?; + Ok(repo) + } +} + +pub(crate) fn init_bare_with_format( + path: &Path, + format: ObjectFormat, +) -> Result { + let object_hash = (format != ObjectFormat::SHA1).then(|| format.kind()); + gix::ThreadSafeRepository::init_opts( + path, + gix::create::Kind::Bare, + gix::create::Options { + object_hash, + ..Default::default() + }, + gix::open::Options::default(), + ) + .map(Into::into) + .map_err(|error| error.to_string()) +} + +fn staging_path(parent: &Path) -> PathBuf { + static COUNTER: AtomicU64 = AtomicU64::new(0); + let nonce = COUNTER.fetch_add(1, Ordering::Relaxed); + parent.join(format!(".knot-staging.{}.{}", std::process::id(), nonce)) +} + +fn init_bare_idempotent(path: PathBuf) -> Result { + if let Ok(git) = gix::open(&path) { + return Ok(assembled(git, path)); + } + let parent = path.parent().ok_or_else(|| GitError::Create { + path: path.clone(), + message: "meta path has no parent directory".to_string(), + })?; + std::fs::create_dir_all(parent).map_err(|error| GitError::Create { + path: path.clone(), + message: error.to_string(), + })?; + let staging = staging_path(parent); + let _ = std::fs::remove_dir_all(&staging); + gix::init_bare(&staging).map_err(|error| GitError::Create { + path: staging.clone(), + message: error.to_string(), + })?; + match std::fs::rename(&staging, &path) { + Ok(()) => Repo::open(path), + Err(_) => { + let _ = std::fs::remove_dir_all(&staging); + Repo::open(path) + } + } +} + +pub struct Repo { + git: gix::Repository, + path: PathBuf, + commit_graph: OnceLock>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RefRecord { + pub name: RefName, + pub target: Oid, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PackHash(String); + +impl PackHash { + pub fn new(value: impl Into) -> Option { + let value = value.into(); + (matches!(value.len(), 40 | 64) && value.bytes().all(|byte| byte.is_ascii_hexdigit())) + .then_some(Self(value)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PackfileUrl(String); + +impl PackfileUrl { + pub fn new(value: impl Into) -> Option { + let value = value.into(); + let authority = value + .strip_prefix("https://") + .or_else(|| value.strip_prefix("http://")) + .filter(|rest| !rest.is_empty() && !rest.starts_with('/')); + (authority.is_some() && !value.chars().any(|c| c.is_whitespace() || c.is_control())) + .then_some(Self(value)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PackfileUri { + pub oid: Oid, + pub pack_hash: PackHash, + pub uri: PackfileUrl, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HeadRef { + pub name: RefName, + pub target: Oid, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReflogUpdate { + pub name: RefName, + pub old: Option, + pub new: Oid, + pub seconds: UnixSeconds, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RefUpdate { + Create { name: RefName, new: Oid }, + Update { name: RefName, old: Oid, new: Oid }, + Delete { name: RefName, old: Oid }, +} + +impl RefUpdate { + pub fn name(&self) -> &RefName { + match self { + RefUpdate::Create { name, .. } + | RefUpdate::Update { name, .. } + | RefUpdate::Delete { name, .. } => name, + } + } + + pub fn transition(&self) -> RefTransition { + match self { + RefUpdate::Create { new, .. } => RefTransition::Create { new: *new }, + RefUpdate::Update { old, new, .. } => RefTransition::Advance { + old: *old, + new: *new, + }, + RefUpdate::Delete { old, .. } => RefTransition::Delete { old: *old }, + } + } +} + +pub fn is_reserved(name: &RefName) -> bool { + screens_reserved(name.as_str()) +} + +pub fn screens_reserved(raw: &str) -> bool { + raw.starts_with(RESERVED_PREFIX) || raw.starts_with(CHECKPOINT_PREFIX) +} + +fn is_hidden(name: &RefName) -> bool { + name.as_str().starts_with(HIDDEN_PREFIX) +} + +pub fn is_branch(name: &RefName) -> bool { + name.as_str().starts_with(HEADS_PREFIX) +} + +pub fn is_public_ref(name: &RefName) -> bool { + !is_reserved(name) && !is_hidden(name) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AdvertScope { + Upload, + Receive, +} + +impl AdvertScope { + fn config_key(self) -> &'static str { + match self { + AdvertScope::Upload => "uploadpack.hideRefs", + AdvertScope::Receive => "receive.hideRefs", + } + } +} + +fn ref_hidden_by(name: &RefName, patterns: &[String]) -> bool { + patterns.iter().any(|pattern| { + name.as_str() == pattern || name.as_str().starts_with(&format!("{pattern}/")) + }) +} + +pub(crate) fn fsync_if_present(path: &Path) -> Result<(), GitError> { + knot_resource::fsync_path(path).map_err(|error| GitError::Fsync { + path: error.path, + message: error.source.to_string(), + }) +} + +impl Repo { + pub fn open(path: impl Into) -> Result { + let path = path.into(); + let git = gix::open(&path).map_err(|error| GitError::Open { + path: path.clone(), + message: error.to_string(), + })?; + Ok(assembled(git, path)) + } + + pub fn create(path: impl Into) -> Result { + Self::create_with_format(path, ObjectFormat::default()) + } + + pub fn create_with_format( + path: impl Into, + format: ObjectFormat, + ) -> Result { + let path = path.into(); + if path.exists() { + return Err(GitError::AlreadyExists(path)); + } + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(|error| GitError::Create { + path: path.clone(), + message: error.to_string(), + })?; + } + let git = init_bare_with_format(&path, format).map_err(|message| GitError::Create { + path: path.clone(), + message, + })?; + Ok(assembled(git, path)) + } + + pub fn git(&self) -> &gix::Repository { + &self.git + } + + pub fn object_format(&self) -> ObjectFormat { + ObjectFormat::from_kind(self.git.object_hash()) + } + + pub fn path(&self) -> &Path { + &self.path + } + + pub fn objects_dir(&self) -> PathBuf { + self.git.git_dir().join("objects") + } + + pub fn references(&self) -> Result, GitError> { + self.git + .references() + .map_err(|error| GitError::Backend(error.to_string()))? + .all() + .map_err(|error| GitError::Backend(error.to_string()))? + .filter_map(|reference| { + let reference = match reference { + Ok(reference) => reference, + Err(error) => return Some(Err(GitError::Backend(error.to_string()))), + }; + let raw = reference.name().as_bstr().to_string(); + let target = Oid::from(self.direct_target(&reference, MAX_SYMREF_DEPTH)?); + let name = RefName::new(raw).ok()?; + Some(Ok(RefRecord { name, target })) + }) + .collect() + } + + pub fn reflog_updates_since(&self, since_seconds: UnixSeconds) -> Vec { + let Ok(references) = self.git.references() else { + return Vec::new(); + }; + let Ok(all) = references.all() else { + return Vec::new(); + }; + all.filter_map(Result::ok) + .filter(|reference| { + let name = reference.name().as_bstr().to_string(); + name.starts_with(HEADS_PREFIX) || name.starts_with(TAGS_PREFIX) + }) + .flat_map(|reference| self.ref_reflog_since(&reference, since_seconds)) + .collect() + } + + fn ref_reflog_since( + &self, + reference: &gix::Reference<'_>, + since_seconds: UnixSeconds, + ) -> Vec { + let Ok(name) = RefName::new(reference.name().as_bstr().to_string()) else { + return Vec::new(); + }; + let mut platform = reference.log_iter(); + let Ok(Some(reverse)) = platform.rev() else { + return Vec::new(); + }; + reverse + .filter_map(Result::ok) + .take_while(|line| UnixSeconds::new(line.signature.time.seconds) >= since_seconds) + .filter_map(|line| { + (!line.new_oid.is_null()).then(|| ReflogUpdate { + name: name.clone(), + old: Some(line.previous_oid) + .filter(|previous| !previous.is_null()) + .map(Oid::from), + new: Oid::from(line.new_oid), + seconds: UnixSeconds::new(line.signature.time.seconds), + }) + }) + .collect() + } + + pub fn find_ref(&self, name: &RefName) -> Result, GitError> { + match self + .git + .try_find_reference(name.as_str()) + .map_err(|error| GitError::Backend(error.to_string()))? + { + Some(reference) => Ok(self + .direct_target(&reference, MAX_SYMREF_DEPTH) + .map(Oid::from)), + None => Ok(None), + } + } + + fn direct_target(&self, reference: &gix::Reference<'_>, depth: usize) -> Option { + match (depth, reference.follow()) { + (_, None) => reference.try_id().map(|id| id.detach()), + (0, Some(_)) => None, + (_, Some(Ok(next))) => self.direct_target(&next, depth - 1), + (_, Some(Err(_))) => None, + } + } + + pub(crate) fn commit_graph(&self) -> Option<&gix::commitgraph::Graph> { + self.commit_graph + .get_or_init(|| self.git.commit_graph().ok()) + .as_ref() + } + + pub fn with_ref_lock(&self, body: impl FnOnce() -> R) -> R { + let state = ref_state(self.git.git_dir()); + let _guard = state + .lock + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + body() + } + + fn locked_value(&self, body: impl FnOnce() -> R) -> R { + self.with_ref_lock(|| { + let outcome = body(); + let state = ref_state(self.git.git_dir()); + let previous = RefGeneration::new(state.generation.fetch_add(1, Ordering::SeqCst)); + advert_cache().invalidate(&(state.epoch, previous)); + outcome + }) + } + + pub(crate) fn locked( + &self, + body: impl FnOnce() -> Result, + ) -> Result { + self.locked_value(body) + } + + pub fn with_ref_txn(&self, body: impl FnOnce(&RefTxn<'_>) -> R) -> R { + self.locked_value(|| body(&RefTxn { repo: self })) + } + + pub fn set_head(&self, target: &RefName) -> Result<(), GitError> { + self.locked(|| self.set_head_locked(target)) + } + + pub fn set_head_sealed( + &self, + target: &RefName, + seal: impl FnOnce() -> R, + ) -> Result { + self.locked(|| { + self.set_head_locked(target)?; + Ok(seal()) + }) + } + + fn set_head_locked(&self, target: &RefName) -> Result<(), GitError> { + let raw = target.as_str(); + let target_name = FullName::try_from(raw).map_err(|error| GitError::Reference { + name: raw.to_string(), + message: error.to_string(), + })?; + let edit = RefEdit { + change: Change::Update { + log: LogChange { + mode: RefLog::AndReference, + force_create_reflog: false, + message: "knot set HEAD".into(), + }, + expected: PreviousValue::Any, + new: Target::Symbolic(target_name), + }, + name: FullName::try_from("HEAD").map_err(|error| GitError::Reference { + name: "HEAD".to_string(), + message: error.to_string(), + })?, + deref: false, + }; + self.git + .edit_reference(edit) + .map_err(|error| GitError::Reference { + name: "HEAD".to_string(), + message: error.to_string(), + })?; + let git_dir = self.git.git_dir(); + fsync_if_present(&git_dir.join("HEAD"))?; + fsync_if_present(git_dir) + } + + fn persist_refs<'a>( + &self, + mut names: impl Iterator, + ) -> Result<(), GitError> { + let git_dir = self.git.git_dir(); + let mut dirs = BTreeSet::from([git_dir.to_path_buf()]); + names.try_for_each(|name| -> Result<(), GitError> { + let ref_path = git_dir.join(name.as_str()); + fsync_if_present(&ref_path)?; + std::iter::successors(ref_path.parent(), |path| path.parent()) + .take_while(|path| path.starts_with(git_dir)) + .for_each(|path| { + dirs.insert(path.to_path_buf()); + }); + Ok(()) + })?; + fsync_if_present(&git_dir.join("packed-refs"))?; + dirs.iter().try_for_each(|dir| fsync_if_present(dir)) + } + + pub fn origin_url(&self) -> Option { + self.git + .config_snapshot() + .string("remote.origin.url") + .map(|value| value.to_string()) + } + + pub fn set_origin_url(&self, url: &str) -> Result<(), GitError> { + let path = self.git.git_dir().join("config"); + let report = |message: String| GitError::Config { + path: path.clone(), + message, + }; + let mut file = + gix::config::File::from_path_no_includes(path.clone(), gix::config::Source::Local) + .map_err(|error| report(error.to_string()))?; + file.set_raw_value_by( + "remote", + Some(gix::bstr::BStr::new("origin")), + "url", + gix::bstr::BStr::new(url), + ) + .map_err(|error| report(error.to_string()))?; + knot_resource::atomic_write(&path, knot_resource::FileMode::Inherited, |out| { + file.write_to(out) + .map_err(|error| report(error.to_string())) + })?; + fsync_if_present(self.git.git_dir()) + } + + pub fn branches(&self) -> Result, GitError> { + self.references().map(|records| { + records + .into_iter() + .filter(|record| record.name.as_str().starts_with(HEADS_PREFIX)) + .collect() + }) + } + + pub fn tags(&self) -> Result, GitError> { + self.references().map(|records| { + records + .into_iter() + .filter(|record| record.name.as_str().starts_with(TAGS_PREFIX)) + .collect() + }) + } + + pub fn advertised_refs(&self) -> Result>, GitError> { + let state = ref_state(self.git.git_dir()); + let key = ( + state.epoch, + RefGeneration::new(state.generation.load(Ordering::SeqCst)), + ); + advert_cache() + .get_or_try_insert_with(key, || self.public_refs().map(Arc::new)) + .map_err(|error: Arc| GitError::Backend(error.to_string())) + } + + fn public_refs(&self) -> Result, GitError> { + self.references().map(|records| { + records + .into_iter() + .filter(|record| is_public_ref(&record.name)) + .collect() + }) + } + + pub fn advertised_refs_for(&self, scope: AdvertScope) -> Result, GitError> { + let patterns = self.hidden_ref_patterns(scope); + let base = self.advertised_refs()?; + if patterns.is_empty() { + return Ok(base.to_vec()); + } + Ok(base + .iter() + .filter(|record| !ref_hidden_by(&record.name, &patterns)) + .cloned() + .collect()) + } + + pub fn blob_packfile_uris(&self) -> Vec { + let snapshot = self.git.config_snapshot(); + snapshot + .strings("uploadpack.blobPackfileUri") + .into_iter() + .flatten() + .filter_map(|value| { + let text = value.to_string(); + let mut parts = text.split_whitespace(); + let oid = Oid::from_hex(parts.next()?).ok()?; + let pack_hash = PackHash::new(parts.next()?)?; + let uri = PackfileUrl::new(parts.next()?)?; + Some(PackfileUri { + oid, + pack_hash, + uri, + }) + }) + .collect() + } + + fn hidden_ref_patterns(&self, scope: AdvertScope) -> Vec { + let snapshot = self.git.config_snapshot(); + ["transfer.hideRefs", scope.config_key()] + .into_iter() + .filter_map(|key| snapshot.strings(key)) + .flatten() + .map(|value| value.to_string()) + .collect() + } + + pub fn head(&self) -> Option { + let target = self.git.head_id().ok()?.detach(); + let raw = self.git.head_name().ok()??.as_bstr().to_string(); + let name = RefName::new(raw).ok()?; + Some(HeadRef { + name, + target: Oid::from(target), + }) + } + + pub fn default_branch(&self) -> Option { + let raw = self.git.head_name().ok()??.as_bstr().to_string(); + RefName::new(raw).ok() + } + + pub fn contains(&self, oid: Oid) -> bool { + self.git.has_object(oid.object_id()) + } + + pub fn is_shallow(&self) -> bool { + self.git.is_shallow() + } + + pub(crate) fn shallow_grafts(&self) -> Result, GitError> { + Ok(self + .git + .shallow_commits() + .map_err(|error| GitError::Decode(format!("shallow file: {error}")))? + .map(|commits| commits.iter().copied().collect()) + .unwrap_or_default()) + } + + pub fn rev_walk(&self, wants: Wants, haves: Haves) -> Result, GitError> { + let mut walked = Walked::new(PackBudget::unbounded()); + self.rev_walk_each(wants, haves, &mut walked) + } + + pub(crate) fn rev_walk_each( + &self, + wants: Wants<'_>, + haves: Haves<'_>, + walked: &mut Walked, + ) -> Result, GitError> { + let present: Vec = haves + .as_slice() + .iter() + .copied() + .filter(|oid| self.contains(*oid)) + .map(Oid::object_id) + .collect(); + let mut probe = *walked; + let collected = self + .git + .rev_walk(wants.as_slice().iter().copied().map(Oid::object_id)) + .with_hidden(present.iter().copied()) + .all() + .ok() + .and_then(|walk| { + walk.map(|info| { + probe.tick()?; + info.map(|info| Oid::from(info.id)) + .map_err(|error| GitError::RevWalk(error.to_string())) + }) + .collect::, _>>() + .ok() + }); + match collected { + Some(commits) => { + *walked = probe; + Ok(commits) + } + None => self.rev_walk_lenient(wants.as_slice(), &present, walked), + } + } + + fn rev_walk_lenient( + &self, + wants: &[Oid], + hidden_tips: &[gix::ObjectId], + walked: &mut Walked, + ) -> Result, GitError> { + let mut hidden: HashSet = HashSet::new(); + let mut stack = hidden_tips.to_vec(); + while let Some(oid) = stack.pop() { + if hidden.insert(oid) + && let Ok((_, parents)) = self.commit_tree_and_parents(Oid::from(oid)) + { + stack.extend(parents); + } + } + let mut visited: HashSet = HashSet::new(); + let mut commits = Vec::new(); + let mut stack: Vec = wants.iter().copied().map(Oid::object_id).collect(); + while let Some(oid) = stack.pop() { + if hidden.contains(&oid) || !visited.insert(oid) { + continue; + } + walked.tick()?; + let (_, parents) = self.commit_tree_and_parents(Oid::from(oid))?; + commits.push(Oid::from(oid)); + stack.extend(parents); + } + Ok(commits) + } + + fn ref_edit(update: &RefUpdate, via_head: bool) -> Result { + let edited = if via_head { + "HEAD" + } else { + update.name().as_str() + }; + let name = FullName::try_from(edited).map_err(|error| GitError::Reference { + name: edited.to_string(), + message: error.to_string(), + })?; + let log = LogChange { + mode: RefLog::AndReference, + force_create_reflog: true, + message: "knot ref update".into(), + }; + let change = match update { + RefUpdate::Create { new, .. } => Change::Update { + log, + expected: PreviousValue::MustNotExist, + new: Target::Object(new.object_id()), + }, + RefUpdate::Update { old, new, .. } => Change::Update { + log, + expected: PreviousValue::MustExistAndMatch(Target::Object(old.object_id())), + new: Target::Object(new.object_id()), + }, + RefUpdate::Delete { old, .. } => Change::Delete { + expected: PreviousValue::MustExistAndMatch(Target::Object(old.object_id())), + log: RefLog::AndReference, + }, + }; + Ok(RefEdit { + change, + name, + deref: via_head, + }) + } + + fn updates_head_branch(&self, update: &RefUpdate) -> bool { + !matches!(update, RefUpdate::Delete { .. }) + && self + .default_branch() + .is_some_and(|head| head.as_str() == update.name().as_str()) + } + + pub fn update_ref(&self, update: &RefUpdate) -> Result<(), GitError> { + self.locked(|| self.update_ref_locked(update)) + } + + pub fn update_ref_sealed( + &self, + update: &RefUpdate, + seal: impl FnOnce() -> R, + ) -> Result { + self.locked(|| { + self.update_ref_locked(update)?; + Ok(seal()) + }) + } + + fn update_ref_locked(&self, update: &RefUpdate) -> Result<(), GitError> { + let raw = update.name().as_str().to_string(); + let via_head = self.updates_head_branch(update); + self.git + .edit_reference(Self::ref_edit(update, via_head)?) + .map_err(|error| GitError::Reference { + name: raw, + message: error.to_string(), + })?; + self.persist_refs(std::iter::once(update.name())) + } + + pub fn update_refs(&self, updates: &[RefUpdate]) -> Result<(), GitError> { + self.locked(|| self.update_refs_locked(updates)) + } + + pub fn update_refs_sealed( + &self, + updates: &[RefUpdate], + seal: impl FnOnce() -> R, + ) -> Result { + self.locked(|| { + self.update_refs_locked(updates)?; + Ok(seal()) + }) + } + + fn reject_df_conflicts(&self, updates: &[RefUpdate]) -> Result<(), GitError> { + let creates: Vec<&str> = updates + .iter() + .filter_map(|update| match update { + RefUpdate::Create { name, .. } => Some(name.as_str()), + _ => None, + }) + .collect(); + if creates.is_empty() { + return Ok(()); + } + let deletes: HashSet<&str> = updates + .iter() + .filter_map(|update| match update { + RefUpdate::Delete { name, .. } => Some(name.as_str()), + _ => None, + }) + .collect(); + let names: Vec = self + .references()? + .iter() + .map(|record| record.name.as_str().to_string()) + .filter(|name| !deletes.contains(name.as_str())) + .chain(creates.iter().copied().map(str::to_string)) + .collect(); + let name_set: HashSet<&str> = names.iter().map(String::as_str).collect(); + names + .iter() + .find_map(|name| { + name.match_indices('/') + .map(|(at, _)| &name[..at]) + .find(|ancestor| name_set.contains(ancestor)) + .map(|ancestor| (ancestor.to_string(), name.clone())) + }) + .map_or(Ok(()), |(directory, leaf)| { + Err(GitError::AtomicRefs(format!( + "d/f conflict: {directory} blocks {leaf}" + ))) + }) + } + + fn update_refs_locked(&self, updates: &[RefUpdate]) -> Result<(), GitError> { + self.reject_df_conflicts(updates)?; + let edits = updates + .iter() + .map(|update| { + let via_head = self.updates_head_branch(update); + Self::ref_edit(update, via_head) + }) + .collect::, _>>()?; + self.git + .edit_references(edits) + .map_err(|error| GitError::AtomicRefs(error.to_string()))?; + self.persist_refs(updates.iter().map(RefUpdate::name)) + } +} + +pub struct RefTxn<'a> { + repo: &'a Repo, +} + +impl RefTxn<'_> { + pub fn update_ref(&self, update: &RefUpdate) -> Result<(), GitError> { + self.repo.update_ref_locked(update) + } + + pub fn update_refs(&self, updates: &[RefUpdate]) -> Result<(), GitError> { + self.repo.update_refs_locked(updates) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const A: &str = "1111111111111111111111111111111111111111"; + const B: &str = "2222222222222222222222222222222222222222"; + + fn oid(hex: &str) -> Oid { + Oid::from_hex(hex).unwrap() + } + + fn head_ref() -> RefName { + RefName::new("refs/heads/main").unwrap() + } + + fn repo() -> (tempfile::TempDir, Layout, RepoDid) { + let dir = tempfile::tempdir().unwrap(); + let layout = Layout::new(dir.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + (dir, layout, did) + } + + #[test] + fn layout_paths_shard_and_stay_within_scan() { + let layout = Layout::new("/srv/git"); + let cases: &[(&str, &str)] = &[ + ("did:plc:squid", "plc/sq/uid"), + ("did:web:oyster.cafe", "web/oy/ster.cafe"), + ("did:web:nel.pet", "web/ne/l.pet"), + ]; + cases.iter().for_each(|&(raw, suffix)| { + let path = layout.repo_path(&RepoDid::new(raw).unwrap()).unwrap(); + assert!(path.ends_with(suffix), "{path:?} missing shard {suffix}"); + assert!(path.starts_with("/srv/git"), "{path:?} escaped scan path"); + }); + } + + #[test] + fn dot_only_did_cannot_escape_scan_path() { + let layout = Layout::new("/srv/git/scan"); + ["did:plc:....", "did:web:....", "did:plc:...", "did:plc:.."] + .into_iter() + .map(|raw| RepoDid::new(raw).unwrap()) + .for_each(|did| { + assert!( + matches!(layout.repo_path(&did), Err(GitError::UnsafeRepoDid(_))), + "dot-only method-specific-id must be refused, never resolved to path" + ); + assert!(matches!(layout.open(&did), Err(GitError::UnsafeRepoDid(_)))); + assert!(matches!( + layout.create(&did), + Err(GitError::UnsafeRepoDid(_)) + )); + }); + + let real = RepoDid::new("did:web:oyster.cafe").unwrap(); + assert!( + layout.repo_path(&real).is_ok(), + "legitimate did:web with dots in its domain must still resolve" + ); + } + + #[test] + fn meta_repo_path_is_sharded_and_never_collides() { + let layout = Layout::new("/srv/git"); + let knot = KnotId::new("did:web:oyster.cafe").unwrap(); + let meta = layout.meta_path(&knot).unwrap(); + assert!(meta.ends_with("web/oy/ster.cafe")); + ["did:plc:squid", "did:web:nel.pet"] + .into_iter() + .map(|raw| RepoDid::new(raw).unwrap()) + .for_each(|did| { + assert_ne!(layout.repo_path(&did).unwrap(), meta); + }); + } + + #[test] + fn bootstrap_meta_creates_then_opens_idempotently() { + let dir = tempfile::tempdir().unwrap(); + let layout = Layout::new(dir.path()); + let knot = KnotId::new("did:web:oyster.cafe").unwrap(); + + let created = layout.bootstrap_meta(&knot).unwrap(); + assert!(created.references().unwrap().is_empty()); + assert_eq!(created.path(), layout.meta_path(&knot).unwrap()); + + let reopened = layout.bootstrap_meta(&knot).unwrap(); + assert_eq!(reopened.path(), layout.meta_path(&knot).unwrap()); + } + + #[test] + fn concurrent_bootstrap_meta_converges_for_every_caller() { + let dir = tempfile::tempdir().unwrap(); + let layout = Layout::new(dir.path()); + let knot = KnotId::new("did:web:oyster.cafe").unwrap(); + let meta = layout.meta_path(&knot).unwrap(); + + let paths = std::thread::scope(|scope| { + let handles: Vec<_> = (0..8) + .map(|_| { + let layout = layout.clone(); + let knot = knot.clone(); + scope.spawn(move || { + layout + .bootstrap_meta(&knot) + .map(|repo| repo.path().to_path_buf()) + }) + }) + .collect(); + handles + .into_iter() + .map(|handle| handle.join().unwrap()) + .collect::>() + }); + + assert!( + paths + .iter() + .all(|outcome| matches!(outcome, Ok(path) if path == &meta)), + "every racing bootstrap must converge on one meta repo, not fail: {paths:?}" + ); + let reopened = layout.bootstrap_meta(&knot).unwrap(); + assert!(reopened.references().unwrap().is_empty()); + } + + #[test] + fn reserving_meta_refuses_the_knot_did_for_open_and_create() { + let dir = tempfile::tempdir().unwrap(); + let knot = KnotId::new("did:web:oyster.cafe").unwrap(); + let layout = Layout::new(dir.path()).reserving_meta(&knot).unwrap(); + layout.bootstrap_meta(&knot).unwrap(); + + let knot_as_repo = RepoDid::new("did:web:oyster.cafe").unwrap(); + assert!(matches!( + layout.open(&knot_as_repo), + Err(GitError::ReservedDid(_)) + )); + assert!(matches!( + layout.create(&knot_as_repo), + Err(GitError::ReservedDid(_)) + )); + + let ordinary = RepoDid::new("did:plc:squid").unwrap(); + assert!(layout.create(&ordinary).is_ok()); + assert!(layout.open(&ordinary).is_ok()); + } + + #[test] + fn creating_a_bare_repo_is_sha1_and_rejects_a_second_create() { + let (_dir, layout, did) = repo(); + + let repo = layout.create(&did).unwrap(); + assert!(repo.references().unwrap().is_empty()); + assert!(repo.head().is_none()); + assert_eq!(repo.object_format(), ObjectFormat::SHA1); + + assert!(layout.open(&did).unwrap().references().unwrap().is_empty()); + assert!(matches!( + layout.create(&did), + Err(GitError::AlreadyExists(_)) + )); + } + + #[test] + fn with_ref_txn_holds_the_ref_lock_across_its_whole_body() { + use std::sync::Mutex; + use std::sync::mpsc::channel; + + let (_dir, layout, did) = repo(); + let repo = layout.create(&did).unwrap(); + let contender_repo = layout.open(&did).unwrap(); + + let order: Mutex> = Mutex::new(Vec::new()); + let order_ref = ℴ + let (contending, observed) = channel(); + + std::thread::scope(|scope| { + repo.with_ref_txn(|_txn| { + order_ref.lock().unwrap().push("txn-enter"); + scope.spawn(move || { + contending.send(()).unwrap(); + contender_repo.with_ref_lock(|| order_ref.lock().unwrap().push("contender")); + }); + observed.recv().unwrap(); + std::thread::yield_now(); + order_ref.lock().unwrap().push("txn-exit"); + }); + }); + + assert_eq!( + *order.lock().unwrap(), + ["txn-enter", "txn-exit", "contender"], + "object migration runs inside the transaction body, so no other ref-lock holder can observe the half-applied push" + ); + } + + #[test] + fn create_with_sha256_object_format_round_trips() { + let dir = tempfile::tempdir().unwrap(); + let layout = Layout::new(dir.path()).with_object_format(ObjectFormat::SHA256); + let did = RepoDid::new("did:plc:squid").unwrap(); + + let repo = layout.create(&did).unwrap(); + assert_eq!(repo.object_format(), ObjectFormat::SHA256); + let oid = Oid::from(repo.git().write_blob(b"hello sha256\n").unwrap().detach()); + assert_eq!( + oid.to_hex().len(), + 64, + "sha256 repo names objects with 32-byte digests" + ); + + let reopened = layout.open(&did).unwrap(); + assert_eq!( + reopened.object_format(), + ObjectFormat::SHA256, + "object format survives reopen, read from repo config" + ); + } + + #[test] + fn compare_and_swap_governs_every_ref_write() { + let (_dir, layout, did) = repo(); + let repo = layout.create(&did).unwrap(); + let main = head_ref(); + + repo.update_ref(&RefUpdate::Create { + name: main.clone(), + new: oid(A), + }) + .unwrap(); + assert!( + repo.update_ref(&RefUpdate::Create { + name: main.clone(), + new: oid(B), + }) + .is_err() + ); + assert!( + repo.update_ref(&RefUpdate::Update { + name: main.clone(), + old: oid(B), + new: oid(A), + }) + .is_err() + ); + repo.update_ref(&RefUpdate::Update { + name: main.clone(), + old: oid(A), + new: oid(B), + }) + .unwrap(); + let refs = repo.references().unwrap(); + assert_eq!(refs.len(), 1); + assert_eq!(refs[0].target, oid(B)); + + drop(repo); + let repo = layout.open(&did).unwrap(); + assert_eq!( + repo.find_ref(&main).unwrap(), + Some(oid(B)), + "update is visible after reopen" + ); + + repo.update_ref(&RefUpdate::Delete { + name: main.clone(), + old: oid(B), + }) + .unwrap(); + assert!(repo.references().unwrap().is_empty()); + + let x = RefName::new("refs/heads/x").unwrap(); + let y = RefName::new("refs/heads/y").unwrap(); + repo.update_refs(&[ + RefUpdate::Create { + name: x.clone(), + new: oid(A), + }, + RefUpdate::Create { + name: y.clone(), + new: oid(A), + }, + ]) + .unwrap(); + let result = repo.update_refs(&[ + RefUpdate::Update { + name: x.clone(), + old: oid(A), + new: oid(B), + }, + RefUpdate::Update { + name: y.clone(), + old: oid(B), + new: oid(A), + }, + ]); + assert!( + result.is_err(), + "batch with one stale compare-and-swap must fail as a whole" + ); + assert_eq!( + repo.find_ref(&x).unwrap(), + Some(oid(A)), + "valid edit in failed batch must roll back" + ); + assert_eq!(repo.find_ref(&y).unwrap(), Some(oid(A))); + } + + #[test] + fn ref_writes_leave_a_recoverable_reflog_under_the_knot_identity() { + let (_dir, layout, did) = repo(); + let repo = layout.create(&did).unwrap(); + + let committer = repo + .git() + .committer() + .expect("committer is always pinned so reflog writes never depend on ambient config") + .expect("pinned committer signature parses"); + assert_eq!(committer.name.to_string(), REFLOG_COMMITTER_NAME); + assert_eq!(committer.email.to_string(), REFLOG_COMMITTER_EMAIL); + + repo.update_ref(&RefUpdate::Create { + name: head_ref(), + new: oid(A), + }) + .unwrap(); + repo.update_ref(&RefUpdate::Update { + name: head_ref(), + old: oid(A), + new: oid(B), + }) + .unwrap(); + repo.update_ref(&RefUpdate::Create { + name: RefName::new("refs/tags/v1").unwrap(), + new: oid(A), + }) + .unwrap(); + + let logs = repo.git().git_dir().join("logs"); + let branch = std::fs::read_to_string(logs.join("refs/heads/main")) + .expect("branch update must leave reflog so clobbering push is recoverable"); + assert!( + branch.contains(A) && branch.contains(B) && branch.contains(REFLOG_COMMITTER_NAME), + "branch reflog records both tips under knot identity:\n{branch}" + ); + assert!( + logs.join("refs/tags/v1").exists(), + "force_create_reflog must log tags too, not just conventional refs/heads set" + ); + + let updates = repo.reflog_updates_since(UnixSeconds::new(0)); + let head_new: Vec = updates + .iter() + .filter(|update| update.name == head_ref()) + .map(|update| update.new) + .collect(); + assert!( + head_new.contains(&oid(A)) && head_new.contains(&oid(B)), + "both branch tips are recovered from reflog: {head_new:?}" + ); + let create = updates + .iter() + .find(|update| update.name == head_ref() && update.new == oid(A)) + .unwrap(); + assert_eq!(create.old, None, "branch creation has no previous oid"); + let update = updates + .iter() + .find(|update| update.name == head_ref() && update.new == oid(B)) + .unwrap(); + assert_eq!(update.old, Some(oid(A)), "branch update records prior oid"); + assert!( + updates + .iter() + .any(|update| update.name.as_str() == "refs/tags/v1" && update.new == oid(A)), + "tag update is recovered too" + ); + assert!( + repo.reflog_updates_since(UnixSeconds::new(i64::MAX)) + .is_empty(), + "horizon past every entry filters whole reflog out" + ); + } + + #[test] + fn advertisement_hides_reserved_refs_and_tracks_each_change() { + let (_dir, layout, did) = repo(); + let repo = layout.create(&did).unwrap(); + let main = head_ref(); + let feature = RefName::new("refs/heads/feature").unwrap(); + + repo.update_ref(&RefUpdate::Create { + name: main.clone(), + new: oid(A), + }) + .unwrap(); + let first = repo.advertised_refs().unwrap(); + assert_eq!(first.len(), 1); + assert_eq!(first[0].name.as_str(), "refs/heads/main"); + assert_eq!( + repo.advertised_refs().unwrap(), + first, + "repeated advertisement with no ref change serves same answer" + ); + + repo.update_ref(&RefUpdate::Create { + name: RefName::new("refs/cobs/sh.tangled.repo.collaborator/limpet").unwrap(), + new: oid(B), + }) + .unwrap(); + let advertised = repo.advertised_refs().unwrap(); + assert_eq!(advertised.len(), 1, "cob ref is hidden from advertisement"); + assert_eq!(advertised[0].name.as_str(), "refs/heads/main"); + assert_eq!(repo.references().unwrap().len(), 2); + assert!(is_reserved(&RefName::new("refs/cobs/x/y").unwrap())); + + repo.update_ref(&RefUpdate::Create { + name: feature.clone(), + new: oid(B), + }) + .unwrap(); + assert_eq!( + repo.advertised_refs().unwrap().len(), + 2, + "ref created after advertisement invalidates cached answer" + ); + repo.update_ref(&RefUpdate::Delete { + name: feature, + old: oid(B), + }) + .unwrap(); + assert_eq!( + repo.advertised_refs().unwrap().len(), + 1, + "delete after advertisement invalidates cached answer" + ); + + drop(repo); + layout.remove(&did).unwrap(); + let repo = layout.create(&did).unwrap(); + repo.update_ref(&RefUpdate::Create { + name: RefName::new("refs/heads/new").unwrap(), + new: oid(B), + }) + .unwrap(); + let advertised = repo.advertised_refs().unwrap(); + assert_eq!( + advertised.len(), + 1, + "recreated repo advertises only its own ref" + ); + assert_eq!( + advertised[0].name.as_str(), + "refs/heads/new", + "deleted repo's cached advertisement mustn't survive recreation at same path" + ); + } + + #[test] + fn advertisement_reflects_each_update_under_concurrent_readers() { + let (_dir, layout, did) = repo(); + let repo = layout.create(&did).unwrap(); + let main = head_ref(); + repo.update_ref(&RefUpdate::Create { + name: main.clone(), + new: oid(A), + }) + .unwrap(); + drop(repo); + + let stop = std::sync::atomic::AtomicBool::new(false); + std::thread::scope(|scope| { + (0..4).for_each(|_| { + scope.spawn(|| { + let reader = layout.open(&did).unwrap(); + while !stop.load(Ordering::Relaxed) { + let refs = reader.advertised_refs().unwrap(); + assert_eq!(refs.len(), 1, "live branch is advertised exactly once"); + assert!( + refs[0].target == oid(A) || refs[0].target == oid(B), + "reader must never observe value branch never held" + ); + } + }); + }); + + let writer = layout.open(&did).unwrap(); + (0..64).for_each(|round| { + let (old, new) = if round % 2 == 0 { + (oid(A), oid(B)) + } else { + (oid(B), oid(A)) + }; + writer + .update_ref(&RefUpdate::Update { + name: main.clone(), + old, + new, + }) + .unwrap(); + assert_eq!( + writer.advertised_refs().unwrap()[0].target, + new, + "advertisement taken after update reflects that update" + ); + }); + stop.store(true, Ordering::Relaxed); + }); + } + + #[test] + fn create_honors_configured_default_branch() { + let dir = tempfile::tempdir().unwrap(); + let layout = Layout::new(dir.path()).with_default_branch(BranchName::new("trunk").unwrap()); + let repo = layout + .create(&RepoDid::new("did:plc:squid").unwrap()) + .unwrap(); + assert_eq!(repo.default_branch().unwrap().as_str(), "refs/heads/trunk"); + } + + #[test] + fn concurrent_create_has_exactly_one_winner() { + let (_dir, layout, did) = repo(); + layout.create(&did).unwrap(); + let main = head_ref(); + + const C: &str = "3333333333333333333333333333333333333333"; + const D: &str = "4444444444444444444444444444444444444444"; + let winners = std::thread::scope(|scope| { + let handles = [A, B, C, D].map(|hex| { + let layout = layout.clone(); + let did = did.clone(); + let main = main.clone(); + scope.spawn(move || { + layout + .open(&did) + .unwrap() + .update_ref(&RefUpdate::Create { + name: main, + new: oid(hex), + }) + .is_ok() + }) + }); + handles + .into_iter() + .map(|handle| handle.join().unwrap()) + .filter(|created| *created) + .count() + }); + + assert_eq!(winners, 1, "concurrent creates of one ref mustn't both win"); + assert_eq!(layout.open(&did).unwrap().references().unwrap().len(), 1); + } + + #[test] + fn symref_cycle_does_not_overflow_references() { + let (_dir, layout, did) = repo(); + let repo = layout.create(&did).unwrap(); + let symref = |name: &str, target: &str| RefEdit { + change: Change::Update { + log: LogChange { + mode: RefLog::AndReference, + force_create_reflog: false, + message: "cycle".into(), + }, + expected: PreviousValue::Any, + new: Target::Symbolic(FullName::try_from(target).unwrap()), + }, + name: FullName::try_from(name).unwrap(), + deref: false, + }; + repo.git() + .edit_reference(symref("refs/cycle/a", "refs/cycle/b")) + .unwrap(); + repo.git() + .edit_reference(symref("refs/cycle/b", "refs/cycle/a")) + .unwrap(); + + let refs = repo.references().unwrap(); + assert!( + refs.iter() + .all(|record| !record.name.as_str().starts_with("refs/cycle/")), + "cyclic symref must be skipped, not resolved" + ); + } +} diff --git a/knot2/crates/knot-git/src/staging.rs b/knot2/crates/knot-git/src/staging.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-git/src/staging.rs @@ -0,0 +1,144 @@ +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use crate::error::GitError; +use crate::repo::{Repo, init_bare_with_format}; + +pub const INCOMING_PREFIX: &str = ".knot-incoming."; + +fn incoming_path(git_dir: &Path) -> PathBuf { + static COUNTER: AtomicU64 = AtomicU64::new(0); + let nonce = COUNTER.fetch_add(1, Ordering::Relaxed); + git_dir.join(format!("{INCOMING_PREFIX}{}.{nonce}", std::process::id())) +} + +pub struct Staging { + dir: PathBuf, + repo: Repo, +} + +impl Staging { + pub fn new(live: &Repo) -> Result { + let dir = incoming_path(live.path()); + let _ = std::fs::remove_dir_all(&dir); + init_bare_with_format(&dir, live.object_format()) + .map_err(|error| GitError::Staging(format!("init: {error}")))?; + let info = dir.join("objects").join("info"); + std::fs::create_dir_all(&info) + .map_err(|error| GitError::Staging(format!("objects/info: {error}")))?; + std::fs::write( + info.join("alternates"), + format!("{}\n", live.objects_dir().display()), + ) + .map_err(|error| GitError::Staging(format!("alternates: {error}")))?; + match Repo::open(&dir) { + Ok(repo) => Ok(Self { dir, repo }), + Err(error) => { + let _ = std::fs::remove_dir_all(&dir); + Err(error) + } + } + } + + pub fn repo(&self) -> &Repo { + &self.repo + } + + pub fn migrate_into(&self, live: &Repo) -> Result<(), GitError> { + migrate( + &StagingObjects(self.repo.objects_dir()), + &LiveObjects(live.objects_dir()), + ) + } +} + +impl Drop for Staging { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.dir); + } +} + +fn is_shard(name: &str) -> bool { + name.len() == 2 && name.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +fn fsync_object_path(path: &Path) -> Result<(), GitError> { + let report = + |error: std::io::Error| GitError::Staging(format!("fsync {}: {error}", path.display())); + match std::fs::File::open(path) { + Ok(file) => file.sync_all().map_err(report), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(report(error)), + } +} + +struct StagingObjects(PathBuf); + +struct LiveObjects(PathBuf); + +fn migrate(from_objects: &StagingObjects, to_objects: &LiveObjects) -> Result<(), GitError> { + migrate_packs(from_objects, to_objects)?; + migrate_loose(from_objects, to_objects) +} + +fn migrate_packs(from_objects: &StagingObjects, to_objects: &LiveObjects) -> Result<(), GitError> { + let from_pack = from_objects.0.join("pack"); + let entries = match std::fs::read_dir(&from_pack) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(GitError::Staging(error.to_string())), + }; + let to_pack = to_objects.0.join("pack"); + std::fs::create_dir_all(&to_pack).map_err(|error| GitError::Staging(error.to_string()))?; + let names: Vec = entries + .filter_map(Result::ok) + .filter(|entry| { + entry + .file_type() + .map(|kind| kind.is_file()) + .unwrap_or(false) + }) + .map(|entry| entry.file_name()) + .collect(); + let is_idx = + |name: &std::ffi::OsString| Path::new(name).extension().is_some_and(|ext| ext == "idx"); + let ordered = names + .iter() + .filter(|name| !is_idx(name)) + .chain(names.iter().filter(|name| is_idx(name))); + ordered.into_iter().try_for_each(|name| { + let dest = to_pack.join(name); + std::fs::rename(from_pack.join(name), &dest) + .map_err(|error| GitError::Staging(format!("migrate pack file: {error}")))?; + fsync_object_path(&dest) + })?; + fsync_object_path(&to_pack) +} + +fn migrate_loose(from_objects: &StagingObjects, to_objects: &LiveObjects) -> Result<(), GitError> { + let entries = + std::fs::read_dir(&from_objects.0).map_err(|error| GitError::Staging(error.to_string()))?; + entries + .filter_map(Result::ok) + .filter(|entry| { + entry.file_type().map(|kind| kind.is_dir()).unwrap_or(false) + && entry.file_name().to_str().is_some_and(is_shard) + }) + .try_for_each(|shard| { + let dest_shard = to_objects.0.join(shard.file_name()); + std::fs::create_dir_all(&dest_shard) + .map_err(|error| GitError::Staging(error.to_string()))?; + std::fs::read_dir(shard.path()) + .map_err(|error| GitError::Staging(error.to_string()))? + .filter_map(Result::ok) + .try_for_each(|object| { + let dest = dest_shard.join(object.file_name()); + std::fs::rename(object.path(), &dest).map_err(|error| { + GitError::Staging(format!("migrate loose object: {error}")) + })?; + fsync_object_path(&dest) + })?; + fsync_object_path(&dest_shard) + })?; + fsync_object_path(&to_objects.0) +} diff --git a/knot2/crates/knot-git/tests/apply.rs b/knot2/crates/knot-git/tests/apply.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-git/tests/apply.rs @@ -0,0 +1,436 @@ +use std::path::Path; + +use knot_git::{ + ApplyOutcome, ConflictReason, Identity, Layout, NewCommit, PatchApplier, RefUpdate, Repo, + is_format_patch, parse_mailbox, parse_patch, +}; +use knot_types::{AuthorName, Email, Oid, RefName, RepoDid, UnixSeconds}; + +mod common; +use common::{git_ok as git, seeded}; + +fn stage_and_commit(work: &Path, message: &str) { + git(work, &["add", "-A"]); + git(work, &["commit", "-q", "-m", message]); +} + +fn push_main(work: &Path, layout: &Layout, did: &RepoDid) { + let bare = layout.repo_path(did).unwrap(); + git(work, &["push", "-q", bare.to_str().unwrap(), "main"]); + git(&bare, &["symbolic-ref", "HEAD", "refs/heads/main"]); +} + +fn main_ref() -> RefName { + RefName::new("refs/heads/main").unwrap() +} + +fn committer() -> Identity { + Identity { + name: AuthorName::new("Tangled"), + email: Email::new("noreply@tangled.sh"), + time: UnixSeconds::new(1_700_000_000), + offset_seconds: 0, + } +} + +fn apply_mailbox_natively(bare: &Repo, patch: &str) -> Vec { + let mails = parse_mailbox(patch).unwrap(); + let tip = bare.find_ref(&main_ref()).unwrap().unwrap(); + let base_tree = bare.find_commit(tip).unwrap().tree; + let mut applier = PatchApplier::new(bare, tip); + let (commits, _, new_tip) = mails.iter().fold( + (Vec::new(), base_tree, tip), + |(mut commits, tree, parent), mail| { + let staged = match applier.step(&mail.files).unwrap() { + ApplyOutcome::Clean(staged) => staged, + ApplyOutcome::Conflicted(conflicts) => { + panic!("expected clean apply, got conflicts {conflicts:?}") + } + }; + let next_tree = bare.write_staged_tree(tree, &staged).unwrap(); + let commit = bare + .write_commit(&NewCommit { + tree: next_tree, + parents: vec![parent], + author: Identity { + name: mail.author_name.clone(), + email: mail.author_email.clone(), + time: UnixSeconds::new(1_700_000_000), + offset_seconds: 0, + }, + committer: committer(), + message: mail.commit_message(), + extra_headers: mail + .change_id + .iter() + .map(|id| ("change-id".to_string(), id.as_str().as_bytes().to_vec())) + .collect(), + }) + .unwrap(); + commits.push(commit); + (commits, next_tree, commit) + }, + ); + bare.update_ref(&RefUpdate::Update { + name: main_ref(), + old: tip, + new: new_tip, + }) + .unwrap(); + commits +} + +#[test] +fn a_format_patch_series_applies_tree_identical_to_git_am() { + let (_scan, work_dir, layout, did) = seeded(); + let work = work_dir.path(); + + let blob: Vec = (0u32..8192).map(|i| (i * 31 % 251) as u8).collect(); + std::fs::write(work.join("a.txt"), "alpha\nbeta\ngamma\n").unwrap(); + std::fs::create_dir_all(work.join("sub")).unwrap(); + std::fs::write(work.join("sub/inner.txt"), "nested\n").unwrap(); + std::fs::write(work.join("noeol.txt"), "tail without newline").unwrap(); + std::fs::write(work.join("data.bin"), &blob).unwrap(); + std::fs::write(work.join("drop.bin"), [0u8, 1, 2, 3, 0, 9]).unwrap(); + stage_and_commit(work, "base"); + push_main(work, &layout, &did); + let base = git(work, &["rev-parse", "HEAD"]); + + std::fs::write(work.join("a.txt"), "alpha\nBETA\ngamma\n").unwrap(); + stage_and_commit(work, "first subject\n\nfirst body line"); + + git(work, &["mv", "a.txt", "moved.txt"]); + std::fs::write(work.join("moved.txt"), "alpha\nBETA\ngamma\ndelta\n").unwrap(); + std::fs::write(work.join("run.sh"), "#!/bin/sh\necho reef\n").unwrap(); + git(work, &["add", "-A"]); + git(work, &["update-index", "--chmod=+x", "run.sh"]); + git(work, &["commit", "-q", "-m", "second"]); + + std::fs::remove_file(work.join("noeol.txt")).unwrap(); + std::fs::write(work.join("sp ace.txt"), "spaced\n").unwrap(); + std::fs::write(work.join("café.txt"), "unicode\n").unwrap(); + stage_and_commit(work, "third"); + + let mutated: Vec = blob + .iter() + .copied() + .chain([0u8, 255, 254, 7]) + .map(|byte| match byte { + 42 => 24, + other => other, + }) + .collect(); + std::fs::write(work.join("data.bin"), &mutated).unwrap(); + std::fs::remove_file(work.join("drop.bin")).unwrap(); + std::fs::write(work.join("new.bin"), [9u8, 0, 8, 0, 7]).unwrap(); + stage_and_commit(work, "binary churn"); + + let patch = git( + work, + &["format-patch", "--stdout", &format!("{base}..HEAD")], + ); + assert!(is_format_patch(&patch)); + assert!(patch.contains("GIT binary patch")); + + let bare = layout.open(&did).unwrap(); + let ours = apply_mailbox_natively(&bare, &patch); + + let expected: Vec = git(work, &["rev-list", "--reverse", &format!("{base}..HEAD")]) + .lines() + .map(str::to_string) + .collect(); + assert_eq!(ours.len(), expected.len()); + ours.iter().zip(&expected).for_each(|(our_oid, real)| { + let our_commit = bare.find_commit(*our_oid).unwrap(); + let real_tree = + Oid::from_hex(&git(work, &["rev-parse", &format!("{real}^{{tree}}")])).unwrap(); + assert_eq!( + our_commit.tree, real_tree, + "natively applied tree must be byte-identical to git am's" + ); + let real_message = git(work, &["log", "-1", "--format=%B", real]); + assert_eq!(our_commit.message.trim_end(), real_message.trim_end()); + assert_eq!(our_commit.author.name.as_str(), "nel"); + assert_eq!(our_commit.author.email.as_str(), "nel@oyster.cafe"); + assert_eq!(our_commit.committer.name.as_str(), "Tangled"); + assert_eq!(our_commit.committer.email.as_str(), "noreply@tangled.sh"); + }); + assert_eq!( + bare.find_ref(&main_ref()).unwrap(), + Some(*ours.last().unwrap()) + ); + + git(work, &["checkout", "-q", "-b", "subline"]); + std::fs::write(work.join("seed.txt"), "one\n").unwrap(); + stage_and_commit(work, "seed one"); + let old_oid = git(work, &["rev-parse", "HEAD"]); + std::fs::write(work.join("seed.txt"), "two\n").unwrap(); + stage_and_commit(work, "seed two"); + let new_oid = git(work, &["rev-parse", "HEAD"]); + git( + work, + &[ + "update-index", + "--add", + "--cacheinfo", + &format!("160000,{old_oid},vendor"), + ], + ); + git(work, &["commit", "-q", "-m", "add submodule"]); + let sub_base = git(work, &["rev-parse", "HEAD"]); + let bare_path = layout.repo_path(&did).unwrap(); + git( + work, + &[ + "push", + "-q", + "-f", + bare_path.to_str().unwrap(), + &format!("{sub_base}:refs/heads/main"), + ], + ); + + git( + work, + &[ + "update-index", + "--cacheinfo", + &format!("160000,{new_oid},vendor"), + ], + ); + git(work, &["commit", "-q", "-m", "bump submodule"]); + let bump = git( + work, + &["format-patch", "--stdout", &format!("{sub_base}..HEAD")], + ); + assert!(bump.contains("Subproject commit")); + + let bumped = layout.open(&did).unwrap(); + let applied = apply_mailbox_natively(&bumped, &bump); + let real_tree = Oid::from_hex(&git(work, &["rev-parse", "HEAD^{tree}"])).unwrap(); + assert_eq!(bumped.find_commit(applied[0]).unwrap().tree, real_tree); +} + +#[test] +fn a_unified_diff_applies_tree_identical_to_git_apply() { + let (_scan, work_dir, layout, did) = seeded(); + let work = work_dir.path(); + + std::fs::write(work.join("a.txt"), "one\ntwo\nthree\n").unwrap(); + std::fs::write(work.join("gone.txt"), "doomed\n").unwrap(); + std::fs::write(work.join("noeol.txt"), "no newline here").unwrap(); + stage_and_commit(work, "base"); + push_main(work, &layout, &did); + let base = git(work, &["rev-parse", "HEAD"]); + + std::fs::write(work.join("a.txt"), "one\nTWO\nthree\nfour\n").unwrap(); + std::fs::remove_file(work.join("gone.txt")).unwrap(); + std::fs::write(work.join("fresh.txt"), "brand new\n").unwrap(); + std::fs::write(work.join("noeol.txt"), "still no newline").unwrap(); + stage_and_commit(work, "changes"); + + let patch = git(work, &["diff", &base, "HEAD"]); + assert!(!is_format_patch(&patch)); + let files = parse_patch(&patch).unwrap(); + + let bare = layout.open(&did).unwrap(); + let tip = bare.find_ref(&main_ref()).unwrap().unwrap(); + let base_tree = bare.find_commit(tip).unwrap().tree; + let mut applier = PatchApplier::new(&bare, tip); + let staged = match applier.step(&files).unwrap() { + ApplyOutcome::Clean(staged) => staged, + ApplyOutcome::Conflicted(conflicts) => panic!("unexpected conflicts {conflicts:?}"), + }; + let our_tree = bare.write_staged_tree(base_tree, &staged).unwrap(); + + let real_tree = Oid::from_hex(&git(work, &["rev-parse", "HEAD^{tree}"])).unwrap(); + assert_eq!(our_tree, real_tree); +} + +#[test] +fn a_stale_patch_conflicts_instead_of_applying() { + let (_scan, work_dir, layout, did) = seeded(); + let work = work_dir.path(); + + std::fs::write(work.join("a.txt"), "original\n").unwrap(); + stage_and_commit(work, "base"); + let base = git(work, &["rev-parse", "HEAD"]); + + std::fs::write(work.join("a.txt"), "patched from original\n").unwrap(); + stage_and_commit(work, "feature"); + let patch = git(work, &["diff", &base, "HEAD"]); + + git(work, &["checkout", "-q", &base]); + git(work, &["checkout", "-q", "-b", "drifted"]); + std::fs::write(work.join("a.txt"), "diverged\n").unwrap(); + stage_and_commit(work, "drift"); + git(work, &["branch", "-q", "-f", "main", "HEAD"]); + push_main(work, &layout, &did); + + let bare = layout.open(&did).unwrap(); + let tip = bare.find_ref(&main_ref()).unwrap().unwrap(); + let files = parse_patch(&patch).unwrap(); + let mut applier = PatchApplier::new(&bare, tip); + match applier.step(&files).unwrap() { + ApplyOutcome::Conflicted(conflicts) => { + assert_eq!(conflicts.len(), 1); + assert_eq!(conflicts[0].path, "a.txt"); + assert_eq!(conflicts[0].reason, ConflictReason::DoesNotApply); + } + ApplyOutcome::Clean(_) => panic!("stale patch mustn't apply cleanly"), + } +} + +fn git_apply_applies(cwd: &Path, patch: &str) -> bool { + use std::io::Write; + use std::process::Stdio; + let mut child = knot_fixtures::command(cwd) + .args(["apply", "--check"]) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("git is available"); + child + .stdin + .take() + .expect("stdin is piped") + .write_all(patch.as_bytes()) + .expect("write patch to git apply"); + child.wait().expect("git apply completes").success() +} + +#[test] +fn apply_verdict_matches_git_apply() { + let (_scan, work_dir, layout, did) = seeded(); + let work = work_dir.path(); + std::fs::write(work.join("a.txt"), "one\ntwo\nthree\nfour\nfive\n").unwrap(); + stage_and_commit(work, "base"); + push_main(work, &layout, &did); + + let bare = layout.open(&did).unwrap(); + let tip = bare.find_ref(&main_ref()).unwrap().unwrap(); + + let cases: [&str; 8] = [ + "diff --git a/a.txt b/a.txt\n--- a/a.txt\n+++ b/a.txt\n@@ -1,3 +1,3 @@\n one\n-two\n+TWO\n three\n", + "diff --git a/a.txt b/a.txt\n--- a/a.txt\n+++ b/a.txt\n@@ -1,3 +1,3 @@\n nope\n-gone\n+HERE\n zero\n", + "diff --git a/a.txt b/a.txt\nnew file mode 100644\n--- /dev/null\n+++ b/a.txt\n@@ -0,0 +1 @@\n+x\n", + "diff --git a/ghost.txt b/ghost.txt\ndeleted file mode 100644\n--- a/ghost.txt\n+++ /dev/null\n@@ -1 +0,0 @@\n-x\n", + "diff --git a/fresh.txt b/fresh.txt\nnew file mode 100644\n--- /dev/null\n+++ b/fresh.txt\n@@ -0,0 +1 @@\n+brand new\n", + "diff --git a/a.txt b/a.txt\n--- a/a.txt\n+++ b/a.txt\n@@ -3,1 +3,1 @@\n-three\n+THREE\n", + "diff --git a/a.txt b/a.txt\n--- a/a.txt\n+++ b/a.txt\n@@ -5,1 +5,1 @@\n-five\n+FIVE\n", + "diff --git a/a.txt b/a.txt\n--- a/a.txt\n+++ b/a.txt\n@@ -2,1 +1,0 @@\n-two\n", + ]; + + cases.iter().for_each(|patch| { + let git_clean = git_apply_applies(work, patch); + let mut applier = PatchApplier::new(&bare, tip); + let ours_clean = matches!( + applier.step(&parse_patch(patch).unwrap()).unwrap(), + ApplyOutcome::Clean(_) + ); + assert_eq!( + git_clean, ours_clean, + "verdict disagrees with git apply for patch:\n{patch}" + ); + }); +} + +fn git_apply_to_worktree_fails(cwd: &Path, patch: &str) -> bool { + use std::io::Write; + use std::process::Stdio; + let mut child = knot_fixtures::command(cwd) + .args(["apply"]) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("git is available"); + child + .stdin + .take() + .expect("stdin is piped") + .write_all(patch.as_bytes()) + .expect("write patch to git apply"); + !child.wait().expect("git apply completes").success() +} + +#[test] +fn refused_patches_conflict_with_the_reason_git_apply_rejects() { + let (_scan, work_dir, layout, did) = seeded(); + let work = work_dir.path(); + std::fs::write(work.join("a.txt"), "present\n").unwrap(); + std::fs::write(work.join("dir"), "i am a file\n").unwrap(); + stage_and_commit(work, "base"); + push_main(work, &layout, &did); + + let bare = layout.open(&did).unwrap(); + let tip = bare.find_ref(&main_ref()).unwrap().unwrap(); + + let create_existing = concat!( + "diff --git a/a.txt b/a.txt\n", + "new file mode 100644\n", + "--- /dev/null\n", + "+++ b/a.txt\n", + "@@ -0,0 +1 @@\n", + "+x\n", + ); + let delete_missing = concat!( + "diff --git a/ghost.txt b/ghost.txt\n", + "deleted file mode 100644\n", + "--- a/ghost.txt\n", + "+++ /dev/null\n", + "@@ -1 +0,0 @@\n", + "-x\n", + ); + let escape = concat!( + "diff --git a/../escape.txt b/../escape.txt\n", + "new file mode 100644\n", + "--- /dev/null\n", + "+++ b/../escape.txt\n", + "@@ -0,0 +1 @@\n", + "+boom\n", + ); + let under_a_file = concat!( + "diff --git a/dir/inner.txt b/dir/inner.txt\n", + "new file mode 100644\n", + "--- /dev/null\n", + "+++ b/dir/inner.txt\n", + "@@ -0,0 +1 @@\n", + "+nested\n", + ); + + let cases: &[(&str, ConflictReason, &str)] = &[ + ( + create_existing, + ConflictReason::AlreadyExists, + "file already exists", + ), + ( + delete_missing, + ConflictReason::DoesNotExist, + "file doesn't exist", + ), + (escape, ConflictReason::DoesNotApply, "patch doesn't apply"), + ( + under_a_file, + ConflictReason::DoesNotApply, + "patch doesn't apply", + ), + ]; + + cases.iter().for_each(|(patch, reason, message)| { + assert!( + git_apply_to_worktree_fails(work, patch), + "git apply must refuse:\n{patch}" + ); + let mut applier = PatchApplier::new(&bare, tip); + match applier.step(&parse_patch(patch).unwrap()).unwrap() { + ApplyOutcome::Conflicted(conflicts) => { + assert_eq!(conflicts[0].reason, *reason); + assert_eq!(conflicts[0].reason.as_str(), *message); + } + ApplyOutcome::Clean(_) => panic!("must conflict, not apply cleanly:\n{patch}"), + } + }); +} diff --git a/knot2/crates/knot-git/tests/bitmap.rs b/knot2/crates/knot-git/tests/bitmap.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-git/tests/bitmap.rs @@ -0,0 +1,217 @@ +use std::collections::HashSet; +use std::io::Read; +use std::path::{Path, PathBuf}; + +use knot_git::{ + Haves, Repo, Wants, reachable_via_bitmap, verbatim_clone_pack, write_bitmap, write_midx_bitmap, +}; +use knot_types::Oid; + +mod common; +use common::{commit_file as commit, git, git_available, git_ok as ok}; + +fn test_bitmap(path: &Path) { + let (ok, report) = git(path, &["rev-list", "--test-bitmap", "HEAD"]); + assert!(ok, "git rejected the bitmap: {report}"); +} + +fn seed(format: &str) -> tempfile::TempDir { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path(); + ok( + path, + &[ + "init", + "-q", + "--object-format", + format, + "--initial-branch", + "main", + ], + ); + commit(path, "a.txt", "alpha\n", "root"); + commit(path, "b.txt", "beta\n", "second"); + ok(path, &["checkout", "-q", "-b", "feature"]); + commit(path, "c.txt", "gamma\n", "feature work"); + ok(path, &["checkout", "-q", "main"]); + commit(path, "d.txt", "delta\n", "more main"); + ok(path, &["tag", "-a", "v1", "-m", "release one"]); + ok(path, &["repack", "-adq"]); + dir +} + +fn find_pack(path: &Path, suffix: &str) -> PathBuf { + std::fs::read_dir(path.join(".git/objects/pack")) + .unwrap() + .filter_map(Result::ok) + .map(|e| e.path()) + .find(|p| { + p.file_name() + .and_then(|n| n.to_str()) + .is_some_and(|n| n.ends_with(suffix)) + }) + .unwrap_or_else(|| panic!("a pack file ending {suffix}")) +} + +fn idx_count(path: &Path) -> usize { + std::fs::read_dir(path.join(".git/objects/pack")) + .unwrap() + .filter_map(Result::ok) + .filter(|e| e.path().extension().is_some_and(|ext| ext == "idx")) + .count() +} + +fn rev_parse(path: &Path, name: &str) -> Oid { + Oid::from_hex(ok(path, &["rev-parse", name]).trim()).unwrap() +} + +fn rev_list(path: &Path, revs: &[&str]) -> HashSet { + let args: Vec<&str> = ["rev-list", "--objects"] + .into_iter() + .chain(revs.iter().copied()) + .collect(); + ok(path, &args) + .lines() + .filter_map(|line| line.split_whitespace().next()) + .filter_map(|token| Oid::from_hex(token).ok()) + .collect() +} + +fn closure(repo: &Repo, wants: &[Oid], haves: &[Oid]) -> HashSet { + reachable_via_bitmap(repo, Wants::new(wants), Haves::new(haves)) + .unwrap() + .expect("bitmap fast path resolves") + .into_iter() + .collect() +} + +fn walked(repo: &Repo, want: Oid) -> HashSet { + repo.select_pack_objects(Wants::new(&[want]), Haves::new(&[])) + .unwrap() + .into_iter() + .collect() +} + +fn reject_corruption(repo: &Repo, bitmap: &Path, head: Oid) { + let intact = std::fs::read(bitmap).unwrap(); + let mut flipped = intact.clone(); + *flipped.last_mut().unwrap() ^= 0xff; + std::fs::write(bitmap, &flipped).unwrap(); + assert!( + reachable_via_bitmap(repo, Wants::new(&[head]), Haves::new(&[])).is_err(), + "a corrupt checksum is rejected" + ); + std::fs::write(bitmap, &intact[..intact.len() / 2]).unwrap(); + assert!( + reachable_via_bitmap(repo, Wants::new(&[head]), Haves::new(&[])).is_err(), + "a truncated bitmap is rejected without a panic" + ); +} + +fn run_lifecycle(format: &str) { + if !git_available() { + eprintln!("skipping bitmap lifecycle: git unavailable"); + return; + } + let dir = seed(format); + let path = dir.path(); + let repo = Repo::open(path).unwrap(); + let idx = find_pack(path, ".idx"); + assert!( + write_bitmap(&repo, &idx).unwrap(), + "a single-pack repo gets a bitmap" + ); + test_bitmap(path); + + let head = rev_parse(path, "HEAD"); + let ours = closure(&repo, &[head], &[]); + assert_eq!( + ours, + rev_list(path, &["HEAD"]), + "closure equals rev-list --objects HEAD" + ); + assert_eq!(ours, walked(&repo, head), "closure equals the walk closure"); + assert_eq!( + ours, + closure(&repo, &vec![head; 2048], &[]), + "duplicate wants fold to one closure" + ); + + let feature = rev_parse(path, "feature"); + let main = rev_parse(path, "main"); + assert_eq!( + closure(&repo, &[feature], &[main]), + rev_list(path, &["feature", "^main"]), + "feature minus main" + ); + + let tag = rev_parse(path, "v1"); + let mut whole = verbatim_clone_pack(&repo, Wants::new(&[main, feature, tag])) + .unwrap() + .expect("full clone reuses whole pack"); + let mut header = [0u8; 12]; + whole.read_exact(&mut header).unwrap(); + assert_eq!( + &header[..4], + b"PACK", + "verbatim reuse returns the on-disk pack" + ); + let count = u32::from_be_bytes([header[8], header[9], header[10], header[11]]) as usize; + assert_eq!( + count, + rev_list(path, &["--all"]).len(), + "verbatim reuse streams every object" + ); + assert!( + verbatim_clone_pack(&repo, Wants::new(&[main])) + .unwrap() + .is_none(), + "a partial closure never reuses the whole pack" + ); + + ok(path, &["tag", "-a", "v2", "-m", "release two", "HEAD"]); + let v2 = rev_parse(path, "v2"); + assert_ne!(v2, head); + assert!( + reachable_via_bitmap(&repo, Wants::new(&[v2]), Haves::new(&[])) + .unwrap() + .is_none(), + "a want outside the bitmapped pack fails closed" + ); + assert!( + walked(&repo, v2).contains(&v2), + "the fallback walk includes the wanted tag" + ); + + std::fs::remove_file(idx.with_extension("bitmap")).unwrap(); + commit(path, "e.txt", "epsilon\n", "post-bitmap growth"); + commit(path, "f.txt", "zeta\n", "more growth"); + ok(path, &["repack", "-dq"]); + ok(path, &["multi-pack-index", "write"]); + assert!(idx_count(path) >= 2, "the repo now spans multiple packs"); + + let repo = Repo::open(path).unwrap(); + assert!( + write_midx_bitmap(&repo).unwrap(), + "a multi-pack repo gets a midx bitmap" + ); + test_bitmap(path); + let head = rev_parse(path, "HEAD"); + assert_eq!( + closure(&repo, &[head], &[]), + rev_list(path, &["HEAD"]), + "midx closure equals rev-list HEAD" + ); + + reject_corruption(&repo, &find_pack(path, ".bitmap"), head); +} + +#[test] +fn bitmap_single_pack_then_midx_round_trips_against_canonical_git_sha1() { + run_lifecycle("sha1"); +} + +#[test] +fn bitmap_single_pack_then_midx_round_trips_against_canonical_git_sha256() { + run_lifecycle("sha256"); +} diff --git a/knot2/crates/knot-git/tests/fuzz_smoke.rs b/knot2/crates/knot-git/tests/fuzz_smoke.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-git/tests/fuzz_smoke.rs @@ -0,0 +1,10 @@ +use proptest::prelude::*; + +proptest! { + #![proptest_config(ProptestConfig::with_cases(1024))] + + #[test] + fn the_patch_parsers_never_panic(data in proptest::collection::vec(any::(), 0..4096)) { + knot_git::fuzz::patch(&data); + } +} diff --git a/knot2/crates/knot-git/tests/reads.rs b/knot2/crates/knot-git/tests/reads.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-git/tests/reads.rs @@ -0,0 +1,707 @@ +use std::path::Path; + +use knot_git::{CommitRange, EntryKind, FileChange, Layout, LineCount, LogLimit, LogSkip, Repo}; +use knot_types::{Listing, Oid, RefName, RepoDid, RepoPath}; + +fn rp(path: &str) -> RepoPath { + RepoPath::new(path).unwrap() +} + +mod common; +use common::{commit_file, git_ok as git}; + +#[test] +fn typed_reads_over_a_seeded_repo() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let bare = layout.create(&did).unwrap(); + let bare_path = layout.repo_path(&did).unwrap(); + let bare_str = bare_path.to_str().unwrap(); + + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + git(work, &["init", "-q", "-b", "main"]); + commit_file(work, "a.txt", "one\n", "first"); + git(work, &["push", "-q", bare_str, "main"]); + commit_file(work, "b.txt", "two\n", "second"); + std::fs::write(work.join("a.txt"), "one updated\n").unwrap(); + git(work, &["add", "-A"]); + git(work, &["commit", "-q", "-m", "third"]); + git(work, &["push", "-q", bare_str, "main"]); + git( + bare_path.as_path(), + &["symbolic-ref", "HEAD", "refs/heads/main"], + ); + + let head = Oid::from_hex(&git(work, &["rev-parse", "HEAD"])).unwrap(); + let parent = Oid::from_hex(&git(work, &["rev-parse", "HEAD~1"])).unwrap(); + let root = Oid::from_hex(&git(work, &["rev-parse", "HEAD~2"])).unwrap(); + let tree = Oid::from_hex(&git(work, &["rev-parse", "HEAD^{tree}"])).unwrap(); + let blob_b = Oid::from_hex(&git(work, &["rev-parse", "HEAD:b.txt"])).unwrap(); + + let head_ref = bare.head().expect("HEAD resolves"); + assert_eq!(head_ref.name.as_str(), "refs/heads/main"); + assert_eq!(head_ref.target, head); + + let commit = bare.find_commit(head).unwrap(); + assert_eq!(commit.id, head); + assert_eq!(commit.tree, tree); + assert_eq!(commit.parents, vec![parent]); + assert_eq!(commit.author.name.as_str(), "nel"); + assert!(commit.message.starts_with("third")); + + let tree_entries = bare.find_tree(tree).unwrap(); + let names: Vec<&str> = tree_entries + .entries + .iter() + .map(|entry| entry.name.as_str()) + .collect(); + assert!(names.contains(&"a.txt")); + assert!(names.contains(&"b.txt")); + assert!( + tree_entries + .entries + .iter() + .all(|entry| entry.kind == EntryKind::Blob) + ); + + assert_eq!(bare.read_blob(blob_b).unwrap(), b"two\n"); + + let changes = bare.diff(CommitRange { base: root, head }).unwrap(); + assert!(changes.iter().any( + |change| matches!(change, FileChange::Added { path, .. } if path.as_str() == "b.txt") + )); + assert!(changes.iter().any( + |change| matches!(change, FileChange::Modified { path, .. } if path.as_str() == "a.txt") + )); + + let comparison = bare.compare(CommitRange { base: root, head }).unwrap(); + assert_eq!(comparison.commits.len(), 2); + assert!(comparison.commits.contains(&head)); + assert!(comparison.commits.contains(&parent)); + assert_eq!(comparison.changes, changes); +} + +fn seed_main() -> (tempfile::TempDir, Layout, RepoDid, std::path::PathBuf, Oid) { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + layout.create(&did).unwrap(); + let bare_path = layout.repo_path(&did).unwrap(); + + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + git(work, &["init", "-q", "-b", "main"]); + commit_file(work, "a.txt", "one\n", "first"); + git(work, &["push", "-q", bare_path.to_str().unwrap(), "main"]); + let head = Oid::from_hex(&git(work, &["rev-parse", "HEAD"])).unwrap(); + (scan, layout, did, bare_path, head) +} + +fn seed_rich() -> (tempfile::TempDir, tempfile::TempDir, Layout, RepoDid) { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + layout.create(&did).unwrap(); + let bare_path = layout.repo_path(&did).unwrap(); + + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + git(work, &["init", "-q", "-b", "main"]); + commit_file(work, "a.txt", "one\ntwo\nthree\n", "first"); + std::fs::create_dir_all(work.join("src")).unwrap(); + commit_file(work, "src/lib.rs", "pub fn nel() {}\n", "add lib"); + commit_file(work, "a.txt", "one\ntwo\nthree\nfour\n", "extend a"); + git(work, &["tag", "light"]); + git(work, &["tag", "-a", "v1.0.0", "-m", "release one"]); + commit_file( + work, + "src/lib.rs", + "pub fn nel() {}\npub fn teq() {}\n", + "extend lib", + ); + git( + work, + &["push", "-q", "--tags", bare_path.to_str().unwrap(), "main"], + ); + git( + bare_path.as_path(), + &["symbolic-ref", "HEAD", "refs/heads/main"], + ); + (scan, work_dir, layout, did) +} + +#[test] +fn typed_reads_over_a_rich_repo() { + let (_scan, work_dir, layout, did) = seed_rich(); + let work = work_dir.path(); + let bare = layout.open(&did).unwrap(); + let head = Oid::from_hex(&git(work, &["rev-parse", "HEAD"])).unwrap(); + let parent = Oid::from_hex(&git(work, &["rev-parse", "HEAD~1"])).unwrap(); + let base = Oid::from_hex(&git(work, &["rev-parse", "HEAD~2"])).unwrap(); + let root = Oid::from_hex(&git(work, &["rev-parse", "HEAD~3"])).unwrap(); + + assert_eq!(bare.resolve_revision("main"), Some(head)); + assert_eq!(bare.resolve_revision("HEAD"), Some(head)); + assert_eq!(bare.resolve_revision(&head.to_hex()), Some(head)); + assert_eq!(bare.resolve_revision("does-not-exist"), None); + let tag_object = bare.resolve_revision("v1.0.0").unwrap(); + assert_eq!( + bare.peel_to_commit(tag_object).unwrap(), + Oid::from_hex(&git(work, &["rev-parse", "v1.0.0^{commit}"])).unwrap(), + "annotated tag peels to its commit" + ); + + let expected: Vec = git(work, &["rev-list", "HEAD"]) + .lines() + .map(str::to_string) + .collect(); + let (walked, total) = bare + .log_window(head, LogSkip::new(0), LogLimit::new(100)) + .unwrap(); + let walked: Vec = walked.iter().map(|commit| commit.id.to_hex()).collect(); + assert_eq!(walked, expected, "log order matches git rev-list"); + assert_eq!(total, expected.len()); + + let (page, total) = bare + .log_window(head, LogSkip::new(1), LogLimit::new(2)) + .unwrap(); + assert_eq!(page.len(), 2); + assert_eq!(page[0].id.to_hex(), expected[1]); + assert_eq!(page[1].id.to_hex(), expected[2]); + assert_eq!(total, expected.len(), "window still reports full count"); + + let between: Vec = bare + .commits_between(CommitRange { base, head }, LogLimit::new(100)) + .unwrap() + .iter() + .map(|oid| oid.to_hex()) + .collect(); + let expected_between: Vec = + git(work, &["rev-list", &format!("{}..HEAD", base.to_hex())]) + .lines() + .map(str::to_string) + .collect(); + assert_eq!(between, expected_between); + assert_eq!( + bare.commits_between(CommitRange { base, head }, LogLimit::new(1)) + .unwrap() + .len(), + 1, + "walk stops at limit instead of collecting full range" + ); + assert_eq!(bare.merge_base(head, base).unwrap(), Some(base)); + + let commit = bare.find_commit(head).unwrap(); + assert_eq!(commit.author.name.as_str(), "nel"); + assert!(commit.pgp_signature.is_none()); + assert!(commit.extra_headers.is_empty()); + assert!(commit.change_id().is_none()); + + let branches = bare.branch_list().unwrap(); + assert_eq!(branches.len(), 1); + assert_eq!(branches[0].name.as_str(), "main"); + assert_eq!(branches[0].tip.id(), head); + assert!(matches!(branches[0].tip, knot_git::BranchTip::Commit(_))); + + let tags = bare.tag_list().unwrap(); + assert_eq!(tags.len(), 2); + let light = tags + .iter() + .find(|tag| tag.name.as_str() == "light") + .unwrap(); + assert!(light.annotated.is_none()); + assert!(light.message.starts_with("extend a")); + let annotated = tags + .iter() + .find(|tag| tag.name.as_str() == "v1.0.0") + .unwrap(); + let detail = annotated.annotated.as_ref().unwrap(); + assert_eq!(annotated.message, "release one\n"); + assert_eq!(detail.tagger.as_ref().unwrap().name.as_str(), "nel"); + assert_eq!( + detail.target, + Oid::from_hex(&git(work, &["rev-parse", "v1.0.0^{commit}"])).unwrap() + ); + assert_eq!( + annotated.id, + Oid::from_hex(&git(work, &["rev-parse", "v1.0.0"])).unwrap(), + "tag info id is tag object itself" + ); + + let tree_root = bare.tree_entries_at(head, None).unwrap().unwrap(); + let names: Vec<&str> = tree_root.iter().map(|entry| entry.name.as_str()).collect(); + assert_eq!(names, vec!["a.txt", "src"]); + let a = tree_root + .iter() + .find(|entry| entry.name == "a.txt") + .unwrap(); + assert_eq!(a.size, "one\ntwo\nthree\nfour\n".len() as u64); + assert_eq!(a.kind.mode_octal(), "0100644"); + let src = tree_root.iter().find(|entry| entry.name == "src").unwrap(); + assert_eq!(src.kind, EntryKind::Tree); + assert_eq!(src.size, 0); + + let sub = bare + .tree_entries_at(head, Some(&rp("src"))) + .unwrap() + .unwrap(); + assert_eq!(sub.len(), 1); + assert_eq!(sub[0].name, "lib.rs"); + assert_eq!( + bare.tree_entries_at(head, Some(&rp("a.txt"))) + .unwrap() + .unwrap(), + Vec::new(), + "file path lists as empty" + ); + assert!( + bare.tree_entries_at(head, Some(&rp("missing"))) + .unwrap() + .is_none() + ); + assert!(RepoPath::new("../escape").is_err()); + + let entry = bare.entry_at(head, &rp("src/lib.rs")).unwrap().unwrap(); + assert_eq!( + entry.oid, + Oid::from_hex(&git(work, &["rev-parse", "HEAD:src/lib.rs"])).unwrap() + ); + + let deadline = Some(std::time::Instant::now() + std::time::Duration::from_secs(10)); + let names: Vec = tree_root.iter().map(|entry| entry.name.clone()).collect(); + let attributed = bare.last_commits(head, None, &names, deadline).unwrap(); + assert_eq!( + attributed["a.txt"].id.to_hex(), + git(work, &["log", "-1", "--format=%H", "--", "a.txt"]) + ); + assert_eq!( + attributed["src"].id.to_hex(), + git(work, &["log", "-1", "--format=%H", "--", "src"]) + ); + assert_eq!(attributed["a.txt"].subject, "extend a"); + let nested = bare + .last_commits(head, Some(&rp("src")), &["lib.rs".to_string()], deadline) + .unwrap(); + assert_eq!( + nested["lib.rs"].id.to_hex(), + git(work, &["log", "-1", "--format=%H", "--", "src/lib.rs"]) + ); + + let patches = bare + .commit_patches(knot_git::PatchRange { + base: Some(parent), + head, + }) + .unwrap(); + assert_eq!(patches.len(), 1); + let patch = &patches[0]; + assert_eq!(patch.path.as_str(), "src/lib.rs"); + assert_eq!(patch.status, knot_git::PatchStatus::Modified); + assert!(!patch.is_binary); + assert_eq!(patch.hunks.len(), 1); + let hunk = &patch.hunks[0]; + assert_eq!( + ( + hunk.old_start.get(), + hunk.old_lines.get(), + hunk.new_start.get(), + hunk.new_lines.get() + ), + (1, 1, 1, 2) + ); + assert_eq!(hunk.added(), LineCount::new(1)); + assert_eq!(hunk.deleted(), LineCount::new(0)); + assert_eq!( + hunk.lines + .iter() + .map(|line| String::from_utf8_lossy(&line.text).into_owned()) + .collect::>(), + vec!["pub fn nel() {}\n", "pub fn teq() {}\n"] + ); + + let initial = bare + .commit_patches(knot_git::PatchRange { + base: None, + head: root, + }) + .unwrap(); + assert_eq!(initial.len(), 1); + assert_eq!(initial[0].status, knot_git::PatchStatus::Added); + assert_eq!( + initial[0].hunks[0].old_start.get(), + 0, + "added file hunk starts at -0,0" + ); + assert_eq!(initial[0].hunks[0].old_lines.get(), 0); + + let tag_commit = bare.peel_to_commit(tag_object).unwrap(); + assert_eq!( + bare.changed_paths(knot_git::PatchRange { + base: None, + head: tag_object, + }) + .unwrap(), + bare.changed_paths(knot_git::PatchRange { + base: None, + head: tag_commit, + }) + .unwrap(), + "an annotated tag peels to its commit before the trees are diffed" + ); + let created = bare + .changed_paths(knot_git::PatchRange { base: None, head }) + .unwrap(); + assert_eq!( + created.paths(), + [rp("a.txt"), rp("src/lib.rs")], + "a ref creation lists every blob in the tree and no directory of them" + ); + assert_eq!(created.listing(), Listing::Complete); +} + +type TopoRow = (fn(&Path, &Oid), fn(&Repo, &Oid)); + +#[test] +fn ref_topology_reads_are_total() { + let rows: &[TopoRow] = &[ + ( + |bare, _head| { + git(bare, &["update-ref", "-d", "refs/heads/main"]); + }, + |bare, _head| { + assert!(bare.head().is_none()); + assert_eq!(bare.default_branch().unwrap().as_str(), "refs/heads/main"); + assert!(bare.references().unwrap().is_empty()); + assert!(bare.branches().unwrap().is_empty()); + assert!(bare.tags().unwrap().is_empty()); + assert!(bare.advertised_refs().unwrap().is_empty()); + }, + ), + ( + |bare, head| { + git(bare, &["update-ref", "--no-deref", "HEAD", &head.to_hex()]); + }, + |bare, head| { + assert!(bare.head().is_none()); + assert!(bare.default_branch().is_none()); + let branches = bare.branches().unwrap(); + assert_eq!(branches.len(), 1); + assert_eq!(branches[0].target, *head); + assert_eq!( + bare.find_ref(&RefName::new("refs/heads/main").unwrap()) + .unwrap(), + Some(*head) + ); + }, + ), + ( + |bare, _head| { + git(bare, &["symbolic-ref", "HEAD", "refs/heads/nursery"]); + }, + |bare, _head| { + assert!(bare.head().is_none()); + assert_eq!( + bare.default_branch().unwrap().as_str(), + "refs/heads/nursery" + ); + let branches = bare.branches().unwrap(); + assert_eq!(branches.len(), 1); + assert_eq!(branches[0].name.as_str(), "refs/heads/main"); + }, + ), + ( + |bare, _head| { + git( + bare, + &["symbolic-ref", "refs/heads/mirror", "refs/heads/gone"], + ); + }, + |bare, head| { + let refs = bare.references().unwrap(); + assert!( + refs.iter() + .all(|record| record.name.as_str() != "refs/heads/mirror"), + "symref to missing target must be dropped, not panic or resolve" + ); + assert!( + refs.iter() + .any(|record| record.name.as_str() == "refs/heads/main" + && record.target == *head) + ); + assert_eq!( + bare.find_ref(&RefName::new("refs/heads/mirror").unwrap()) + .unwrap(), + None + ); + }, + ), + ( + |bare, _head| { + git(bare, &["pack-refs", "--all"]); + assert!(!bare.join("refs/heads/main").exists()); + assert!(bare.join("packed-refs").exists()); + }, + |bare, head| { + let refs = bare.references().unwrap(); + assert!( + refs.iter() + .any(|record| record.name.as_str() == "refs/heads/main" + && record.target == *head) + ); + assert_eq!( + bare.find_ref(&RefName::new("refs/heads/main").unwrap()) + .unwrap(), + Some(*head) + ); + }, + ), + ]; + + rows.iter().for_each(|(setup, check)| { + let (_scan, layout, did, bare_path, head) = seed_main(); + setup(bare_path.as_path(), &head); + let bare = layout.open(&did).unwrap(); + check(&bare, &head); + }); +} + +#[test] +fn reachable_from_public_excludes_cob_only_commits() { + let (_scan, work_dir, layout, did) = seed_rich(); + let work = work_dir.path(); + let bare_path = layout.repo_path(&did).unwrap(); + let bare = layout.open(&did).unwrap(); + + let head = Oid::from_hex(&git(work, &["rev-parse", "HEAD"])).unwrap(); + let ancestor = Oid::from_hex(&git(work, &["rev-parse", "HEAD~2"])).unwrap(); + let tag_commit = Oid::from_hex(&git(work, &["rev-parse", "v1.0.0^{commit}"])).unwrap(); + assert!(bare.reachable_from_public(head).unwrap()); + assert!(bare.reachable_from_public(ancestor).unwrap()); + assert!(bare.reachable_from_public(tag_commit).unwrap()); + + commit_file(work, "hidden.txt", "secret\n", "hidden"); + let hidden = Oid::from_hex(&git(work, &["rev-parse", "HEAD"])).unwrap(); + git( + work, + &[ + "push", + "-q", + bare_path.to_str().unwrap(), + "HEAD:refs/cobs/sh.tangled.repo.collaborator/secret", + ], + ); + + let bare = layout.open(&did).unwrap(); + assert!(bare.contains(hidden), "object lives in odb"); + assert!( + !bare.reachable_from_public(hidden).unwrap(), + "commit held only by cob ref isn't reachable from any public ref" + ); +} + +#[test] +fn a_branch_tipped_by_a_tag_object_lists_opaquely() { + let (_scan, work_dir, layout, did) = seed_rich(); + let work = work_dir.path(); + let bare_path = layout.repo_path(&did).unwrap(); + let tag_object = git(work, &["rev-parse", "v1.0.0"]); + std::fs::write( + bare_path.join("refs/heads/tagtip"), + format!("{tag_object}\n"), + ) + .unwrap(); + + let bare = layout.open(&did).unwrap(); + let branches = bare.branch_list().unwrap(); + assert_eq!(branches.len(), 2); + let tagtip = branches + .iter() + .find(|branch| branch.name.as_str() == "tagtip") + .unwrap(); + match &tagtip.tip { + knot_git::BranchTip::Opaque { + id, + message, + created_at, + } => { + assert_eq!(*id, Oid::from_hex(&tag_object).unwrap()); + assert_eq!(message, "release one\n"); + assert!(created_at.get() > 0, "annotated tag records tagger time"); + } + other => panic!("expected opaque tip, got {other:?}"), + } +} + +#[test] +fn extended_history_reads() { + let (_scan, work_dir, layout, did) = seed_rich(); + let work = work_dir.path(); + let bare_path = layout.repo_path(&did).unwrap(); + + let seed_head = git(work, &["rev-parse", "HEAD"]); + git( + work, + &[ + "update-index", + "--add", + "--cacheinfo", + &format!("160000,{seed_head},vendor/dep"), + ], + ); + git(work, &["commit", "-q", "-m", "add gitlink"]); + git(work, &["push", "-q", bare_path.to_str().unwrap(), "main"]); + let bare = layout.open(&did).unwrap(); + let linked = Oid::from_hex(&git(work, &["rev-parse", "HEAD"])).unwrap(); + assert!( + bare.tree_entries_at(linked, Some(&rp("vendor/dep"))) + .unwrap() + .is_none(), + "submodule path isn't found" + ); + assert!( + bare.tree_entries_at(linked, Some(&rp("vendor"))) + .unwrap() + .is_some(), + "directory holding gitlink still lists" + ); + + commit_file( + work, + ".gitmodules", + "# top comment\n[submodule \"kelp\"]\n\tpath = libs/kelp ; trailing comment\n\turl = \"https://oyster.cafe/kelp.git\"\n\tbranch = main\n[submodule \"whelk\"]\n\tpath = libs/whelk\n\turl = https://nel.pet/whelk.git # mirror\n", + "add submodules", + ); + git(work, &["push", "-q", bare_path.to_str().unwrap(), "main"]); + let bare = layout.open(&did).unwrap(); + let with_mods = Oid::from_hex(&git(work, &["rev-parse", "HEAD"])).unwrap(); + let submodules = bare.submodules(with_mods).unwrap(); + assert_eq!(submodules.len(), 2); + assert_eq!(submodules[0].name, "kelp"); + assert_eq!(submodules[0].path.as_str(), "libs/kelp"); + assert_eq!(submodules[0].url, "https://oyster.cafe/kelp.git"); + assert_eq!( + submodules[0].branch, + Some(knot_types::BranchName::new("main").unwrap()) + ); + assert_eq!(submodules[1].branch, None); + + std::fs::write(work.join("blob.bin"), [0u8, 159, 146, 150, 0, 1]).unwrap(); + std::fs::write(work.join("noeol.txt"), "no newline at end").unwrap(); + git(work, &["add", "-A"]); + git(work, &["commit", "-q", "-m", "binary and noeol"]); + git(work, &["push", "-q", bare_path.to_str().unwrap(), "main"]); + let bare = layout.open(&did).unwrap(); + let head = Oid::from_hex(&git(work, &["rev-parse", "HEAD"])).unwrap(); + let parent = Oid::from_hex(&git(work, &["rev-parse", "HEAD~1"])).unwrap(); + let patches = bare + .commit_patches(knot_git::PatchRange { + base: Some(parent), + head, + }) + .unwrap(); + let binary = patches + .iter() + .find(|patch| patch.path.as_str() == "blob.bin") + .unwrap(); + assert!(binary.is_binary); + assert!(binary.hunks.is_empty()); + let noeol = patches + .iter() + .find(|patch| patch.path.as_str() == "noeol.txt") + .unwrap(); + let last = noeol.hunks[0].lines.last().unwrap(); + assert_eq!(last.text, b"no newline at end".to_vec()); +} + +#[test] +fn archives_round_trip_through_tar() { + let (_scan, work_dir, layout, did) = seed_rich(); + let work = work_dir.path(); + let bare = layout.open(&did).unwrap(); + let head = Oid::from_hex(&git(work, &["rev-parse", "HEAD"])).unwrap(); + let tree = bare.peel_to_tree(head).unwrap(); + + let mut out = std::io::Cursor::new(Vec::new()); + bare.write_archive( + tree, + knot_git::ArchiveFormat::TarGz, + Some(&knot_git::ArchivePrefix::new("squid-main/").unwrap()), + &mut out, + ) + .unwrap(); + let compressed = out.into_inner(); + assert_eq!( + &compressed[..2], + &[0x1f, 0x8b], + "tar.gz starts with gzip magic" + ); + + let mut decoder = flate2::read::GzDecoder::new(compressed.as_slice()); + let mut tar = Vec::new(); + std::io::Read::read_to_end(&mut decoder, &mut tar).unwrap(); + let needle = b"squid-main/src/lib.rs"; + assert!( + tar.windows(needle.len()).any(|window| window == needle), + "tar contains prefixed entries" + ); +} + +#[test] +fn a_filename_with_a_backslash_is_addressable() { + let (_scan, layout, did, bare_path, _head) = seed_main(); + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + git( + work, + &["clone", "-q", bare_path.to_str().unwrap(), "checkout"], + ); + let clone = work.join("checkout"); + commit_file(&clone, "back\\slash.txt", "escaped\n", "backslash name"); + git(&clone, &["push", "-q", "origin", "main"]); + + let bare = layout.open(&did).unwrap(); + let head = Oid::from_hex(&git(&clone, &["rev-parse", "HEAD"])).unwrap(); + let entry = bare + .entry_at(head, &rp("back\\slash.txt")) + .unwrap() + .expect("backslash in filename is legal and addressable"); + assert_eq!(bare.read_blob(entry.oid).unwrap(), b"escaped\n"); +} + +#[test] +fn an_oversized_blob_diffs_as_binary_without_loading_it() { + let (_scan, layout, did, bare_path, _head) = seed_main(); + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + git( + work, + &["clone", "-q", bare_path.to_str().unwrap(), "checkout"], + ); + let clone = work.join("checkout"); + let oversized = vec![b'a'; (knot_git::MAX_DIFF_BLOB_BYTES + 1) as usize]; + std::fs::write(clone.join("huge.txt"), &oversized).unwrap(); + git(&clone, &["add", "-A"]); + git(&clone, &["commit", "-q", "-m", "huge text file"]); + git(&clone, &["push", "-q", "origin", "main"]); + + let bare = layout.open(&did).unwrap(); + let head = Oid::from_hex(&git(&clone, &["rev-parse", "HEAD"])).unwrap(); + let parent = Oid::from_hex(&git(&clone, &["rev-parse", "HEAD~1"])).unwrap(); + let patches = bare + .commit_patches(knot_git::PatchRange { + base: Some(parent), + head, + }) + .unwrap(); + let huge = patches + .iter() + .find(|patch| patch.path.as_str() == "huge.txt") + .unwrap(); + assert!( + huge.is_binary, + "blob past diff budget falls back to binary instead of being loaded" + ); + assert!(huge.hunks.is_empty()); +} diff --git a/knot2/crates/knot-index/src/coverage.rs b/knot2/crates/knot-index/src/coverage.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-index/src/coverage.rs @@ -0,0 +1,60 @@ +use std::sync::atomic::{AtomicU8, Ordering}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Coverage { + Warming, + Ready, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Resolved { + Warming, + Ready(T), +} + +impl Resolved { + pub fn is_warming(&self) -> bool { + matches!(self, Resolved::Warming) + } + + pub fn map(self, f: impl FnOnce(T) -> U) -> Resolved { + match self { + Resolved::Ready(value) => Resolved::Ready(f(value)), + Resolved::Warming => Resolved::Warming, + } + } +} + +const WARMING: u8 = 0; +const READY: u8 = 1; + +#[derive(Debug)] +pub(crate) struct CoverageCell(AtomicU8); + +impl CoverageCell { + pub(crate) fn new(initial: Coverage) -> Self { + Self(AtomicU8::new(encode(initial))) + } + + pub(crate) fn get(&self) -> Coverage { + decode(self.0.load(Ordering::Acquire)) + } + + pub(crate) fn set(&self, coverage: Coverage) { + self.0.store(encode(coverage), Ordering::Release); + } +} + +fn encode(coverage: Coverage) -> u8 { + match coverage { + Coverage::Warming => WARMING, + Coverage::Ready => READY, + } +} + +fn decode(raw: u8) -> Coverage { + match raw { + READY => Coverage::Ready, + _ => Coverage::Warming, + } +} diff --git a/knot2/crates/knot-index/src/error.rs b/knot2/crates/knot-index/src/error.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-index/src/error.rs @@ -0,0 +1,25 @@ +use knot_cob::{ChangeId, CobError}; +use knot_git::GitError; +use knot_types::TypeName; + +#[derive(Debug, thiserror::Error)] +pub enum IndexError { + #[error(transparent)] + Cob(#[from] CobError), + #[error(transparent)] + Git(#[from] GitError), + #[error("expected at most one {type_name} object, found {count}")] + Ambiguous { type_name: TypeName, count: usize }, + #[error("change {change} in {type_name} projection does not decode: {reason}")] + Decode { + change: ChangeId, + type_name: TypeName, + reason: String, + }, + #[error("change {change} is {found} change in {expected} projection")] + UnexpectedType { + change: ChangeId, + expected: TypeName, + found: TypeName, + }, +} diff --git a/knot2/crates/knot-index/src/intern.rs b/knot2/crates/knot-index/src/intern.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-index/src/intern.rs @@ -0,0 +1,46 @@ +use std::sync::Arc; + +use knot_types::{AccountDid, OwnerDid, RepoDid, RepoRkey}; +use lasso::{Spur, ThreadedRodeo}; + +#[derive(Debug, Clone, Default)] +pub(crate) struct Interner(Arc); + +impl Interner { + pub(crate) fn new() -> Self { + Self(Arc::new(ThreadedRodeo::new())) + } +} + +macro_rules! interned { + ($( + $key:ident of $value:ty { + $intern:ident, $get:ident, $resolve:ident, $label:literal + } + )+) => {$( + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] + pub(crate) struct $key(Spur); + + impl Interner { + pub(crate) fn $intern(&self, value: &$value) -> $key { + $key(self.0.get_or_intern(value.as_str())) + } + + pub(crate) fn $get(&self, value: &$value) -> Option<$key> { + self.0.get(value.as_str()).map($key) + } + + pub(crate) fn $resolve(&self, key: $key) -> $value { + <$value>::new(self.0.resolve(&key.0)) + .expect(concat!("interned ", $label, " is valid ", $label)) + } + } + )+}; +} + +interned! { + AccountKey of AccountDid { intern_account, account, resolve_account, "account DID" } + RepoKey of RepoDid { intern_repo, repo, resolve_repo, "repo DID" } + OwnerKey of OwnerDid { intern_owner, owner, resolve_owner, "owner DID" } + RkeyKey of RepoRkey { intern_rkey, rkey, resolve_rkey, "rkey" } +} diff --git a/knot2/crates/knot-index/src/lib.rs b/knot2/crates/knot-index/src/lib.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-index/src/lib.rs @@ -0,0 +1,231 @@ +mod coverage; +mod error; +mod intern; +mod projections; + +pub use coverage::{Coverage, Resolved}; +pub use error::IndexError; +pub use knot_types::OfferedKey; + +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; + +use knot_cob::{ChangePayload, CobStore}; +use knot_cobs::{ + BlocklistChange, BlocklistCob, CollaboratorsChange, CollaboratorsCob, Grant, MembersChange, + MembersCob, RegistryChange, RepoRegistryCob, +}; +use knot_git::{Layout, Repo}; +use knot_types::{AccountDid, OwnerDid, RepoDid, RepoRkey}; + +use intern::Interner; +use projections::{CollaboratorsProjection, GrantSetProjection, KeyProjection, RegistryProjection}; + +knot_types::scalar_newtype! { + pub struct IndexGeneration(u64); +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct IndexCoverage { + pub members: Coverage, + pub blocklist: Coverage, + pub collaborators: Coverage, + pub registry: Coverage, + pub keys: Coverage, +} + +pub struct Index { + meta_path: PathBuf, + layout: Layout, + interner: Interner, + members: GrantSetProjection, + blocklist: GrantSetProjection, + collaborators: CollaboratorsProjection, + registry: RegistryProjection, + keys: KeyProjection, + generation: AtomicU64, +} + +impl Index { + pub fn new(meta_path: impl Into, layout: Layout) -> Self { + Self { + meta_path: meta_path.into(), + layout, + interner: Interner::new(), + members: GrantSetProjection::new(), + blocklist: GrantSetProjection::new(), + collaborators: CollaboratorsProjection::new(), + registry: RegistryProjection::new(), + keys: KeyProjection::new(), + generation: AtomicU64::new(0), + } + } + + pub fn generation(&self) -> IndexGeneration { + IndexGeneration(self.generation.load(Ordering::Acquire)) + } + + fn bump_generation(&self) { + self.generation.fetch_add(1, Ordering::Release); + } + + pub fn rebuild(&self) -> Result<(), IndexError> { + self.refresh_members()?; + self.refresh_blocklist()?; + self.refresh_registry()?; + Ok(()) + } + + pub fn warm_collaborators(&self) { + self.hosted_repos().iter().for_each(|repo| { + let _ = self.ensure_collaborators(repo); + }); + } + + pub fn refresh_members(&self) -> Result<(), IndexError> { + let meta = Repo::open(&self.meta_path)?; + let store = CobStore::new(&meta); + match store.list::()?.as_slice() { + [] => self.members.reset(), + [object] => self.members.refresh(&self.interner, &store, *object)?, + many => { + return Err(IndexError::Ambiguous { + type_name: MembersChange::type_name(), + count: many.len(), + }); + } + } + self.bump_generation(); + Ok(()) + } + + pub fn refresh_blocklist(&self) -> Result<(), IndexError> { + let meta = Repo::open(&self.meta_path)?; + let store = CobStore::new(&meta); + match store.list::()?.as_slice() { + [] => self.blocklist.reset(), + [object] => self.blocklist.refresh(&self.interner, &store, *object)?, + many => { + return Err(IndexError::Ambiguous { + type_name: BlocklistChange::type_name(), + count: many.len(), + }); + } + } + self.bump_generation(); + Ok(()) + } + + pub fn refresh_registry(&self) -> Result<(), IndexError> { + let meta = Repo::open(&self.meta_path)?; + let store = CobStore::new(&meta); + let evacuated = match store.list::()?.as_slice() { + [] => self.registry.reset(&self.interner), + [object] => self.registry.refresh(&self.interner, &store, *object)?, + many => { + return Err(IndexError::Ambiguous { + type_name: RegistryChange::type_name(), + count: many.len(), + }); + } + }; + evacuated.iter().for_each(|repo| { + if let Some(key) = self.interner.repo(repo) { + self.collaborators.drop_repo(key); + } + }); + self.bump_generation(); + Ok(()) + } + + pub fn ensure_collaborators(&self, repo: &RepoDid) -> Result<(), IndexError> { + if self.collaborators.is_folded(&self.interner, repo) { + return Ok(()); + } + self.refresh_collaborators(repo) + } + + pub fn refresh_collaborators(&self, repo: &RepoDid) -> Result<(), IndexError> { + let git = self.layout.open(repo)?; + let store = CobStore::new(&git); + let repo_key = self.interner.intern_repo(repo); + match store.list::()?.as_slice() { + [] => self.collaborators.mark_repo_empty(repo_key), + [object] => { + self.collaborators + .refresh_repo(&self.interner, &store, repo_key, *object)? + } + many => { + return Err(IndexError::Ambiguous { + type_name: CollaboratorsChange::type_name(), + count: many.len(), + }); + } + } + self.bump_generation(); + Ok(()) + } + + pub fn is_member(&self, did: &AccountDid) -> Resolved { + self.members.contains(&self.interner, did) + } + + pub fn member_entries(&self) -> Resolved> { + self.members.entries(&self.interner) + } + + pub fn is_blocked(&self, did: &AccountDid) -> Resolved { + self.blocklist.contains(&self.interner, did) + } + + pub fn blocked_entries(&self) -> Resolved> { + self.blocklist.entries(&self.interner) + } + + pub fn is_collaborator(&self, repo: &RepoDid, did: &AccountDid) -> Resolved { + self.collaborators.contains(&self.interner, repo, did) + } + + pub fn collaborator_entries(&self, repo: &RepoDid) -> Resolved> { + self.collaborators.entries(&self.interner, repo) + } + + pub fn collaborators_of(&self, repo: &RepoDid) -> Resolved> { + self.collaborator_entries(repo) + .map(|entries| entries.into_iter().map(|grant| grant.subject).collect()) + } + + pub fn resolve_repo(&self, owner: &OwnerDid, rkey: &RepoRkey) -> Resolved> { + self.registry.resolve(&self.interner, owner, rkey) + } + + pub fn owner_of(&self, repo: &RepoDid) -> Resolved> { + self.registry.owner_of(&self.interner, repo) + } + + pub fn rkey_of(&self, repo: &RepoDid) -> Resolved> { + self.registry.rkey_of(&self.interner, repo) + } + + pub fn hosted_repos(&self) -> Vec { + self.registry.hosted_repos(&self.interner) + } + + pub fn owner_of_key(&self, key: &OfferedKey) -> Resolved> { + self.keys.owner(&self.interner, key) + } + + pub fn cache_key(&self, key: OfferedKey, did: &AccountDid) { + self.keys.cache(&self.interner, key, did); + } + + pub fn coverage(&self) -> IndexCoverage { + IndexCoverage { + members: self.members.coverage(), + blocklist: self.blocklist.coverage(), + collaborators: self.collaborators.coverage(), + registry: self.registry.coverage(), + keys: self.keys.coverage(), + } + } +} diff --git a/knot2/crates/knot-index/src/projections.rs b/knot2/crates/knot-index/src/projections.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-index/src/projections.rs @@ -0,0 +1,745 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::hash::{Hash, Hasher}; +use std::marker::PhantomData; +use std::sync::Mutex; + +use knot_cache::{Cache, EntryCount, Lru}; +use knot_cob::{Change, ChangeId, ChangePayload, Checkpoint, CobId, CobStore, Evaluate}; +use knot_cobs::{ + CollaboratorsChange, CollaboratorsCob, Grant, GrantChange, Registration, Registry, + RegistryChange, Rename, RepoRef, RepoRegistryCob, Roster, +}; +use knot_types::{AccountDid, OfferedKey, OwnerDid, RepoDid, RepoRkey, UnixSeconds}; + +use crate::coverage::{Coverage, CoverageCell, Resolved}; +use crate::error::IndexError; +use crate::intern::{AccountKey, Interner, OwnerKey, RepoKey, RkeyKey}; + +const KEY_CACHE_CAPACITY: usize = 16_384; + +#[derive(Debug, Clone, Copy)] +struct Provenance { + added_by: AccountKey, + created_at: UnixSeconds, +} + +impl Provenance { + fn intern(interner: &Interner, grant: &Grant) -> Self { + Self { + added_by: interner.intern_account(&grant.added_by), + created_at: grant.created_at, + } + } + + fn grant(self, interner: &Interner, subject: AccountKey) -> Grant { + Grant { + subject: interner.resolve_account(subject), + added_by: interner.resolve_account(self.added_by), + created_at: self.created_at, + } + } +} + +fn decode_change(change: &Change) -> Result { + if change.type_name != P::type_name() { + return Err(IndexError::UnexpectedType { + change: change.id, + expected: P::type_name(), + found: change.type_name.clone(), + }); + } + P::decode(change.payload()).map_err(|error| IndexError::Decode { + change: change.id, + type_name: P::type_name(), + reason: error.to_string(), + }) +} + +fn decode_delta(changes: &[Change]) -> Result, IndexError> { + changes.iter().map(decode_change::

).collect() +} + +fn group_by(items: Vec, key: impl Fn(&T) -> K) -> BTreeMap> { + items.into_iter().fold(BTreeMap::new(), |mut acc, item| { + acc.entry(key(&item)).or_default().push(item); + acc + }) +} + +pub(crate) struct GrantSetProjection +where + Cob: Evaluate, + Cob::Change: ChangePayload + GrantChange, +{ + membership: scc::HashMap, + coverage: CoverageCell, + tip: Mutex>, + _cob: PhantomData Cob>, +} + +impl GrantSetProjection +where + Cob: Evaluate, + Cob::Change: ChangePayload + GrantChange, +{ + pub(crate) fn new() -> Self { + Self { + membership: scc::HashMap::new(), + coverage: CoverageCell::new(Coverage::Warming), + tip: Mutex::new(None), + _cob: PhantomData, + } + } + + pub(crate) fn coverage(&self) -> Coverage { + self.coverage.get() + } + + pub(crate) fn reset(&self) { + let mut tip = self + .tip + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + self.membership.clear_sync(); + *tip = None; + self.coverage.set(Coverage::Ready); + } + + pub(crate) fn contains(&self, interner: &Interner, did: &AccountDid) -> Resolved { + match self.coverage.get() { + Coverage::Warming => Resolved::Warming, + Coverage::Ready => Resolved::Ready( + interner + .account(did) + .is_some_and(|did| self.membership.contains_sync(&did)), + ), + } + } + + pub(crate) fn entries(&self, interner: &Interner) -> Resolved> { + match self.coverage.get() { + Coverage::Warming => Resolved::Warming, + Coverage::Ready => { + let mut out = BTreeMap::new(); + self.membership.iter_sync(|&subject, slot| { + let grant = slot.grant(interner, subject); + out.insert(grant.subject.clone(), grant); + true + }); + Resolved::Ready(out.into_values().collect()) + } + } + } + + pub(crate) fn refresh( + &self, + interner: &Interner, + store: &CobStore, + object: CobId, + ) -> Result<(), IndexError> + where + Cob: Checkpoint + Evaluate, + { + let mut tip = self + .tip + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + match *tip { + None => { + let (roster, seeded) = store + .materialize::(object) + .inspect_err(|_| self.coverage.set(Coverage::Warming))?; + self.seed(interner, &roster); + *tip = Some(seeded); + } + Some(prev) => { + let delta = store + .changes_since::(object, Some(prev)) + .inspect_err(|_| self.coverage.set(Coverage::Warming))?; + let decoded = decode_delta::(&delta.changes) + .inspect_err(|_| self.coverage.set(Coverage::Warming))?; + self.apply_delta(interner, decoded); + *tip = Some(delta.tip); + } + } + self.coverage.set(Coverage::Ready); + Ok(()) + } + + fn seed(&self, interner: &Interner, roster: &Roster) { + self.membership.clear_sync(); + roster.entries().for_each(|(subject, entry)| { + let slot = Provenance { + added_by: interner.intern_account(&entry.added_by), + created_at: entry.created_at, + }; + let _ = self + .membership + .insert_sync(interner.intern_account(subject), slot); + }); + } + + fn apply_delta(&self, interner: &Interner, changes: Vec) { + group_by(changes, |change| change.subject().clone()) + .into_iter() + .for_each(|(did, ops)| { + let current = interner + .account(&did) + .and_then(|key| self.membership.read_sync(&key, |_, slot| *slot)); + let net = ops + .into_iter() + .fold(current, |slot, change| match change.as_grant() { + Some(grant) => slot.or_else(|| Some(Provenance::intern(interner, grant))), + None => None, + }); + match net { + Some(slot) => { + *self + .membership + .entry_sync(interner.intern_account(&did)) + .or_insert(slot) + .get_mut() = slot; + } + None => { + if let Some(key) = interner.account(&did) { + let _ = self.membership.remove_sync(&key); + } + } + } + }); + } +} + +struct RepoRoster { + tip: Option, + entries: BTreeMap, +} + +const REPO_LOCK_STRIPES: usize = 256; + +pub(crate) struct CollaboratorsProjection { + rosters: scc::HashMap, + locks: [Mutex<()>; REPO_LOCK_STRIPES], +} + +impl CollaboratorsProjection { + pub(crate) fn new() -> Self { + Self { + rosters: scc::HashMap::new(), + locks: std::array::from_fn(|_| Mutex::new(())), + } + } + + fn repo_lock(&self, repo: RepoKey) -> &Mutex<()> { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + repo.hash(&mut hasher); + &self.locks[(hasher.finish() % REPO_LOCK_STRIPES as u64) as usize] + } + + pub(crate) fn coverage(&self) -> Coverage { + Coverage::Ready + } + + pub(crate) fn is_folded(&self, interner: &Interner, repo: &RepoDid) -> bool { + interner + .repo(repo) + .is_some_and(|repo| self.rosters.contains_sync(&repo)) + } + + pub(crate) fn contains( + &self, + interner: &Interner, + repo: &RepoDid, + did: &AccountDid, + ) -> Resolved { + let Some(repo) = interner.repo(repo) else { + return Resolved::Warming; + }; + match self.rosters.read_sync(&repo, |_, roster| { + interner + .account(did) + .is_some_and(|account| roster.entries.contains_key(&account)) + }) { + Some(present) => Resolved::Ready(present), + None => Resolved::Warming, + } + } + + pub(crate) fn entries(&self, interner: &Interner, repo: &RepoDid) -> Resolved> { + let Some(repo) = interner.repo(repo) else { + return Resolved::Warming; + }; + match self + .rosters + .read_sync(&repo, |_, roster| roster_grants(interner, roster)) + { + Some(grants) => Resolved::Ready(grants), + None => Resolved::Warming, + } + } + + pub(crate) fn mark_repo_empty(&self, repo: RepoKey) { + let lock = self.repo_lock(repo); + let _guard = lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + self.install( + repo, + RepoRoster { + tip: None, + entries: BTreeMap::new(), + }, + ); + } + + pub(crate) fn refresh_repo( + &self, + interner: &Interner, + store: &CobStore, + repo: RepoKey, + object: CobId, + ) -> Result<(), IndexError> { + let lock = self.repo_lock(repo); + let _guard = lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + self.fold_repo(interner, store, repo, object) + .inspect_err(|_| self.purge_repo(repo)) + } + + fn fold_repo( + &self, + interner: &Interner, + store: &CobStore, + repo: RepoKey, + object: CobId, + ) -> Result<(), IndexError> { + let prev = self + .rosters + .read_sync(&repo, |_, roster| roster.tip) + .flatten(); + match prev { + None => { + let (roster, tip) = store.materialize::(object)?; + let entries = roster + .entries() + .map(|(subject, entry)| { + ( + interner.intern_account(subject), + Provenance { + added_by: interner.intern_account(&entry.added_by), + created_at: entry.created_at, + }, + ) + }) + .collect(); + self.install( + repo, + RepoRoster { + tip: Some(tip), + entries, + }, + ); + } + Some(prev) => { + let delta = store.changes_since::(object, Some(prev))?; + let decoded = decode_delta::(&delta.changes)?; + let mut occupied = self.rosters.entry_sync(repo).or_insert_with(|| RepoRoster { + tip: None, + entries: BTreeMap::new(), + }); + let roster = occupied.get_mut(); + roster.entries = decoded + .into_iter() + .fold(std::mem::take(&mut roster.entries), |entries, change| { + apply(interner, entries, change) + }); + roster.tip = Some(delta.tip); + } + } + Ok(()) + } + + pub(crate) fn drop_repo(&self, repo: RepoKey) { + let lock = self.repo_lock(repo); + let _guard = lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + self.purge_repo(repo); + } + + fn purge_repo(&self, repo: RepoKey) { + let _ = self.rosters.remove_sync(&repo); + } + + fn install(&self, repo: RepoKey, roster: RepoRoster) { + match self.rosters.entry_sync(repo) { + scc::hash_map::Entry::Occupied(mut occupied) => { + let _ = occupied.insert(roster); + } + scc::hash_map::Entry::Vacant(vacant) => { + vacant.insert_entry(roster); + } + } + } +} + +fn roster_grants(interner: &Interner, roster: &RepoRoster) -> Vec { + roster + .entries + .iter() + .map(|(&subject, slot)| { + let grant = slot.grant(interner, subject); + (grant.subject.clone(), grant) + }) + .collect::>() + .into_values() + .collect() +} + +fn apply( + interner: &Interner, + mut entries: BTreeMap, + change: CollaboratorsChange, +) -> BTreeMap { + match change { + CollaboratorsChange::Add(grant) => { + entries + .entry(interner.intern_account(&grant.subject)) + .or_insert_with(|| Provenance::intern(interner, &grant)); + } + CollaboratorsChange::Remove(removal) => { + if let Some(key) = interner.account(&removal.subject) { + entries.remove(&key); + } + } + } + entries +} + +struct RecordSlot { + owner: OwnerKey, + rkey: RkeyKey, +} + +pub(crate) struct RegistryProjection { + aliases: scc::HashMap<(OwnerKey, RkeyKey), RepoKey>, + records: scc::HashMap, + coverage: CoverageCell, + tip: Mutex>, +} + +impl RegistryProjection { + pub(crate) fn new() -> Self { + Self { + aliases: scc::HashMap::new(), + records: scc::HashMap::new(), + coverage: CoverageCell::new(Coverage::Warming), + tip: Mutex::new(None), + } + } + + pub(crate) fn coverage(&self) -> Coverage { + self.coverage.get() + } + + pub(crate) fn reset(&self, interner: &Interner) -> Vec { + let mut tip = self + .tip + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let evacuated = self.hosted_repos(interner); + self.aliases.clear_sync(); + self.records.clear_sync(); + *tip = None; + self.coverage.set(Coverage::Ready); + evacuated + } + + pub(crate) fn resolve( + &self, + interner: &Interner, + owner: &OwnerDid, + rkey: &RepoRkey, + ) -> Resolved> { + match self.coverage.get() { + Coverage::Warming => Resolved::Warming, + Coverage::Ready => Resolved::Ready( + interner + .owner(owner) + .zip(interner.rkey(rkey)) + .and_then(|key| self.aliases.read_sync(&key, |_, repo| *repo)) + .map(|repo| interner.resolve_repo(repo)), + ), + } + } + + pub(crate) fn owner_of( + &self, + interner: &Interner, + repo: &RepoDid, + ) -> Resolved> { + if self.coverage.get() == Coverage::Warming { + return Resolved::Warming; + } + let Some(target) = interner.repo(repo) else { + return Resolved::Ready(None); + }; + Resolved::Ready( + self.records + .read_sync(&target, |_, slot| slot.owner) + .map(|owner| interner.resolve_owner(owner)), + ) + } + + pub(crate) fn rkey_of( + &self, + interner: &Interner, + repo: &RepoDid, + ) -> Resolved> { + if self.coverage.get() == Coverage::Warming { + return Resolved::Warming; + } + let Some(target) = interner.repo(repo) else { + return Resolved::Ready(None); + }; + Resolved::Ready( + self.records + .read_sync(&target, |_, slot| slot.rkey) + .map(|rkey| interner.resolve_rkey(rkey)), + ) + } + + pub(crate) fn hosted_repos(&self, interner: &Interner) -> Vec { + let mut repos = BTreeSet::new(); + self.records.iter_sync(|repo, _| { + repos.insert(interner.resolve_repo(*repo)); + true + }); + repos.into_iter().collect() + } + + pub(crate) fn refresh( + &self, + interner: &Interner, + store: &CobStore, + object: CobId, + ) -> Result, IndexError> { + let mut tip = self + .tip + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + match *tip { + None => { + let (registry, seeded) = store + .materialize::(object) + .inspect_err(|_| self.coverage.set(Coverage::Warming))?; + self.seed(interner, ®istry); + *tip = Some(seeded); + self.coverage.set(Coverage::Ready); + Ok(Vec::new()) + } + Some(prev) => { + let delta = store + .changes_since::(object, Some(prev)) + .inspect_err(|_| self.coverage.set(Coverage::Warming))?; + let decoded = decode_delta::(&delta.changes) + .inspect_err(|_| self.coverage.set(Coverage::Warming))?; + let displaced = self.apply_delta(interner, decoded); + *tip = Some(delta.tip); + self.coverage.set(Coverage::Ready); + Ok(self.evacuated(interner, displaced)) + } + } + } + + fn seed(&self, interner: &Interner, registry: &Registry) { + self.aliases.clear_sync(); + self.records.clear_sync(); + registry.records().for_each(|(repo, record)| { + self.upsert_record( + interner.intern_repo(repo), + interner.intern_owner(&record.owner), + interner.intern_rkey(&record.rkey), + ); + }); + registry.aliases().for_each(|(owner, rkey, repo)| { + self.upsert_alias( + interner.intern_owner(owner), + interner.intern_rkey(rkey), + interner.intern_repo(repo), + ); + }); + } + + fn evacuated(&self, interner: &Interner, displaced: Vec) -> Vec { + if displaced.is_empty() { + return Vec::new(); + } + let live = self.live_repos(); + displaced + .into_iter() + .collect::>() + .into_iter() + .filter(|repo| !live.contains(repo)) + .map(|repo| interner.resolve_repo(repo)) + .collect() + } + + fn live_repos(&self) -> BTreeSet { + let mut live = BTreeSet::new(); + self.records.iter_sync(|repo, _| { + live.insert(*repo); + true + }); + live + } + + fn apply_delta(&self, interner: &Interner, changes: Vec) -> Vec { + changes + .into_iter() + .fold(Vec::new(), |displaced, change| match change { + RegistryChange::Register(registration) => { + self.apply_register(interner, registration, displaced) + } + RegistryChange::Rename(rename) => self.apply_rename(interner, rename, displaced), + RegistryChange::Deregister(target) => { + self.apply_deregister(interner, target, displaced) + } + }) + } + + fn apply_register( + &self, + interner: &Interner, + registration: Registration, + mut displaced: Vec, + ) -> Vec { + let repo = interner.intern_repo(®istration.repo); + let owner = interner.intern_owner(®istration.owner); + let rkey = interner.intern_rkey(®istration.rkey); + if self.records.contains_sync(&repo) { + self.drop_record(repo); + displaced.push(repo); + } + displaced.extend(self.steal_alias(owner, rkey, repo)); + self.upsert_record(repo, owner, rkey); + self.upsert_alias(owner, rkey, repo); + displaced + } + + fn apply_rename( + &self, + interner: &Interner, + rename: Rename, + mut displaced: Vec, + ) -> Vec { + let repo = interner.intern_repo(&rename.repo); + let owner = interner.intern_owner(&rename.owner); + let rkey = interner.intern_rkey(&rename.rkey); + let held = self + .records + .read_sync(&repo, |_, slot| slot.owner == owner) + .unwrap_or(false); + if !held { + return displaced; + } + displaced.extend(self.steal_alias(owner, rkey, repo)); + self.upsert_record(repo, owner, rkey); + self.upsert_alias(owner, rkey, repo); + displaced + } + + fn apply_deregister( + &self, + interner: &Interner, + target: RepoRef, + mut displaced: Vec, + ) -> Vec { + let Some(owner) = interner.owner(&target.owner) else { + return displaced; + }; + let Some(rkey) = interner.rkey(&target.rkey) else { + return displaced; + }; + let Some(repo) = self.aliases.read_sync(&(owner, rkey), |_, repo| *repo) else { + return displaced; + }; + self.drop_record(repo); + displaced.push(repo); + displaced + } + + fn steal_alias(&self, owner: OwnerKey, rkey: RkeyKey, target: RepoKey) -> Option { + let holder = self.aliases.read_sync(&(owner, rkey), |_, repo| *repo)?; + if holder == target { + return None; + } + let canonical = self + .records + .read_sync(&holder, |_, slot| slot.rkey == rkey) + .unwrap_or(false); + if canonical { + self.drop_record(holder); + Some(holder) + } else { + let _ = self.aliases.remove_sync(&(owner, rkey)); + None + } + } + + fn drop_record(&self, repo: RepoKey) { + let _ = self.records.remove_sync(&repo); + self.aliases.retain_sync(|_, holder| *holder != repo); + } + + fn upsert_record(&self, repo: RepoKey, owner: OwnerKey, rkey: RkeyKey) { + if self + .records + .update_sync(&repo, |_, slot| { + slot.owner = owner; + slot.rkey = rkey; + }) + .is_none() + { + let _ = self.records.insert_sync(repo, RecordSlot { owner, rkey }); + } + } + + fn upsert_alias(&self, owner: OwnerKey, rkey: RkeyKey, repo: RepoKey) { + let key = (owner, rkey); + if self + .aliases + .update_sync(&key, |_, slot| *slot = repo) + .is_none() + { + let _ = self.aliases.insert_sync(key, repo); + } + } +} + +pub(crate) struct KeyProjection { + cache: Lru, +} + +impl KeyProjection { + pub(crate) fn new() -> Self { + Self { + cache: Lru::by_count(EntryCount::new(KEY_CACHE_CAPACITY as u64)), + } + } + + pub(crate) fn coverage(&self) -> Coverage { + Coverage::Ready + } + + pub(crate) fn cache(&self, interner: &Interner, key: OfferedKey, did: &AccountDid) { + self.cache.insert(key, interner.intern_account(did)); + } + + pub(crate) fn owner( + &self, + interner: &Interner, + key: &OfferedKey, + ) -> Resolved> { + Resolved::Ready( + self.cache + .get(key) + .map(|account| interner.resolve_account(account)), + ) + } +} diff --git a/knot2/crates/knot-index/tests/lifecycle.rs b/knot2/crates/knot-index/tests/lifecycle.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-index/tests/lifecycle.rs @@ -0,0 +1,357 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use knot_cob::{CobHome, CobStore}; +use knot_cobs::{CollaboratorsChange, CollaboratorsCob, Grant, MembersChange}; +use knot_git::Repo; +use knot_index::{Coverage, IndexCoverage, IndexError, OfferedKey, Resolved}; + +mod common; +use common::{World, acc, at, grant, meta_home, own, repo_did, rkey}; + +#[test] +fn rebuild_folds_members_and_registry_and_collaborators_fold_on_access() { + let world = World::new(); + let repo = repo_did("squid"); + world.seed_members(); + world.seed_registry(&repo); + world.seed_collaborator(&repo, "lyna"); + + let index = world.index(); + index.rebuild().unwrap(); + + assert_eq!(index.is_member(&acc("nel")), Resolved::Ready(true)); + assert_eq!(index.is_member(&acc("olaren")), Resolved::Ready(true)); + assert_eq!(index.is_member(&acc("teq")), Resolved::Ready(false)); + assert_eq!( + index.resolve_repo(&own("nel"), &rkey("anemone")), + Resolved::Ready(Some(repo.clone())) + ); + assert_eq!( + index.resolve_repo(&own("nel"), &rkey("nautilus")), + Resolved::Ready(None) + ); + assert_eq!( + index.is_collaborator(&repo, &acc("lyna")), + Resolved::Warming, + "rebuild doesn't fold collaborators, so roster reads warming until first access" + ); + + index.ensure_collaborators(&repo).unwrap(); + assert_eq!( + index.is_collaborator(&repo, &acc("lyna")), + Resolved::Ready(true) + ); + assert_eq!( + index.is_collaborator(&repo, &acc("bailey")), + Resolved::Ready(false) + ); + assert_eq!( + index.coverage(), + IndexCoverage { + members: Coverage::Ready, + blocklist: Coverage::Ready, + collaborators: Coverage::Ready, + registry: Coverage::Ready, + keys: Coverage::Ready, + } + ); +} + +#[test] +fn every_accessor_fails_closed_while_warming() { + let world = World::new(); + world.seed_members(); + world.seed_collaborator(&repo_did("squid"), "lyna"); + let index = world.index(); + + assert_eq!(index.is_member(&acc("nel")), Resolved::Warming); + assert_eq!(index.member_entries(), Resolved::Warming); + assert_eq!( + index.is_collaborator(&repo_did("squid"), &acc("lyna")), + Resolved::Warming + ); + assert_eq!( + index.collaborator_entries(&repo_did("squid")), + Resolved::Warming, + "roster that has never been folded fails closed" + ); + assert_eq!( + index.resolve_repo(&own("nel"), &rkey("anemone")), + Resolved::Warming + ); + assert_eq!( + index.owner_of(&repo_did("squid")), + Resolved::Warming, + "repo lookup before rebuild fails closed" + ); + assert_eq!(index.coverage().members, Coverage::Warming); + + assert_eq!( + index.owner_of_key(&OfferedKey::from_bytes(vec![1, 2, 3])), + Resolved::Ready(None), + "key cache is operational from boot, never warming" + ); +} + +#[test] +fn member_entries_record_provenance() { + let world = World::new(); + world.seed_members(); + let index = world.index(); + index.rebuild().unwrap(); + assert_eq!( + index.member_entries(), + Resolved::Ready(vec![grant("nel", "nel", 1), grant("olaren", "nel", 2)]) + ); +} + +#[test] +fn a_re_added_member_keeps_the_first_provenance() { + let world = World::new(); + let members = world.seed_members(); + let index = world.index(); + index.rebuild().unwrap(); + + let meta = Repo::open(&world.meta_path).unwrap(); + let store = CobStore::new(&meta); + store + .update( + &meta_home(), + members, + &MembersChange::Add(grant("olaren", "teq", 9)), + &world.signer, + at(9), + ) + .unwrap(); + index.refresh_members().unwrap(); + + assert_eq!( + index.member_entries(), + Resolved::Ready(vec![grant("nel", "nel", 1), grant("olaren", "nel", 2)]), + "duplicate add never rewrites original provenance, matching canonical roster" + ); +} + +#[test] +fn collaborator_entries_match_the_canonical_roster() { + let world = World::new(); + let repo = repo_did("squid"); + world.seed_registry(&repo); + let object = world.seed_collaborator(&repo, "lyna"); + let git = world.layout.open(&repo).unwrap(); + let store = CobStore::new(&git); + store + .update( + &CobHome::from(&repo), + object, + &CollaboratorsChange::Add(grant("bailey", "olaren", 2)), + &world.signer, + at(2), + ) + .unwrap(); + world.remove_collaborator(&repo, object, "lyna", 3); + store + .update( + &CobHome::from(&repo), + object, + &CollaboratorsChange::Add(grant("lyna", "teq", 5)), + &world.signer, + at(5), + ) + .unwrap(); + + let index = world.index(); + index.rebuild().unwrap(); + index.ensure_collaborators(&repo).unwrap(); + + let canonical = store.get::(object).unwrap(); + let expected: Vec = canonical + .state() + .entries() + .map(|(subject, entry)| Grant { + subject: subject.clone(), + added_by: entry.added_by.clone(), + created_at: entry.created_at, + }) + .collect(); + assert_eq!( + index.collaborator_entries(&repo), + Resolved::Ready(expected), + "projected entries disagree with canonical Evaluate fold" + ); +} + +#[test] +fn a_folded_repo_serves_while_unaccessed_repos_stay_warming() { + let world = World::new(); + let present = repo_did("squid"); + let absent = repo_did("kelp"); + let registry = world.seed_registry(&present); + world.register_extra(&absent, "barnacle", registry); + world.seed_collaborator(&present, "lyna"); + + let index = world.index(); + index.rebuild().unwrap(); + assert_eq!( + index.coverage().collaborators, + Coverage::Ready, + "collaborators projection is operational from boot" + ); + index.ensure_collaborators(&present).unwrap(); + + assert_eq!( + index.is_collaborator(&present, &acc("lyna")), + Resolved::Ready(true) + ); + assert_eq!( + index + .collaborator_entries(&present) + .map(|grants| grants.len()), + Resolved::Ready(1), + "folded repo serves its roster" + ); + assert_eq!( + index.collaborator_entries(&absent), + Resolved::Warming, + "registered repo that was never accessed stays fail-closed until folded" + ); + assert_eq!( + index.is_collaborator(&repo_did("conch"), &acc("lyna")), + Resolved::Warming, + "repo the index never folded cannot answer, so it fails closed" + ); +} + +#[test] +fn an_ambiguous_meta_cob_fails_refresh_and_rebuild() { + let world = World::new(); + let meta = Repo::open(&world.meta_path).unwrap(); + let store = CobStore::new(&meta); + store + .create( + &meta_home(), + &MembersChange::Add(grant("nel", "nel", 1)), + &world.signer, + at(1), + ) + .unwrap(); + store + .create( + &meta_home(), + &MembersChange::Add(grant("olaren", "olaren", 2)), + &world.signer, + at(2), + ) + .unwrap(); + + let index = world.index(); + assert!(matches!( + index.refresh_members(), + Err(IndexError::Ambiguous { count: 2, .. }) + )); + assert!( + matches!(index.rebuild(), Err(IndexError::Ambiguous { .. })), + "broken meta COB fails whole boot instead of reporting partial one" + ); +} + +#[test] +fn a_repo_missing_on_disk_does_not_fail_the_boot_and_isolates_its_fold() { + let world = World::new(); + let present = repo_did("squid"); + let absent = repo_did("kelp"); + world.seed_members(); + let registry = world.seed_registry(&present); + world.register_extra(&absent, "barnacle", registry); + world.seed_collaborator(&present, "lyna"); + + let index = world.index(); + index + .rebuild() + .expect("boot folds members and registry only, so missing repo dir never fails it"); + + index.ensure_collaborators(&present).unwrap(); + assert_eq!( + index.is_collaborator(&present, &acc("lyna")), + Resolved::Ready(true) + ); + assert!( + index.ensure_collaborators(&absent).is_err(), + "folding repo with no dir on disk fails for that repo alone" + ); + assert_eq!( + index.is_collaborator(&absent, &acc("lyna")), + Resolved::Warming, + "repo the index couldn't fold stays fail-closed" + ); +} + +#[test] +fn concurrent_refreshes_of_distinct_repos_all_land() { + let world = World::new(); + let repos = ["squid", "clam", "whelk", "conch"]; + repos.iter().for_each(|repo| { + world.seed_collaborator(&repo_did(repo), "lyna"); + }); + + let index = Arc::new(world.index()); + std::thread::scope(|scope| { + repos.iter().for_each(|repo| { + let index = Arc::clone(&index); + let repo = repo_did(repo); + scope.spawn(move || index.refresh_collaborators(&repo).unwrap()); + }); + }); + + repos.iter().for_each(|repo| { + assert_eq!( + index.is_collaborator(&repo_did(repo), &acc("lyna")), + Resolved::Ready(true) + ); + }); +} + +#[test] +fn a_refresh_is_eventually_consistent_not_an_atomic_snapshot() { + let world = World::new(); + let members = world.seed_members(); + let index = Arc::new(world.index()); + index.rebuild().unwrap(); + + (0..32).for_each(|i| world.add_member(members, &format!("m{i}"), 10 + i as i64)); + + let done = Arc::new(AtomicBool::new(false)); + std::thread::scope(|scope| { + let writer = Arc::clone(&index); + let writer_done = Arc::clone(&done); + scope.spawn(move || { + writer.refresh_members().unwrap(); + writer_done.store(true, Ordering::Release); + }); + + let reader = Arc::clone(&index); + let reader_done = Arc::clone(&done); + scope.spawn(move || { + while !reader_done.load(Ordering::Acquire) { + assert_eq!( + reader.is_member(&acc("nel")), + Resolved::Ready(true), + "stable member stays visible and read never blocks on writer" + ); + assert!( + !reader.is_member(&acc("m0")).is_warming(), + "already-ready projection serves reads mid-refresh, it never re-warms" + ); + } + }); + }); + + (0..32).for_each(|i| { + assert_eq!( + index.is_member(&acc(&format!("m{i}"))), + Resolved::Ready(true), + "once writer returns, whole delta has converged" + ); + }); +} diff --git a/knot2/crates/knot-index/tests/meta_repo.rs b/knot2/crates/knot-index/tests/meta_repo.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-index/tests/meta_repo.rs @@ -0,0 +1,98 @@ +use knot_cob::{CobHome, CobStore}; +use knot_cobs::{CollaboratorsChange, MembersChange, Registration, RegistryChange}; +use knot_git::Layout; +use knot_index::{Index, Resolved}; +use knot_runtime::{K256Signer, SeededEntropy}; +use knot_types::{KnotId, RepoDid, RepoName}; + +mod common; +use common::{acc, at, grant, own, rkey}; + +#[test] +fn the_meta_repo_round_trips_every_projection_from_git() { + let dir = tempfile::tempdir().unwrap(); + let layout = Layout::new(dir.path()); + let knot = KnotId::new("did:web:oyster.cafe").unwrap(); + let signer = K256Signer::generate(&SeededEntropy::new(1)); + + let meta = layout.bootstrap_meta(&knot).unwrap(); + let store = CobStore::new(&meta); + let members = store + .create( + &CobHome::from(&knot), + &MembersChange::Add(grant("nel", "nel", 1)), + &signer, + at(1), + ) + .unwrap(); + store + .update( + &CobHome::from(&knot), + members.object, + &MembersChange::Add(grant("olaren", "nel", 2)), + &signer, + at(2), + ) + .unwrap(); + + let repo = RepoDid::new("did:plc:squid").unwrap(); + store + .create( + &CobHome::from(&knot), + &RegistryChange::Register(Registration { + owner: own("nel"), + rkey: rkey("anemone"), + name: RepoName::new("anemone").unwrap(), + repo: repo.clone(), + created_at: at(1), + }), + &signer, + at(1), + ) + .unwrap(); + + let git = layout.create(&repo).unwrap(); + CobStore::new(&git) + .create( + &CobHome::from(&repo), + &CollaboratorsChange::Add(grant("lyna", "nel", 1)), + &signer, + at(1), + ) + .unwrap(); + + let meta_path = layout.meta_path(&knot).unwrap(); + let boot = || { + let index = Index::new(meta_path.clone(), layout.clone()); + index.rebuild().unwrap(); + index.warm_collaborators(); + index + }; + + let first = boot(); + assert_eq!(first.is_member(&acc("nel")), Resolved::Ready(true)); + assert_eq!(first.is_member(&acc("olaren")), Resolved::Ready(true)); + assert_eq!( + first.resolve_repo(&own("nel"), &rkey("anemone")), + Resolved::Ready(Some(repo.clone())) + ); + assert_eq!( + first.is_collaborator(&repo, &acc("lyna")), + Resolved::Ready(true) + ); + + let second = boot(); + ["nel", "olaren", "lyna", "stranger"] + .into_iter() + .for_each(|who| { + assert_eq!(first.is_member(&acc(who)), second.is_member(&acc(who))); + assert_eq!( + first.is_collaborator(&repo, &acc(who)), + second.is_collaborator(&repo, &acc(who)), + ); + }); + assert_eq!( + first.resolve_repo(&own("nel"), &rkey("anemone")), + second.resolve_repo(&own("nel"), &rkey("anemone")), + ); +} diff --git a/knot2/crates/knot-index/tests/projections.rs b/knot2/crates/knot-index/tests/projections.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-index/tests/projections.rs @@ -0,0 +1,567 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use knot_cob::{ChangePayload, CobHome, CobId, CobStore}; +use knot_cobs::{CollaboratorsChange, MembersChange, RegistryChange, Removal, Rename, RepoRef}; +use knot_git::{RefUpdate, Repo}; +use knot_index::{Coverage, IndexError, OfferedKey, Resolved}; +use knot_types::{RefName, RepoName}; +use serde::{Deserialize, Serialize}; + +mod common; +use common::{World, acc, at, grant, meta_home, own, registration, repo_did, rkey}; + +#[derive(Serialize, Deserialize)] +#[serde(tag = "op", content = "data", rename_all = "snake_case")] +enum BadMembers { + Explode(u8), +} +impl ChangePayload for BadMembers { + const TYPE: &'static str = "sh.tangled.knot.member"; +} + +#[test] +fn a_concurrent_reader_never_sees_a_net_absent_subject() { + let world = World::new(); + let object = world.seed_members(); + let index = Arc::new(world.index()); + index.rebuild().unwrap(); + assert_eq!(index.is_member(&acc("teq")), Resolved::Ready(false)); + + let meta = Repo::open(&world.meta_path).unwrap(); + let store = CobStore::new(&meta); + store + .update( + &meta_home(), + object, + &MembersChange::Add(grant("teq", "nel", 10)), + &world.signer, + at(10), + ) + .unwrap(); + (0..1000).for_each(|i| { + store + .update( + &meta_home(), + object, + &MembersChange::Add(grant(&format!("f{i}"), "nel", 11 + i)), + &world.signer, + at(11 + i), + ) + .unwrap(); + }); + store + .update( + &meta_home(), + object, + &MembersChange::Remove(Removal { + subject: acc("teq"), + }), + &world.signer, + at(20_000), + ) + .unwrap(); + + let done = Arc::new(AtomicBool::new(false)); + let leaked = Arc::new(AtomicBool::new(false)); + std::thread::scope(|scope| { + let reader = Arc::clone(&index); + let reader_done = Arc::clone(&done); + let reader_leaked = Arc::clone(&leaked); + scope.spawn(move || { + while !reader_done.load(Ordering::Acquire) { + if reader.is_member(&acc("teq")) == Resolved::Ready(true) { + reader_leaked.store(true, Ordering::Release); + } + } + }); + index.refresh_members().unwrap(); + done.store(true, Ordering::Release); + }); + + assert!( + !leaked.load(Ordering::Acquire), + "net-absent subject is never written, so no reader can observe it mid-delta" + ); + assert_eq!(index.is_member(&acc("teq")), Resolved::Ready(false)); + assert_eq!(index.is_member(&acc("f0")), Resolved::Ready(true)); + assert_eq!(index.is_member(&acc("f999")), Resolved::Ready(true)); +} + +#[test] +fn a_concurrent_reader_never_sees_a_collaborator_roster_emptied_mid_refresh() { + let world = World::new(); + let repo = repo_did("squid"); + let git = world.layout.create(&repo).unwrap(); + let store = CobStore::new(&git); + let object = store + .create( + &CobHome::from(&repo), + &CollaboratorsChange::Add(grant("anchor", "nel", 1)), + &world.signer, + at(1), + ) + .unwrap() + .object; + (0..128).for_each(|i| { + store + .update( + &CobHome::from(&repo), + object, + &CollaboratorsChange::Add(grant(&format!("c{i}"), "nel", 2 + i)), + &world.signer, + at(2 + i), + ) + .unwrap(); + }); + + let index = Arc::new(world.index()); + index.rebuild().unwrap(); + index.ensure_collaborators(&repo).unwrap(); + assert_eq!( + index.is_collaborator(&repo, &acc("anchor")), + Resolved::Ready(true) + ); + + let done = Arc::new(AtomicBool::new(false)); + let leaked = Arc::new(AtomicBool::new(false)); + std::thread::scope(|scope| { + let reader = Arc::clone(&index); + let reader_done = Arc::clone(&done); + let reader_leaked = Arc::clone(&leaked); + let target = repo.clone(); + scope.spawn(move || { + while !reader_done.load(Ordering::Acquire) { + if reader.is_collaborator(&target, &acc("anchor")) != Resolved::Ready(true) { + reader_leaked.store(true, Ordering::Release); + } + } + }); + (0..500).for_each(|_| index.refresh_collaborators(&repo).unwrap()); + done.store(true, Ordering::Release); + }); + + assert!( + !leaked.load(Ordering::Acquire), + "in-place mem::take runs under per-repo lock, so anchor present in both \ + pre- and post-refresh roster is never observed absent or warming mid-refresh" + ); + assert_eq!( + index.is_collaborator(&repo, &acc("anchor")), + Resolved::Ready(true) + ); +} + +fn cob_ref(type_name: &str, object: CobId) -> RefName { + RefName::new(format!("refs/cobs/{type_name}/{}", object.oid())).unwrap() +} + +#[test] +fn a_diverged_collaborators_tip_purges_the_roster_instead_of_serving_it_stale() { + let world = World::new(); + let repo = repo_did("squid"); + let git = world.layout.create(&repo).unwrap(); + let store = CobStore::new(&git); + let created = store + .create( + &CobHome::from(&repo), + &CollaboratorsChange::Add(grant("lyna", "nel", 1)), + &world.signer, + at(1), + ) + .unwrap(); + let tip = store + .update( + &CobHome::from(&repo), + created.object, + &CollaboratorsChange::Add(grant("bailey", "nel", 2)), + &world.signer, + at(2), + ) + .unwrap(); + + let index = world.index(); + index.rebuild().unwrap(); + index.ensure_collaborators(&repo).unwrap(); + assert_eq!( + index.is_collaborator(&repo, &acc("lyna")), + Resolved::Ready(true) + ); + + git.update_ref(&RefUpdate::Update { + name: cob_ref(CollaboratorsChange::TYPE, created.object), + old: tip.oid(), + new: created.object.oid(), + }) + .unwrap(); + + assert!( + index.refresh_collaborators(&repo).is_err(), + "tip that no longer descends from folded tip is structural error" + ); + assert_eq!( + index.is_collaborator(&repo, &acc("lyna")), + Resolved::Warming, + "diverged COB tip purges roster and fails closed, it does not serve \ + pre-divergence collaborators" + ); +} + +#[test] +fn a_diverged_members_tip_fails_closed_to_warming() { + let world = World::new(); + let object = world.seed_members(); + let meta = Repo::open(&world.meta_path).unwrap(); + let store = CobStore::new(&meta); + let tip = store + .update( + &meta_home(), + object, + &MembersChange::Add(grant("teq", "nel", 3)), + &world.signer, + at(3), + ) + .unwrap(); + + let index = world.index(); + index.rebuild().unwrap(); + assert_eq!(index.is_member(&acc("nel")), Resolved::Ready(true)); + + meta.update_ref(&RefUpdate::Update { + name: cob_ref(MembersChange::TYPE, object), + old: tip.oid(), + new: object.oid(), + }) + .unwrap(); + + assert!( + index.refresh_members().is_err(), + "tip that no longer descends from folded tip is structural error" + ); + assert_eq!(index.coverage().members, Coverage::Warming); + assert_eq!( + index.is_member(&acc("nel")), + Resolved::Warming, + "diverged members COB fails projection closed instead of serving stale members" + ); +} + +#[test] +fn an_undecodable_change_fails_closed_with_no_partial_apply() { + let world = World::new(); + let object = world.seed_members(); + let index = world.index(); + index.rebuild().unwrap(); + + let meta = Repo::open(&world.meta_path).unwrap(); + let store = CobStore::new(&meta); + store + .update( + &meta_home(), + object, + &MembersChange::Add(grant("teq", "nel", 3)), + &world.signer, + at(3), + ) + .unwrap(); + store + .update( + &meta_home(), + object, + &BadMembers::Explode(0), + &world.signer, + at(4), + ) + .unwrap(); + store + .update( + &meta_home(), + object, + &MembersChange::Remove(Removal { + subject: acc("teq"), + }), + &world.signer, + at(5), + ) + .unwrap(); + + assert!(matches!( + index.refresh_members(), + Err(IndexError::Decode { .. }) + )); + + assert_eq!(index.coverage().members, Coverage::Warming); + assert_eq!( + index.is_member(&acc("teq")), + Resolved::Warming, + "no partial apply: teq from pre-error change was never committed" + ); + assert_eq!( + index.is_member(&acc("nel")), + Resolved::Warming, + "structurally broken COB fails whole projection closed" + ); + + assert!(matches!( + index.refresh_members(), + Err(IndexError::Decode { .. }) + )); + assert_eq!(index.coverage().members, Coverage::Warming); +} + +#[test] +fn deregister_purges_collaborators_fail_closed() { + let world = World::new(); + let repo = repo_did("clam"); + + let meta = Repo::open(&world.meta_path).unwrap(); + let store = CobStore::new(&meta); + let registry = store + .create( + &meta_home(), + &RegistryChange::Register(registration("nel", "anemone", &repo, 1)), + &world.signer, + at(1), + ) + .unwrap() + .object; + + let git = world.layout.create(&repo).unwrap(); + let cstore = CobStore::new(&git); + cstore + .create( + &CobHome::from(&repo), + &CollaboratorsChange::Add(grant("lyna", "nel", 1)), + &world.signer, + at(1), + ) + .unwrap(); + + let index = world.index(); + index.rebuild().unwrap(); + index.warm_collaborators(); + assert_eq!( + index.is_collaborator(&repo, &acc("lyna")), + Resolved::Ready(true) + ); + + store + .update( + &meta_home(), + registry, + &RegistryChange::Deregister(RepoRef { + owner: own("nel"), + rkey: rkey("anemone"), + }), + &world.signer, + at(2), + ) + .unwrap(); + index.refresh_registry().unwrap(); + + assert_eq!( + index.resolve_repo(&own("nel"), &rkey("anemone")), + Resolved::Ready(None) + ); + assert_eq!( + index.is_collaborator(&repo, &acc("lyna")), + Resolved::Warming, + "deregistered repo's collaborators are purged and fail closed, not served stale" + ); +} + +#[test] +fn a_renamed_repo_keeps_both_rkeys_and_its_collaborators() { + let world = World::new(); + let repo = repo_did("squid"); + + let meta = Repo::open(&world.meta_path).unwrap(); + let store = CobStore::new(&meta); + let registry = store + .create( + &meta_home(), + &RegistryChange::Register(registration("nel", "anemone", &repo, 1)), + &world.signer, + at(1), + ) + .unwrap() + .object; + + let git = world.layout.create(&repo).unwrap(); + CobStore::new(&git) + .create( + &CobHome::from(&repo), + &CollaboratorsChange::Add(grant("lyna", "nel", 1)), + &world.signer, + at(1), + ) + .unwrap(); + + let index = world.index(); + index.rebuild().unwrap(); + index.warm_collaborators(); + assert_eq!(index.rkey_of(&repo), Resolved::Ready(Some(rkey("anemone")))); + + store + .update( + &meta_home(), + registry, + &RegistryChange::Rename(Rename { + owner: own("nel"), + rkey: rkey("barnacle"), + name: RepoName::new("barnacle").unwrap(), + repo: repo.clone(), + }), + &world.signer, + at(2), + ) + .unwrap(); + index.refresh_registry().unwrap(); + + assert_eq!( + index.resolve_repo(&own("nel"), &rkey("anemone")), + Resolved::Ready(Some(repo.clone())), + "prior rkey keeps resolving as alias after rename is delta-applied" + ); + assert_eq!( + index.resolve_repo(&own("nel"), &rkey("barnacle")), + Resolved::Ready(Some(repo.clone())) + ); + assert_eq!( + index.rkey_of(&repo), + Resolved::Ready(Some(rkey("barnacle"))), + "new rkey is canonical" + ); + assert_eq!( + index.is_collaborator(&repo, &acc("lyna")), + Resolved::Ready(true), + "rename never evacuates repo, so its collaborators survive" + ); + + store + .update( + &meta_home(), + registry, + &RegistryChange::Deregister(RepoRef { + owner: own("nel"), + rkey: rkey("anemone"), + }), + &world.signer, + at(3), + ) + .unwrap(); + index.refresh_registry().unwrap(); + assert_eq!( + index.resolve_repo(&own("nel"), &rkey("barnacle")), + Resolved::Ready(None), + "deregistering through retained alias removes repo and every alias" + ); + assert_eq!( + index.is_collaborator(&repo, &acc("lyna")), + Resolved::Warming, + "deregistered repo's collaborators are evacuated" + ); +} + +#[test] +fn a_repo_moved_within_one_delta_is_not_evacuated() { + let world = World::new(); + let repo = repo_did("squid"); + + let meta = Repo::open(&world.meta_path).unwrap(); + let store = CobStore::new(&meta); + let registry = store + .create( + &meta_home(), + &RegistryChange::Register(registration("nel", "anemone", &repo, 1)), + &world.signer, + at(1), + ) + .unwrap() + .object; + + let git = world.layout.create(&repo).unwrap(); + CobStore::new(&git) + .create( + &CobHome::from(&repo), + &CollaboratorsChange::Add(grant("lyna", "nel", 1)), + &world.signer, + at(1), + ) + .unwrap(); + + let index = world.index(); + index.rebuild().unwrap(); + index.warm_collaborators(); + assert_eq!( + index.is_collaborator(&repo, &acc("lyna")), + Resolved::Ready(true) + ); + + store + .update( + &meta_home(), + registry, + &RegistryChange::Deregister(RepoRef { + owner: own("nel"), + rkey: rkey("anemone"), + }), + &world.signer, + at(2), + ) + .unwrap(); + store + .update( + &meta_home(), + registry, + &RegistryChange::Register(registration("nel", "barnacle", &repo, 3)), + &world.signer, + at(3), + ) + .unwrap(); + index.refresh_registry().unwrap(); + + assert_eq!( + index.resolve_repo(&own("nel"), &rkey("anemone")), + Resolved::Ready(None) + ); + assert_eq!( + index.resolve_repo(&own("nel"), &rkey("barnacle")), + Resolved::Ready(Some(repo.clone())) + ); + assert_eq!( + index.is_collaborator(&repo, &acc("lyna")), + Resolved::Ready(true), + "deregister and re-register within single delta leaves repo hosted, so its collaborators survive" + ); +} + +#[test] +fn key_cache_evicts_least_recently_used() { + const CAP: u32 = 16_384; + let world = World::new(); + let index = world.index(); + let key = |i: u32| OfferedKey::from_bytes(i.to_le_bytes().to_vec()); + + (0..CAP).for_each(|i| index.cache_key(key(i), &acc("nel"))); + assert_eq!( + index.owner_of_key(&key(0)), + Resolved::Ready(Some(acc("nel"))) + ); + index.cache_key(key(CAP), &acc("nel")); + + assert_eq!( + index.owner_of_key(&key(1)), + Resolved::Ready(None), + "least-recently-used key is evicted" + ); + assert_eq!( + index.owner_of_key(&key(0)), + Resolved::Ready(Some(acc("nel"))), + "recently-used key survives despite being inserted first" + ); + assert_eq!( + index.owner_of_key(&key(CAP)), + Resolved::Ready(Some(acc("nel"))) + ); +} diff --git a/knot2/crates/knot-index/tests/properties.rs b/knot2/crates/knot-index/tests/properties.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-index/tests/properties.rs @@ -0,0 +1,307 @@ +use knot_cob::{CobHome, CobStore}; +use knot_cobs::{ + CollaboratorsChange, CollaboratorsCob, Grant, MembersChange, MembersCob, Registration, + RegistryChange, Removal, Rename, RepoRef, RepoRegistryCob, +}; +use knot_git::Repo; +use knot_index::{Index, Resolved}; +use knot_types::{AccountDid, OwnerDid, RepoDid, RepoName, RepoRkey, UnixSeconds}; +use proptest::prelude::*; + +mod common; +use common::{World, meta_home}; + +fn acc(n: u8) -> AccountDid { + AccountDid::new(format!("did:plc:s{n}")).unwrap() +} + +fn owner(n: u8) -> OwnerDid { + match n { + 0 => OwnerDid::new("did:plc:nel").unwrap(), + _ => OwnerDid::new("did:plc:olaren").unwrap(), + } +} + +fn repo_rkey(n: u8) -> RepoRkey { + RepoRkey::new(format!("r{n}")).unwrap() +} + +fn repo_did(n: u8) -> RepoDid { + RepoDid::new(format!("did:plc:r{n}")).unwrap() +} + +fn grant(subject: u8, t: i64) -> Grant { + Grant { + subject: acc(subject), + added_by: AccountDid::new("did:plc:nel").unwrap(), + created_at: UnixSeconds::new(t), + } +} + +fn member_change(op: u8, subject: u8, t: i64) -> MembersChange { + match op { + 0 => MembersChange::Add(grant(subject, t)), + _ => MembersChange::Remove(Removal { + subject: acc(subject), + }), + } +} + +fn collaborator_change(op: u8, subject: u8, t: i64) -> CollaboratorsChange { + match op { + 0 => CollaboratorsChange::Add(grant(subject, t)), + _ => CollaboratorsChange::Remove(Removal { + subject: acc(subject), + }), + } +} + +fn registry_change(op: u8, who: u8, rkey: u8, repo: u8, t: i64) -> RegistryChange { + match op { + 0 => RegistryChange::Register(Registration { + owner: owner(who), + rkey: repo_rkey(rkey), + name: RepoName::new(format!("r{rkey}")).unwrap(), + repo: repo_did(repo), + created_at: UnixSeconds::new(t), + }), + 1 => RegistryChange::Rename(Rename { + owner: owner(who), + rkey: repo_rkey(rkey), + name: RepoName::new(format!("r{rkey}")).unwrap(), + repo: repo_did(repo), + }), + _ => RegistryChange::Deregister(RepoRef { + owner: owner(who), + rkey: repo_rkey(rkey), + }), + } +} + +proptest! { + #![proptest_config(ProptestConfig { cases: 40, ..ProptestConfig::default() })] + + #[test] + fn members_fold_equals_canonical_evaluate( + ops in prop::collection::vec((0u8..2, 0u8..4), 1..14) + ) { + let world = World::seeded(7); + let meta = Repo::open(&world.meta_path).unwrap(); + let store = CobStore::new(&meta); + + let incremental = Index::new(&world.meta_path, world.layout.clone()); + + let (op0, subject0) = ops[0]; + let object = store + .create(&meta_home(), &member_change(op0, subject0, 1), &world.signer, UnixSeconds::new(1)) + .unwrap() + .object; + incremental.refresh_members().unwrap(); + + ops.iter().enumerate().skip(1).for_each(|(index, (op, subject))| { + let t = index as i64 + 1; + store + .update(&meta_home(), object, &member_change(*op, *subject, t), &world.signer, UnixSeconds::new(t)) + .unwrap(); + incremental.refresh_members().unwrap(); + }); + + let full = Index::new(&world.meta_path, world.layout.clone()); + full.rebuild().unwrap(); + + let canonical = store.get::(object).unwrap(); + let roster = canonical.state(); + let expected: Vec> = (0u8..4) + .map(|subject| Resolved::Ready(roster.contains(&acc(subject)))) + .collect(); + prop_assert_eq!( + (0u8..4).map(|s| incremental.is_member(&acc(s))).collect::>(), + expected.clone() + ); + prop_assert_eq!( + (0u8..4).map(|s| full.is_member(&acc(s))).collect::>(), + expected + ); + } + + #[test] + fn collaborators_fold_equals_canonical_evaluate( + ops in prop::collection::vec((0u8..2, 0u8..4), 1..14) + ) { + let world = World::seeded(8); + let repo = repo_did(0); + let git = world.layout.create(&repo).unwrap(); + let store = CobStore::new(&git); + + let incremental = Index::new(&world.meta_path, world.layout.clone()); + + let (op0, subject0) = ops[0]; + let object = store + .create(&CobHome::from(&repo), &collaborator_change(op0, subject0, 1), &world.signer, UnixSeconds::new(1)) + .unwrap() + .object; + incremental.rebuild().unwrap(); + incremental.refresh_collaborators(&repo).unwrap(); + + ops.iter().enumerate().skip(1).for_each(|(index, (op, subject))| { + let t = index as i64 + 1; + store + .update(&CobHome::from(&repo), object, &collaborator_change(*op, *subject, t), &world.signer, UnixSeconds::new(t)) + .unwrap(); + incremental.refresh_collaborators(&repo).unwrap(); + }); + + let full = Index::new(&world.meta_path, world.layout.clone()); + full.rebuild().unwrap(); + full.refresh_collaborators(&repo).unwrap(); + + let canonical = store.get::(object).unwrap(); + let roster = canonical.state(); + let expected: Vec> = (0u8..4) + .map(|subject| Resolved::Ready(roster.contains(&acc(subject)))) + .collect(); + prop_assert_eq!( + (0u8..4).map(|s| incremental.is_collaborator(&repo, &acc(s))).collect::>(), + expected.clone() + ); + prop_assert_eq!( + (0u8..4).map(|s| full.is_collaborator(&repo, &acc(s))).collect::>(), + expected + ); + } + + #[test] + fn registry_fold_equals_canonical_evaluate( + ops in prop::collection::vec((0u8..3, 0u8..2, 0u8..4, 0u8..4), 1..14) + ) { + let world = World::seeded(9); + let meta = Repo::open(&world.meta_path).unwrap(); + let store = CobStore::new(&meta); + + let incremental = Index::new(&world.meta_path, world.layout.clone()); + + let (op0, who0, name0, repo0) = ops[0]; + let object = store + .create(&meta_home(), ®istry_change(op0, who0, name0, repo0, 1), &world.signer, UnixSeconds::new(1)) + .unwrap() + .object; + incremental.refresh_registry().unwrap(); + + ops.iter().enumerate().skip(1).for_each(|(index, (op, who, name, repo))| { + let t = index as i64 + 1; + store + .update(&meta_home(), object, ®istry_change(*op, *who, *name, *repo, t), &world.signer, UnixSeconds::new(t)) + .unwrap(); + incremental.refresh_registry().unwrap(); + }); + + let full = Index::new(&world.meta_path, world.layout.clone()); + full.rebuild().unwrap(); + + let canonical = store.get::(object).unwrap(); + let registry = canonical.state(); + let lookups: Vec<(u8, u8)> = (0u8..2) + .flat_map(|who| (0u8..4).map(move |rkey| (who, rkey))) + .collect(); + let expected: Vec>> = lookups + .iter() + .map(|(who, rkey)| { + Resolved::Ready(registry.resolve(&owner(*who), &repo_rkey(*rkey)).cloned()) + }) + .collect(); + prop_assert_eq!( + lookups + .iter() + .map(|(who, rkey)| incremental.resolve_repo(&owner(*who), &repo_rkey(*rkey))) + .collect::>(), + expected.clone() + ); + prop_assert_eq!( + lookups + .iter() + .map(|(who, rkey)| full.resolve_repo(&owner(*who), &repo_rkey(*rkey))) + .collect::>(), + expected + ); + let expected_records: Vec<_> = (0u8..4) + .map(|n| { + let record = registry.record_of(&repo_did(n)); + ( + Resolved::Ready(record.map(|record| record.owner.clone())), + Resolved::Ready(record.map(|record| record.rkey.clone())), + ) + }) + .collect(); + prop_assert_eq!( + (0u8..4) + .map(|n| (incremental.owner_of(&repo_did(n)), incremental.rkey_of(&repo_did(n)))) + .collect::>(), + expected_records.clone() + ); + prop_assert_eq!( + (0u8..4) + .map(|n| (full.owner_of(&repo_did(n)), full.rkey_of(&repo_did(n)))) + .collect::>(), + expected_records + ); + } + + #[test] + fn two_rebuilds_are_observably_identical( + members in prop::collection::vec((0u8..2, 0u8..6), 0..16), + registry in prop::collection::vec((0u8..3, 0u8..2, 0u8..4, 0u8..4), 0..10), + ) { + let world = World::seeded(10); + let meta = Repo::open(&world.meta_path).unwrap(); + let store = CobStore::new(&meta); + + if let Some(((op, subject), rest)) = members.split_first() { + let object = store + .create(&meta_home(), &member_change(*op, *subject, 1), &world.signer, UnixSeconds::new(1)) + .unwrap() + .object; + rest.iter().enumerate().for_each(|(index, (op, subject))| { + let t = index as i64 + 2; + store + .update(&meta_home(), object, &member_change(*op, *subject, t), &world.signer, UnixSeconds::new(t)) + .unwrap(); + }); + } + + if let Some(((op, who, name, repo), rest)) = registry.split_first() { + let object = store + .create(&meta_home(), ®istry_change(*op, *who, *name, *repo, 1), &world.signer, UnixSeconds::new(1)) + .unwrap() + .object; + rest.iter().enumerate().for_each(|(index, (op, who, name, repo))| { + let t = index as i64 + 2; + store + .update(&meta_home(), object, ®istry_change(*op, *who, *name, *repo, t), &world.signer, UnixSeconds::new(t)) + .unwrap(); + }); + } + + let first = Index::new(&world.meta_path, world.layout.clone()); + first.rebuild().unwrap(); + let second = Index::new(&world.meta_path, world.layout.clone()); + second.rebuild().unwrap(); + + prop_assert_eq!( + (0u8..6).map(|s| first.is_member(&acc(s))).collect::>(), + (0u8..6).map(|s| second.is_member(&acc(s))).collect::>() + ); + let lookups: Vec<(u8, u8)> = (0u8..2) + .flat_map(|who| (0u8..4).map(move |rkey| (who, rkey))) + .collect(); + prop_assert_eq!( + lookups + .iter() + .map(|(who, rkey)| first.resolve_repo(&owner(*who), &repo_rkey(*rkey))) + .collect::>(), + lookups + .iter() + .map(|(who, rkey)| second.resolve_repo(&owner(*who), &repo_rkey(*rkey))) + .collect::>() + ); + } +} diff --git a/knot2/crates/knot-langs/src/langs.rs b/knot2/crates/knot-langs/src/langs.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-langs/src/langs.rs @@ -0,0 +1,127 @@ +use std::collections::HashMap; +use std::ops::ControlFlow; +use std::time::Instant; + +use gengo_language::{Category, Language}; +use knot_git::{EntryKind, GitError, MAX_TREE_DEPTH, Repo, SizedEntry}; +use knot_types::{LanguageBytes, Oid}; + +use crate::linguist; + +const READ_LIMIT: usize = 16 * 1024; +const SIZE_LIMIT: u64 = 1024 * 1024; + +pub use knot_types::LanguageName; + +fn looks_binary(content: &[u8]) -> bool { + content.contains(&0) +} + +fn category_of(name: &'static str, fallback: Category) -> Category { + name.parse::() + .map(|language| language.category()) + .unwrap_or(fallback) +} + +pub fn analyze( + repo: &Repo, + commit: Oid, + deadline: Option, +) -> Result, GitError> { + let mut sizes: HashMap = HashMap::new(); + let root = repo.peel_to_tree(commit)?; + let _budget = walk(repo, root, "", 0, deadline, &mut sizes)?; + Ok(sizes) +} + +fn walk( + repo: &Repo, + tree: Oid, + dir: &str, + depth: usize, + deadline: Option, + sizes: &mut HashMap, +) -> Result, GitError> { + if depth > MAX_TREE_DEPTH || deadline.is_some_and(|deadline| Instant::now() >= deadline) { + return Ok(ControlFlow::Break(())); + } + let entries = repo.tree_entries(tree)?; + entries + .iter() + .try_fold(ControlFlow::Continue(()), |flow, entry| { + if flow.is_break() || deadline.is_some_and(|deadline| Instant::now() >= deadline) { + return Ok(ControlFlow::Break(())); + } + let path = match dir.is_empty() { + true => entry.name.clone(), + false => format!("{dir}/{}", entry.name), + }; + match entry.kind { + EntryKind::Tree => match linguist::is_vendor_dir(&path) { + true => Ok(ControlFlow::Continue(())), + false => walk(repo, entry.oid, &path, depth + 1, deadline, sizes), + }, + EntryKind::Blob | EntryKind::BlobExecutable => { + if !linguist::is_skipped_path(&path) { + tally(repo, entry, &path, sizes)?; + } + Ok(ControlFlow::Continue(())) + } + EntryKind::Link | EntryKind::Commit => Ok(ControlFlow::Continue(())), + } + }) +} + +fn tally( + repo: &Repo, + entry: &SizedEntry, + path: &str, + sizes: &mut HashMap, +) -> Result<(), GitError> { + let content = match entry.size <= SIZE_LIMIT { + true => { + let blob = repo.read_blob(entry.oid)?; + blob[..blob.len().min(READ_LIMIT)].to_vec() + } + false => Vec::new(), + }; + if looks_binary(&content) { + return Ok(()); + } + let Some(language) = Language::pick(path, &content, READ_LIMIT) else { + return Ok(()); + }; + let name = LanguageName::new(linguist::group(language.name())); + if !matches!( + category_of(name.as_str(), language.category()), + Category::Programming | Category::Markup + ) { + return Ok(()); + } + let slot = sizes.entry(name).or_default(); + *slot = slot.saturating_add_bytes(entry.size); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn language_name_round_trips_through_as_str() { + let rust = LanguageName::new("Rust"); + assert_eq!(rust.as_str(), "Rust"); + } + + #[test] + fn language_names_compare_and_hash_by_value() { + use std::collections::HashSet; + + assert_eq!(LanguageName::new("Go"), LanguageName::new("Go")); + assert_ne!(LanguageName::new("Go"), LanguageName::new("Zig")); + let set: HashSet = [LanguageName::new("Go"), LanguageName::new("Go")] + .into_iter() + .collect(); + assert_eq!(set.len(), 1); + } +} diff --git a/knot2/crates/knot-langs/src/lib.rs b/knot2/crates/knot-langs/src/lib.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-langs/src/lib.rs @@ -0,0 +1,4 @@ +mod langs; +mod linguist; + +pub use langs::{LanguageName, analyze}; diff --git a/knot2/crates/knot-langs/src/linguist.rs b/knot2/crates/knot-langs/src/linguist.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-langs/src/linguist.rs @@ -0,0 +1,196 @@ +use std::sync::LazyLock; + +use regex::Regex; + +static VENDOR: LazyLock = LazyLock::new(|| { + Regex::new(r"(?:^(?:(?:[Dd]ependencies/)|(?:debian/)|(?:deps/)|(?:rebar$)))|(?:(?:^|/)(?:(?:BuddyBuildSDK\.framework/)|(?:Carthage/)|(?:Chart\.js$)|(?:Control\.FullScreen\.css)|(?:Control\.FullScreen\.js)|(?:Crashlytics\.framework/)|(?:Fabric\.framework/)|(?:Godeps/_workspace/)|(?:Jenkinsfile$)|(?:Leaflet\.Coordinates-\d+\.\d+\.\d+\.src\.js$)|(?:MathJax/)|(?:MochiKit\.js$)|(?:RealmSwift\.framework)|(?:Realm\.framework)|(?:Sparkle/)|(?:Vagrantfile$)|(?:[Bb]ourbon/.*\.(css|less|scss|styl)$)|(?:[Cc]ode[Mm]irror/(\d+\.\d+/)?(lib|mode|theme|addon|keymap|demo))|(?:[Ee]xtern(als?)?/)|(?:[Mm]icrosoft([Mm]vc)?([Aa]jax|[Vv]alidation)(\.debug)?\.js$)|(?:[Pp]ackages/.+\.\d+/)|(?:[Ss]pecs?/fixtures/)|(?:[Tt]ests?/fixtures/)|(?:[Vv]+endor/)|(?:\.[Dd][Ss]_[Ss]tore$)|(?:\.gitattributes$)|(?:\.github/)|(?:\.gitignore$)|(?:\.gitmodules$)|(?:\.gitpod\.Dockerfile$)|(?:\.google_apis/)|(?:\.indent\.pro)|(?:\.mvn/wrapper/)|(?:\.obsidian/)|(?:\.osx$)|(?:\.sublime-project)|(?:\.sublime-workspace)|(?:\.teamcity/)|(?:\.vscode/)|(?:\.yarn/plugins/)|(?:\.yarn/releases/)|(?:\.yarn/sdks/)|(?:\.yarn/unplugged/)|(?:\.yarn/versions/)|(?:_esy$)|(?:ace-builds/)|(?:aclocal\.m4)|(?:activator$)|(?:activator\.bat$)|(?:admin_media/)|(?:angular([^.]*)\.js$)|(?:animate\.(css|less|scss|styl)$)|(?:bootbox\.js)|(?:bootstrap([^/.]*)(\..*)?\.(js|css|less|scss|styl)$)|(?:bootstrap-datepicker/)|(?:bower_components/)|(?:bulma\.(css|sass|scss)$)|(?:cache/)|(?:ckeditor\.js$)|(?:config\.guess$)|(?:config\.sub$)|(?:configure$)|(?:controls\.js$)|(?:cordova([^.]*)\.js$)|(?:cordova\-\d\.\d(\.\d)?\.js$)|(?:cpplint\.py)|(?:custom\.bootstrap([^\s]*)(js|css|less|scss|styl)$)|(?:dist/)|(?:docs?/_?(build|themes?|templates?|static)/)|(?:dojo\.js$)|(?:dotnet-install\.(ps1|sh)$)|(?:dragdrop\.js$)|(?:effects\.js$)|(?:env/)|(?:erlang\.mk)|(?:extjs/.*?\.html$)|(?:extjs/.*?\.js$)|(?:extjs/.*?\.properties$)|(?:extjs/.*?\.txt$)|(?:extjs/.*?\.xml$)|(?:extjs/\.sencha/)|(?:extjs/builds/)|(?:extjs/cmd/)|(?:extjs/docs/)|(?:extjs/examples/)|(?:extjs/locale/)|(?:extjs/packages/)|(?:extjs/plugins/)|(?:extjs/resources/)|(?:extjs/src/)|(?:extjs/welcome/)|(?:fabfile\.py$)|(?:flow-typed/.*\.js$)|(?:font-?awesome/.*\.(css|less|scss|styl)$)|(?:font-?awesome\.(css|less|scss|styl)$)|(?:fontello(.*?)\.css$)|(?:foundation(\..*)?\.js$)|(?:foundation\.(css|less|scss|styl)$)|(?:fuelux\.js)|(?:gradle/wrapper/)|(?:gradlew$)|(?:gradlew\.bat$)|(?:html5shiv\.js$)|(?:inst/extdata/)|(?:jquery([^.]*)\.js$)|(?:jquery([^.]*)\.unobtrusive\-ajax\.js$)|(?:jquery([^.]*)\.validate(\.unobtrusive)?\.js$)|(?:jquery\-\d\.\d+(\.\d+)?\.js$)|(?:jquery\-ui(\-\d\.\d+(\.\d+)?)?(\.\w+)?\.(js|css)$)|(?:jquery\.(ui|effects)\.([^.]*)\.(js|css)$)|(?:jquery\.dataTables\.js)|(?:jquery\.fancybox\.(js|css))|(?:jquery\.fileupload(-\w+)?\.js$)|(?:jquery\.fn\.gantt\.js)|(?:knockout-(\d+\.){3}(debug\.)?js$)|(?:leaflet\.draw-src\.js)|(?:leaflet\.draw\.css)|(?:leaflet\.spin\.js)|(?:libtool\.m4)|(?:ltoptions\.m4)|(?:ltsugar\.m4)|(?:ltversion\.m4)|(?:lt~obsolete\.m4)|(?:materialize\.(css|less|scss|styl|js)$)|(?:modernizr\-\d\.\d+(\.\d+)?\.js$)|(?:modernizr\.custom\.\d+\.js$)|(?:mootools([^.]*)\d+\.\d+.\d+([^.]*)\.js$)|(?:mvnw$)|(?:mvnw\.cmd$)|(?:node_modules/)|(?:normalize\.(css|less|scss|styl)$)|(?:octicons\.css)|(?:pdf\.worker\.js)|(?:proguard-rules\.pro$)|(?:proguard\.pro$)|(?:prototype(.*)\.js$)|(?:puphpet/)|(?:react(-[^.]*)?\.js$)|(?:run\.n$)|(?:select2/.*\.(css|scss|js)$)|(?:shBrush([^.]*)\.js$)|(?:shCore\.js$)|(?:shLegacy\.js$)|(?:skeleton\.(css|less|scss|styl)$)|(?:slick\.\w+.js$)|(?:sprockets-octicons\.scss)|(?:testdata/)|(?:tiny_mce([^.]*)\.js$)|(?:tiny_mce/(langs|plugins|themes|utils))|(?:vendors?/)|(?:waf$)|(?:wicket-leaflet\.js)|(?:xvba_modules/)|(?:yahoo-([^.]*)\.js$)|(?:yui([^.]*)\.js$)))|(?:(.*?)\.d\.ts$)|(?:(3rd|[Tt]hird)[-_]?[Pp]arty/)|(?:([^\s]*)import\.(css|less|scss|styl)$)|(?:(\.|-)min\.(js|css)$)|(?:(^|/)d3(\.v\d+)?([^.]*)\.js$)|(?:-vsdoc\.js$)|(?:\.imageset/)|(?:\.intellisense\.js$)|(?:\.xctemplate/)").expect("enry vendor regex compiles") +}); + +static DOCUMENTATION: LazyLock = LazyLock::new(|| { + Regex::new(r"(?:^[Dd]ocs?/)|(?:(^|/)[Dd]ocumentation/)|(?:(^|/)[Gg]roovydoc/)|(?:(^|/)[Jj]avadoc/)|(?:^[Mm]an/)|(?:^[Ee]xamples/)|(?:^[Dd]emos?/)|(?:(^|/)inst/doc/)|(?:(^|/)CITATION(\.cff|(S)?(\.(bib|md))?)$)|(?:(^|/)CHANGE(S|LOG)?(\.|$))|(?:(^|/)CONTRIBUTING(\.|$))|(?:(^|/)COPYING(\.|$))|(?:(^|/)INSTALL(\.|$))|(?:(^|/)LICEN[CS]E(\.|$))|(?:(^|/)[Ll]icen[cs]e(\.|$))|(?:(^|/)README(\.|$))|(?:(^|/)[Rr]eadme(\.|$))|(?:^[Ss]amples?/)").expect("enry documentation regex compiles") +}); + +static GENERATED_NAME: LazyLock = LazyLock::new(|| { + Regex::new(r"(?:(?:^|/)\.idea/)|(?:(^Pods|/Pods)/)|(?:(^|/)Carthage/Build/)|(?:(?i)\.designer\.(cs|vb)$)|(?:(?i)\.feature\.cs$)|(?:vendor/([-0-9A-Za-z]+\.)+(com|edu|gov|in|me|net|org|fm|io))|(?:(^|/)(\w+\.)?esy.lock$)|(?:(^|/)\.pnp\..*$)|(?:.\.zep\.(?:c|h|php)$)|(?:(^|/)flake\.lock$)|(?:(^|/)MODULE\.bazel\.lock$)|(?:(?:^|/)\.terraform\.lock\.hcl$)|(?:(?i)_tlb\.pas$)|(?:(?:^|/)htmlcov/)|(?:(?:^|.*/)\.sqlx/query-.+\.json$)").expect("enry generated-name regex compiles") +}); + +const GENERATED_SUFFIXES: &[&str] = &[ + "Gopkg.lock", + "glide.lock", + "poetry.lock", + "pdm.lock", + "uv.lock", + "deno.lock", + "npm-shrinkwrap.json", + "package-lock.json", + "pnpm-lock.yaml", + "composer.lock", + "Cargo.lock", + "Cargo.toml.orig", + "Pipfile.lock", + "bun.lock", +]; + +const GENERATED_CONTAINS: &[&str] = &["node_modules/", "Godeps/", "__generated__/"]; + +const GENERATED_EXTENSIONS: &[&str] = &[".nib", ".xcworkspacedata", ".xcuserstate"]; + +fn is_dotfile(path: &str) -> bool { + path.rsplit('/') + .next() + .is_some_and(|base| base.starts_with('.') && base != ".") +} + +fn is_generated_name(path: &str) -> bool { + GENERATED_SUFFIXES + .iter() + .any(|suffix| path.ends_with(suffix)) + || GENERATED_CONTAINS + .iter() + .any(|needle| path.contains(needle)) + || GENERATED_EXTENSIONS + .iter() + .any(|extension| path.ends_with(extension)) + || GENERATED_NAME.is_match(path) +} + +pub(crate) fn is_skipped_path(path: &str) -> bool { + is_dotfile(path) + || VENDOR.is_match(path) + || DOCUMENTATION.is_match(path) + || is_generated_name(path) +} + +pub(crate) fn is_vendor_dir(path: &str) -> bool { + VENDOR.is_match(&format!("{path}/")) +} + +fn language_group(name: &str) -> Option<&'static str> { + match name { + "Alpine Abuild" => Some("Shell"), + "Apollo Guidance Computer" => Some("Assembly"), + "BibTeX" => Some("TeX"), + "Bison" => Some("Yacc"), + "Bluespec BH" => Some("Bluespec"), + "C2hs Haskell" => Some("Haskell"), + "Cairo" => Some("Cairo"), + "Cairo Zero" => Some("Cairo"), + "CameLIGO" => Some("LigoLANG"), + "ColdFusion CFC" => Some("ColdFusion"), + "Cylc" => Some("INI"), + "ECLiPSe" => Some("Prolog"), + "Easybuild" => Some("Python"), + "Ecere Projects" => Some("JavaScript"), + "Ecmarkup" => Some("HTML"), + "EditorConfig" => Some("INI"), + "Elvish Transcript" => Some("Elvish"), + "Filterscript" => Some("RenderScript"), + "Fortran" => Some("Fortran"), + "Fortran Free Form" => Some("Fortran"), + "Gentoo Ebuild" => Some("Shell"), + "Gentoo Eclass" => Some("Shell"), + "Git Config" => Some("INI"), + "Glimmer JS" => Some("JavaScript"), + "Glimmer TS" => Some("TypeScript"), + "Gradle Kotlin DSL" => Some("Gradle"), + "Groovy Server Pages" => Some("Groovy"), + "HTML+ECR" => Some("HTML"), + "HTML+EEX" => Some("HTML"), + "HTML+ERB" => Some("HTML"), + "HTML+PHP" => Some("HTML"), + "HTML+Razor" => Some("HTML"), + "Isabelle ROOT" => Some("Isabelle"), + "JFlex" => Some("Lex"), + "JSON with Comments" => Some("JSON"), + "Java Server Pages" => Some("Java"), + "Java Template Engine" => Some("Java"), + "JavaScript+ERB" => Some("JavaScript"), + "Jison" => Some("Yacc"), + "Jison Lex" => Some("Lex"), + "Julia REPL" => Some("Julia"), + "Lean 4" => Some("Lean"), + "LigoLANG" => Some("LigoLANG"), + "Literate Agda" => Some("Agda"), + "Literate CoffeeScript" => Some("CoffeeScript"), + "Literate Haskell" => Some("Haskell"), + "M4Sugar" => Some("M4"), + "MUF" => Some("Forth"), + "Maven POM" => Some("XML"), + "Motorola 68K Assembly" => Some("Assembly"), + "NPM Config" => Some("INI"), + "NumPy" => Some("Python"), + "OASv2-json" => Some("OpenAPI Specification v2"), + "OASv2-yaml" => Some("OpenAPI Specification v2"), + "OASv3-json" => Some("OpenAPI Specification v3"), + "OASv3-yaml" => Some("OpenAPI Specification v3"), + "OpenCL" => Some("C"), + "OpenRC runscript" => Some("Shell"), + "Parrot Assembly" => Some("Parrot"), + "Parrot Internal Representation" => Some("Parrot"), + "Pic" => Some("Roff"), + "PostCSS" => Some("CSS"), + "Python console" => Some("Python"), + "Python traceback" => Some("Python"), + "RBS" => Some("Ruby"), + "Readline Config" => Some("INI"), + "ReasonLIGO" => Some("LigoLANG"), + "Roff Manpage" => Some("Roff"), + "SSH Config" => Some("INI"), + "STON" => Some("Smalltalk"), + "Simple File Verification" => Some("Checksums"), + "Snakemake" => Some("Python"), + "TSX" => Some("TypeScript"), + "Tcsh" => Some("Shell"), + "Terraform Template" => Some("HCL"), + "Unified Parallel C" => Some("C"), + "Unix Assembly" => Some("Assembly"), + "Wget Config" => Some("INI"), + "X BitMap" => Some("C"), + "X PixMap" => Some("C"), + "XML Property List" => Some("XML"), + "cURL Config" => Some("INI"), + "fish" => Some("Shell"), + "nanorc" => Some("INI"), + _ => None, + } +} + +pub(crate) fn group(name: &'static str) -> &'static str { + language_group(name).unwrap_or(name) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn enry_matchers_compile_and_match_known_paths() { + assert!(is_skipped_path("node_modules/left-pad/index.js")); + assert!(is_skipped_path("src/jquery-3.6.0.js")); + assert!(is_skipped_path("docs/guide.md")); + assert!(is_skipped_path("README.md")); + assert!(is_skipped_path("LICENSE")); + assert!(is_skipped_path("Cargo.lock")); + assert!(is_skipped_path("package-lock.json")); + assert!(is_skipped_path("app/main.designer.cs")); + assert!(is_skipped_path(".github/workflows/ci.yml")); + assert!(is_skipped_path("third_party/zlib/zlib.c")); + assert!(is_skipped_path("web/app.min.js")); + assert!(!is_skipped_path("src/main.rs")); + assert!(!is_skipped_path("internal/server.go")); + } + + #[test] + fn vendor_dir_pruning_matches_directory_paths() { + assert!(is_vendor_dir("node_modules")); + assert!(is_vendor_dir("a/b/dist")); + assert!(!is_vendor_dir("src")); + } + + #[test] + fn grouping_folds_known_languages() { + assert_eq!(group("TSX"), "TypeScript"); + assert_eq!(group("HTML+ERB"), "HTML"); + assert_eq!(group("Rust"), "Rust"); + } +} diff --git a/knot2/crates/knot-lexicons/src/lib.rs b/knot2/crates/knot-lexicons/src/lib.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-lexicons/src/lib.rs @@ -0,0 +1,18 @@ +extern crate alloc; + +#[path = "_lex/lib.rs"] +#[allow(non_snake_case, unused_imports, unused_extern_crates)] +#[allow( + clippy::absurd_extreme_comparisons, + clippy::collapsible_if, + clippy::manual_strip, + clippy::needless_update, + clippy::new_ret_no_self, + clippy::new_without_default, + clippy::should_implement_trait, + clippy::type_complexity +)] +#[rustfmt::skip] +mod _lex; + +pub use _lex::*; diff --git a/knot2/crates/knot-lfs/fuzz/.gitignore b/knot2/crates/knot-lfs/fuzz/.gitignore new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-lfs/fuzz/.gitignore @@ -0,0 +1,4 @@ +target +corpus +artifacts +coverage diff --git a/knot2/crates/knot-lfs/fuzz/Cargo.lock b/knot2/crates/knot-lfs/fuzz/Cargo.lock new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-lfs/fuzz/Cargo.lock @@ -0,0 +1,5414 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + +[[package]] +name = "arc-swap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "async-compression" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "atomic-polyfill" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" +dependencies = [ + "critical-section", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "aws-lc-rs" +version = "1.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + +[[package]] +name = "base-x" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" + +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + +[[package]] +name = "base256emoji" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e9430d9a245a77c92176e649af6e275f20839a48389859d1661e9a128d077c" +dependencies = [ + "const-str", + "match-lookup", +] + +[[package]] +name = "base45" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240e56f4d3c453c36faacb695c535a4d5f8c7d23dac175014f32eb0a71012a03" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bon" +version = "3.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a602c73c7b0148ec6d12af6fd5cc7a46e2eacc8878271a999abac56eed12f561" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + +[[package]] +name = "borrow-or-share" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" + +[[package]] +name = "borsh" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3f6da4992df95bbcd9af42a6c7dcb994498fc9048230405f3b36ff7cd3f145" +dependencies = [ + "bytes", + "cfg_aliases", +] + +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "bstr" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cee35f73844aa3014bb606320a6c1f010249dbdf43342fe54b5a4f6a8ed4b79" +dependencies = [ + "memchr", + "regex-automata", + "serde_core", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +dependencies = [ + "serde", +] + +[[package]] +name = "bytesize" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7c8918969267b2932ffd5655509bbbea0833823058c378876953217f5fc50e" + +[[package]] +name = "cbor4ii" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544cf8c89359205f4f990d0e6f3828db42df85b5dac95d09157a250eb0749c4" +dependencies = [ + "serde", +] + +[[package]] +name = "cc" +version = "1.2.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "cid" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a304f95f84d169a6f31c4d0a30d784643aaa0bbc9c1e449a2c23e963ec4971" +dependencies = [ + "multibase", + "multihash", + "serde", + "serde_bytes", + "unsigned-varint 0.8.0", +] + +[[package]] +name = "clru" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "197fd99cb113a8d5d9b6376f3aa817f32c1078f2343b714fff7d2ca44fdf67d5" +dependencies = [ + "hashbrown 0.16.1", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.18", +] + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "brotli", + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "const-str" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f421161cb492475f1661ddc9815a745a1c894592070661180fdec3d4872e9c3" + +[[package]] +name = "cordyceps" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "688d7fbb8092b8de775ef2536f36c8c31f2bc4006ece2e8d8ad2d17d00ce0a2a" +dependencies = [ + "loom", + "tracing", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "crossbeam-channel" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "data-encoding-macro" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3259c913752a86488b501ed8680446a5ed2d5aeac6e596cb23ba3800768ea32c" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccc2776f0c61eca1ca32528f85548abd1a4be8fb53d1b21c013e4f18da1e7090" +dependencies = [ + "data-encoding", + "syn", +] + +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.18", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "derive_more" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "diatomic-waker" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab03c107fafeb3ee9f5925686dbb7a73bc76e3932abb0d2b365cb64b169cf04c" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.6", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest 0.10.7", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "rand_core 0.6.4", + "serde", + "sha2 0.10.9", + "subtle", + "zeroize", +] + +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "pem-rfc7468", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "faster-hex" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7223ae2d2f179b803433d9c830478527e92b8117eab39460edae7f1614d9fb73" +dependencies = [ + "heapless 0.8.0", + "serde", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "fluent-uri" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-buffered" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4421cb78ee172b6b06080093479d3c50f058e7c81b7d577bbb8d118d551d4cd5" +dependencies = [ + "cordyceps", + "diatomic-waker", + "futures-core", + "pin-project-lite", + "spin 0.10.0", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generator" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + +[[package]] +name = "generic-array" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +dependencies = [ + "typenum", + "version_check", + "zeroize", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", + "wasm-bindgen", +] + +[[package]] +name = "gix" +version = "0.84.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae54ae0ebd1a5a3c3f8d95dd3b5ca6e63f4fed9bfd585e13801a97d7bde8f9ce" +dependencies = [ + "gix-actor", + "gix-archive", + "gix-attributes", + "gix-blame", + "gix-command", + "gix-commitgraph", + "gix-config", + "gix-credentials", + "gix-date", + "gix-diff", + "gix-dir", + "gix-discover", + "gix-error", + "gix-features", + "gix-filter", + "gix-fs", + "gix-glob", + "gix-hash", + "gix-hashtable", + "gix-ignore", + "gix-index", + "gix-lock", + "gix-mailmap", + "gix-negotiate", + "gix-object", + "gix-odb", + "gix-pack", + "gix-path", + "gix-pathspec", + "gix-prompt", + "gix-protocol", + "gix-ref", + "gix-refspec", + "gix-revision", + "gix-revwalk", + "gix-sec", + "gix-shallow", + "gix-status", + "gix-submodule", + "gix-tempfile", + "gix-trace", + "gix-traverse", + "gix-url", + "gix-utils", + "gix-validate", + "gix-worktree", + "gix-worktree-state", + "gix-worktree-stream", + "nonempty", + "parking_lot", + "regex", + "signal-hook", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-actor" +version = "0.41.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bc998b8f746dda8565450d08a63b792ced9165d8c27a1ed3f02799ec6a7820f" +dependencies = [ + "bstr", + "gix-date", + "gix-error", +] + +[[package]] +name = "gix-archive" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16909cacc78936ab96f6c3be08379d0a2e88bfa3a7527972d2ed75c7517ef31e" +dependencies = [ + "bstr", + "flate2", + "gix-date", + "gix-error", + "gix-object", + "gix-path", + "gix-worktree-stream", + "rawzip", + "tar", +] + +[[package]] +name = "gix-attributes" +version = "0.33.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39b40888d0ed415c0744a6cdc61eebf0304c9d26ab726725b718443c322e5ba4" +dependencies = [ + "bstr", + "gix-glob", + "gix-path", + "gix-quote", + "gix-trace", + "kstring", + "smallvec", + "thiserror 2.0.18", + "unicode-bom", +] + +[[package]] +name = "gix-bitmap" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ebef0c26ad305747649e727bbcd56a7b7910754eb7cea88f6dff6f93c51283" +dependencies = [ + "gix-error", +] + +[[package]] +name = "gix-blame" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d39a0c14af94c2edaa5eefe06d5ef2cdea55316ae9a9321314288e3f55fa4c0" +dependencies = [ + "gix-commitgraph", + "gix-date", + "gix-diff", + "gix-error", + "gix-hash", + "gix-object", + "gix-revwalk", + "gix-trace", + "gix-traverse", + "gix-worktree", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-chunk" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9faee47943b638e58ddd5e275a4906ad3e4b6c8584f1d41bd18ab9032ec52afb" +dependencies = [ + "gix-error", +] + +[[package]] +name = "gix-command" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00706d4fef135ef4b01680d5218c6ee40cda8baf697b864296cbc887d19118f6" +dependencies = [ + "bstr", + "gix-path", + "gix-quote", + "gix-trace", + "shell-words", +] + +[[package]] +name = "gix-commitgraph" +version = "0.37.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f675d0df484a7f6a47e64bd6f311af489d947c0323b0564f36d14f3d7762abb" +dependencies = [ + "bstr", + "gix-chunk", + "gix-error", + "gix-hash", + "memmap2", + "nonempty", +] + +[[package]] +name = "gix-config" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2372d4b49ca28431e7d150cab9d25edc1890f0184bd57eb0e917c7799e63de" +dependencies = [ + "bstr", + "gix-config-value", + "gix-features", + "gix-glob", + "gix-path", + "gix-ref", + "gix-sec", + "smallvec", + "thiserror 2.0.18", + "unicode-bom", +] + +[[package]] +name = "gix-config-value" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed42168329552f6c2e5df09665c104199d45d84bedb53683738a49b57fe1baab" +dependencies = [ + "bitflags 2.13.0", + "bstr", + "gix-path", + "libc", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-credentials" +version = "0.38.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40cd22f0dd71988be12d6e78b1709de2370e1957c5f107ff31e56caeba3745d" +dependencies = [ + "bstr", + "gix-command", + "gix-config-value", + "gix-date", + "gix-path", + "gix-prompt", + "gix-sec", + "gix-trace", + "gix-url", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-date" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d63f9e28b59ddeb1a1eb9e5cf986a9222b5d484947445edbc20473939cc7fd0" +dependencies = [ + "bstr", + "gix-error", + "itoa", + "jiff", +] + +[[package]] +name = "gix-diff" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b6d9528f32d94cef2edf39a1ac01fe5a0fc44ddbb18d9e44099936047c3302b" +dependencies = [ + "bstr", + "gix-attributes", + "gix-command", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-imara-diff", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-tempfile", + "gix-trace", + "gix-traverse", + "gix-worktree", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-dir" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21bb2a53a6fd917ec499ed0bfb5b6887de7a15bd79197dcea7c987938749a9f1" +dependencies = [ + "bstr", + "gix-discover", + "gix-fs", + "gix-ignore", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-trace", + "gix-utils", + "gix-worktree", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-discover" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77bacdd12b7879d2178a80c58c2f319995e4654e1a7a23e3181e5c8a12b824f7" +dependencies = [ + "bstr", + "dunce", + "gix-fs", + "gix-path", + "gix-ref", + "gix-sec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-error" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57831e199be480af90dcd7e459abed8a174c09ec9a6e2cc8f7ca6c54598b06b" +dependencies = [ + "bstr", +] + +[[package]] +name = "gix-features" +version = "0.48.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1849ae154d38bc403185be14fa871e38e3c93ee606875d94e207fdb9fba52dbc" +dependencies = [ + "bytes", + "bytesize", + "crc32fast", + "crossbeam-channel", + "gix-path", + "gix-trace", + "gix-utils", + "libc", + "once_cell", + "parking_lot", + "prodash", + "thiserror 2.0.18", + "walkdir", + "zlib-rs", +] + +[[package]] +name = "gix-filter" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecf74b7d16f6694ce4a3049074c41be0c7987105743674f1671807bd6dce09fa" +dependencies = [ + "bstr", + "encoding_rs", + "gix-attributes", + "gix-command", + "gix-hash", + "gix-object", + "gix-packetline", + "gix-path", + "gix-quote", + "gix-trace", + "gix-utils", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-fs" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cdff46db8798e47e2f727d84b9379aac5add3dd3d9d0b07bb4d7d5d640771fe" +dependencies = [ + "bstr", + "fastrand", + "gix-features", + "gix-path", + "gix-utils", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-glob" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1fcb8ef5b16bcf874abe9b68d8abb3c0493c876d367ab824151f30a0f3f3756" +dependencies = [ + "bitflags 2.13.0", + "bstr", + "gix-features", + "gix-path", +] + +[[package]] +name = "gix-hash" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0926d3819c837750b4e03c7754901e73f68b8c9b690753a6372a1bed4eedce" +dependencies = [ + "faster-hex", + "gix-features", + "sha1-checked", + "sha2 0.11.0", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-hashtable" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e261d54091f0d1c729bc83f54548c071bdec60a697de1e58e88bdfd7a99d24e" +dependencies = [ + "gix-hash", + "hashbrown 0.17.1", + "parking_lot", +] + +[[package]] +name = "gix-ignore" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d491bab9bf2c9f341dc754f425c31d5d3f63aca615312167b82e1deeaca97d8d" +dependencies = [ + "bstr", + "gix-glob", + "gix-path", + "gix-trace", + "unicode-bom", +] + +[[package]] +name = "gix-imara-diff" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b305d85504de270ad3525d726a6b69cc59ee7b2269b014387651107ab9f0755b" +dependencies = [ + "bstr", + "hashbrown 0.15.5", +] + +[[package]] +name = "gix-index" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e6b28cc592dc753adb58302bb14a64e412ee591a3bec77aa4df87bff74fa80d" +dependencies = [ + "bitflags 2.13.0", + "bstr", + "filetime", + "fnv", + "gix-bitmap", + "gix-features", + "gix-fs", + "gix-hash", + "gix-lock", + "gix-object", + "gix-traverse", + "gix-utils", + "gix-validate", + "hashbrown 0.17.1", + "itoa", + "libc", + "memmap2", + "rustix", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-lock" +version = "23.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9dedd9e90b0d47624d2ed241d394e09294118364e87b9b7e5f1fe755f3c2c" +dependencies = [ + "gix-tempfile", + "gix-utils", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-mailmap" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "195fd20808055824531be2fd0d34136d900e5fbca3ffb0a3c07e8beeefb9c828" +dependencies = [ + "bstr", + "gix-actor", + "gix-date", + "gix-error", +] + +[[package]] +name = "gix-negotiate" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "890c936a215bae25818c076cb881cb2e54d2c66ba947ba58b8dd47cff921bf55" +dependencies = [ + "bitflags 2.13.0", + "gix-commitgraph", + "gix-date", + "gix-hash", + "gix-object", + "gix-revwalk", +] + +[[package]] +name = "gix-object" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5cd857e29429c7213bdef3f5aef83f8cc124774fe8ae0d27b1607d218d6d525" +dependencies = [ + "bstr", + "gix-actor", + "gix-date", + "gix-features", + "gix-hash", + "gix-hashtable", + "gix-utils", + "gix-validate", + "itoa", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-odb" +version = "0.81.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d004c32858b1556f2d7874405edb3c97dc78fc09beaa87d57bb077ee2858a7d" +dependencies = [ + "arc-swap", + "gix-features", + "gix-fs", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-pack", + "gix-path", + "gix-quote", + "memmap2", + "parking_lot", + "tempfile", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-pack" +version = "0.71.0" +dependencies = [ + "clru", + "gix-chunk", + "gix-diff", + "gix-error", + "gix-features", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-path", + "gix-tempfile", + "gix-traverse", + "parking_lot", + "smallvec", + "thiserror 2.0.18", + "uluru", +] + +[[package]] +name = "gix-packetline" +version = "0.21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b217dd0ee0c4021ecf169a4a519b1b4f80d15e3f3765f3dc466223dc0ac891d7" +dependencies = [ + "bstr", + "faster-hex", + "gix-trace", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-path" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afa6ac14cd14939ea94a496ce7460daa6511c09f5b84757e9cfc6f9c8d0f93a6" +dependencies = [ + "bstr", + "gix-trace", + "gix-validate", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-pathspec" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3050783b41ee11511e1e8fb35623df81806194f4030395f14f48ea37c2798c9f" +dependencies = [ + "bitflags 2.13.0", + "bstr", + "gix-attributes", + "gix-config-value", + "gix-glob", + "gix-path", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-prompt" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee604d7746080ae7e1023bf47204bcc2c5f307bfbe2306a3c90b1bfd1a2c6d8" +dependencies = [ + "gix-command", + "gix-config-value", + "parking_lot", + "rustix", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-protocol" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51dea3acb390707ab868f1f9584f18449eb95d869deffae96768e47d303595ee" +dependencies = [ + "bstr", + "gix-date", + "gix-features", + "gix-hash", + "gix-ref", + "gix-shallow", + "gix-transport", + "gix-utils", + "maybe-async", + "nonempty", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-quote" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6e541fc33cc2b783b7979040d445a0c86a2eca747c8faea4ca84230d06ae6ef" +dependencies = [ + "bstr", + "gix-error", + "gix-utils", +] + +[[package]] +name = "gix-ref" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c04f64c37eb7e6feb73c7060f8dc6f381cc5de5d53249bfd450bc48a86b2e8b" +dependencies = [ + "gix-actor", + "gix-features", + "gix-fs", + "gix-hash", + "gix-lock", + "gix-object", + "gix-path", + "gix-tempfile", + "gix-utils", + "gix-validate", + "memmap2", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-refspec" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b216ae06ec74b5f24ad0142026a997fb0a935b7410eaf9c1616fc3f0e6c5a6d3" +dependencies = [ + "bstr", + "gix-error", + "gix-glob", + "gix-hash", + "gix-revision", + "gix-validate", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-revision" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b47c88884dd3c1a19a39da19d10211fcdea2809aadc86869b6e824a1774340f" +dependencies = [ + "bitflags 2.13.0", + "bstr", + "gix-commitgraph", + "gix-date", + "gix-error", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-revwalk", + "gix-trace", + "nonempty", +] + +[[package]] +name = "gix-revwalk" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85f5756abffe0917827aac683b13684ed99875bc398fa1f9b8f479b0681ef9e6" +dependencies = [ + "gix-commitgraph", + "gix-date", + "gix-error", + "gix-hash", + "gix-hashtable", + "gix-object", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-sec" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab8519976e4c7e486270740a5400369f37940779b80bd1377d94cfa1125d01b3" +dependencies = [ + "bitflags 2.13.0", + "gix-path", + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "gix-shallow" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a292fc2fe548c5dfa575479d16b445b0ddf1dd2f56f1fec6aed386f82553cd97" +dependencies = [ + "bstr", + "gix-hash", + "gix-lock", + "nonempty", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-status" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22042e385d28a34275e029d98f4970285045be14b9073658ca897923f2ed8700" +dependencies = [ + "bstr", + "filetime", + "gix-diff", + "gix-dir", + "gix-features", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-worktree", + "portable-atomic", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-submodule" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3059890ef054066c22a94bfc6a3eaba0d806aedcd630a0bc9e5783fd88884781" +dependencies = [ + "bstr", + "gix-config", + "gix-path", + "gix-pathspec", + "gix-refspec", + "gix-url", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-tempfile" +version = "23.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ef60812443484e67bf84e444cc71b4c78ae62deb822221774a4fa0c57fdb17f" +dependencies = [ + "dashmap", + "gix-fs", + "libc", + "parking_lot", + "signal-hook", + "signal-hook-registry", + "tempfile", +] + +[[package]] +name = "gix-trace" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44dc45eae785c0eb14173e0f152e6e224dcf4d45b6a6999a3aed22af541ad678" + +[[package]] +name = "gix-transport" +version = "0.57.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "186874f7ad1fb2f9a2f2aa9c2dabc7f9dd087bef74c1a0eee2b4a9cf0248fcb3" +dependencies = [ + "bstr", + "gix-command", + "gix-features", + "gix-packetline", + "gix-quote", + "gix-sec", + "gix-url", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-traverse" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8de590ecc86a3b2870665f2288324fa9f7f8672c7fc2d4e020fdd81cd1f7aed" +dependencies = [ + "bitflags 2.13.0", + "gix-commitgraph", + "gix-date", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-revwalk", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-url" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bb01ec69d55e82ccb7a19e264501ead4e6aac38463a8cebfdd81e22bb67ab2" +dependencies = [ + "bstr", + "gix-path", + "percent-encoding", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-utils" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66c50966184123caf580ffa64e28031a878597f1c7fceb8fe19566c38eb1b771" +dependencies = [ + "bstr", + "fastrand", + "unicode-normalization", +] + +[[package]] +name = "gix-validate" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bc6fc771c4063ba7cd2f47b91fb6076251c6a823b64b7fe7b8874b0fe4afae3" +dependencies = [ + "bstr", +] + +[[package]] +name = "gix-worktree" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef414ed275e8407cd5d53d301e83be19700b0dd3f859d2434417b58f454a2d1" +dependencies = [ + "bstr", + "gix-attributes", + "gix-fs", + "gix-glob", + "gix-hash", + "gix-ignore", + "gix-index", + "gix-object", + "gix-path", + "gix-validate", +] + +[[package]] +name = "gix-worktree-state" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bffae8b3ca258fdd50370cd51f06deb4c76a3b43db3868bc28dde45ffa77d69" +dependencies = [ + "bstr", + "gix-features", + "gix-filter", + "gix-fs", + "gix-index", + "gix-object", + "gix-path", + "gix-worktree", + "io-close", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-worktree-stream" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25e9ed30100c63f7590bc581c225e53f731a53e06aa79a245739c07f7dcc557" +dependencies = [ + "gix-attributes", + "gix-error", + "gix-features", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-object", + "gix-path", + "gix-traverse", + "parking_lot", +] + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "h2" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "heapless" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" +dependencies = [ + "atomic-polyfill", + "hash32 0.2.1", + "rustc_version", + "serde", + "spin 0.9.8", + "stable_deref_trait", +] + +[[package]] +name = "heapless" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32 0.3.1", + "stable_deref_trait", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "human_format" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaec953f16e5bcf6b8a3cb3aa959b17e5577dbd2693e94554c462c08be22624b" + +[[package]] +name = "hybrid-array" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "inventory" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +dependencies = [ + "rustversion", +] + +[[package]] +name = "io-close" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cadcf447f06744f8ce713d2d6239bb5bde2c357a452397a9ed90c625da390bc" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "ipld-core" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090f624976d72f0b0bb71b86d58dc16c15e069193067cb3a3a09d655246cbbda" +dependencies = [ + "cid", + "serde", + "serde_bytes", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "iroh-car" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f8cd4cb9aa083fba8b52e921764252d0b4dcb1cd6d120b809dbfe1106e81a" +dependencies = [ + "anyhow", + "cid", + "futures", + "serde", + "serde_ipld_dagcbor", + "thiserror 1.0.69", + "tokio", + "unsigned-varint 0.7.2", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jacquard-api" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5c803a3c097e3ef8aea63747b4fe3fc9e339cd18272dd0366b1d10dd90d5c3f" +dependencies = [ + "jacquard-common", + "jacquard-derive", + "jacquard-lexicon", + "miette", + "serde", + "thiserror 2.0.18", +] + +[[package]] +name = "jacquard-common" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fec8b6661e6fcfb4d8b9eb1132503be71bc3e3c6f6ff15fa5500ec62655f6da7" +dependencies = [ + "base64", + "bon", + "bytes", + "chrono", + "ciborium", + "ciborium-io", + "cid", + "ed25519-dalek", + "fluent-uri", + "futures", + "getrandom 0.2.17", + "getrandom 0.3.4", + "hashbrown 0.15.5", + "http", + "ipld-core", + "k256", + "maitake-sync", + "miette", + "multibase", + "multihash", + "n0-future", + "oxilangtag", + "p256", + "phf", + "postcard", + "rand 0.9.4", + "regex", + "regex-automata", + "regex-lite", + "reqwest 0.12.28", + "rustversion", + "serde", + "serde_bytes", + "serde_html_form", + "serde_ipld_dagcbor", + "serde_json", + "signature", + "smol_str", + "spin 0.10.0", + "thiserror 2.0.18", + "tokio", + "tokio-tungstenite-wasm", + "tokio-util", + "trait-variant", + "unicode-segmentation", +] + +[[package]] +name = "jacquard-derive" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e41b82a9a72c0d2b907a5e0734e0692615e717cda903502c6c034ae6f3ddddf7" +dependencies = [ + "heck", + "jacquard-lexicon", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jacquard-lexicon" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "160049c269e3d7ec56f130736d25d61953fc6216da660c8b5e102f6b052dd3a1" +dependencies = [ + "cid", + "dashmap", + "heck", + "inventory", + "jacquard-common", + "miette", + "multihash", + "prettyplease", + "proc-macro2", + "quote", + "serde", + "serde_ipld_dagcbor", + "serde_json", + "serde_path_to_error", + "serde_repr", + "serde_with", + "sha2 0.10.9", + "syn", + "thiserror 2.0.18", + "unicode-segmentation", +] + +[[package]] +name = "jacquard-repo" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98986367bb78dadaa0f2f07196bab357786c0e3670d8311b350585b91f84d6eb" +dependencies = [ + "bytes", + "cid", + "ed25519-dalek", + "iroh-car", + "jacquard-api", + "jacquard-common", + "jacquard-derive", + "k256", + "miette", + "multihash", + "n0-future", + "p256", + "serde", + "serde_bytes", + "serde_ipld_dagcbor", + "sha2 0.10.9", + "smol_str", + "thiserror 2.0.18", + "tokio", + "trait-variant", +] + +[[package]] +name = "jiff" +version = "0.2.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccfe6121cbe750cf81efa362d85c0bde7ea298ec43092d3a193baca59cdbd634" +dependencies = [ + "defmt", + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link", +] + +[[package]] +name = "jiff-static" +version = "0.2.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e165e897f662d428f3cd3828a919dbe067c2d42bb1031eede74ef9d27ecdedd2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6142247df1a93c2b3587402a19710be3e6e942f1581a1702e76408f2c21d6590" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "k256" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" +dependencies = [ + "cfg-if", + "ecdsa", + "elliptic-curve", + "once_cell", + "sha2 0.10.9", + "signature", +] + +[[package]] +name = "knot-git" +version = "0.1.0" +dependencies = [ + "base64", + "dashmap", + "flate2", + "gix", + "gix-archive", + "gix-bitmap", + "gix-hash", + "gix-pack", + "knot-resource", + "knot-types", + "moka", + "thiserror 2.0.18", + "walkdir", +] + +[[package]] +name = "knot-lfs" +version = "0.1.0" +dependencies = [ + "gix-packetline", + "knot-git", + "knot-resource", + "knot-runtime", + "knot-types", + "serde", + "serde_json", + "sha2 0.11.0", + "tempfile", + "thiserror 2.0.18", + "tracing", + "url", +] + +[[package]] +name = "knot-lfs-fuzz" +version = "0.0.0" +dependencies = [ + "knot-lfs", + "libfuzzer-sys", +] + +[[package]] +name = "knot-resource" +version = "0.1.0" +dependencies = [ + "rustix", +] + +[[package]] +name = "knot-runtime" +version = "0.1.0" +dependencies = [ + "bytes", + "futures", + "getrandom 0.4.3", + "http", + "k256", + "reqwest 0.13.1", + "thiserror 2.0.18", + "tokio", + "url", +] + +[[package]] +name = "knot-types" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "chrono", + "cid", + "gix-hash", + "http", + "jacquard-common", + "jacquard-derive", + "jacquard-lexicon", + "jacquard-repo", + "miette", + "serde", + "serde_json", + "thiserror 2.0.18", + "walkdir", +] + +[[package]] +name = "kstring" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "558bf9508a558512042d3095138b1f7b8fe90c5467d94f9f1da28b3731c5dbd1" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "maitake-sync" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6816ab14147f80234c675b80ed6dc4f440d8a1cefc158e766067aedb84c0bcd5" +dependencies = [ + "cordyceps", + "loom", + "mycelium-bitfield", + "pin-project", + "portable-atomic", +] + +[[package]] +name = "match-lookup" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "757aee279b8bdbb9f9e676796fd459e4207a1f986e87886700abf589f5abf771" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "maybe-async" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "746873a384ad60adc5db74471dfaba74bd278afbdcfd81db93fafcdfc8b5ca0c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + +[[package]] +name = "miette" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" +dependencies = [ + "cfg-if", + "miette-derive", + "unicode-width", +] + +[[package]] +name = "miette-derive" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "moka" +version = "0.12.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "event-listener", + "futures-util", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "multibase" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e0e4a371cbf1dfd666b658ba137763edb23c45beb43cfe369b5593cd6b437b6" +dependencies = [ + "base-x", + "base256emoji", + "base45", + "data-encoding", + "data-encoding-macro", +] + +[[package]] +name = "multihash" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "577c63b00ad74d57e8c9aa870b5fccebf2fd64a308a5aee9f1bb88e4aea19447" +dependencies = [ + "serde", + "unsigned-varint 0.8.0", +] + +[[package]] +name = "mycelium-bitfield" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24e0cc5e2c585acbd15c5ce911dff71e1f4d5313f43345873311c4f5efd741cc" + +[[package]] +name = "n0-future" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb0e5d99e681ab3c938842b96fcb41bf8a7bb4bfdb11ccbd653a7e83e06c794" +dependencies = [ + "cfg_aliases", + "derive_more", + "futures-buffered", + "futures-lite", + "futures-util", + "js-sys", + "pin-project", + "send_wrapper", + "tokio", + "tokio-util", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "nonempty" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9737e026353e5cd0736f98eddae28665118eb6f6600902a7f50db585621fecb6" + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "oxilangtag" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d3b4eb570abd4a1dcb062c31fd37b832264d9dc7292c3e69acfe926c87b063f" +dependencies = [ + "serde", +] + +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2 0.10.9", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand 0.8.6", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "heapless 0.7.17", + "serde", +] + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prodash" +version = "31.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "962200e2d7d551451297d9fdce85138374019ada198e30ea9ede38034e27604c" +dependencies = [ + "bytesize", + "human_format", + "parking_lot", +] + +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.2", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.52.0", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + +[[package]] +name = "rawzip" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d9575f44c8cf85bc843ad666dcdf20d05a7753772bef56eb2a5140282b32150" + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.0", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "regex" +version = "1.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "mime", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "reqwest" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04e9018c9d814e5f30cc16a0f03271aeab3571e609612d9fe78c1aa8d11c2f62" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.13.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.23.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" +dependencies = [ + "aws-lc-rs", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.13.0", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "send_wrapper" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_html_form" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acf96b1d9364968fce46ebb548f1c0e1d7eceae27bdff73865d42e6c7369d94" +dependencies = [ + "form_urlencoded", + "indexmap", + "itoa", + "serde_core", +] + +[[package]] +name = "serde_ipld_dagcbor" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46182f4f08349a02b45c998ba3215d3f9de826246ba02bb9dddfe9a2a2100778" +dependencies = [ + "cbor4ii", + "ipld-core", + "scopeguard", + "serde", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" +dependencies = [ + "base64", + "bs58", + "chrono", + "hex", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha1-checked" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89f599ac0c323ebb1c6082821a54962b839832b03984598375bff3975b804423" +dependencies = [ + "digest 0.10.7", + "sha1", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shell-words" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a0c28ca5908dbdbcd52e6fdaa00358ab88637f8ab33e1f188dd510eb44b53d" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "smol_str" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa7368fcf4852a4c2dd92df0cace6a71f2091ca0a23391ce7f3a31833f1523" +dependencies = [ + "borsh", + "serde_core", +] + +[[package]] +name = "socket2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags 2.13.0", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.52.0", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" +dependencies = [ + "futures-util", + "log", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tungstenite", +] + +[[package]] +name = "tokio-tungstenite-wasm" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e21a5c399399c3db9f08d8297ac12b500e86bca82e930253fdc62eaf9c0de6ae" +dependencies = [ + "futures-channel", + "futures-util", + "http", + "httparse", + "js-sys", + "rustls", + "thiserror 1.0.69", + "tokio", + "tokio-tungstenite", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "async-compression", + "bitflags 2.13.0", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tokio", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "trait-variant" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.8.6", + "rustls", + "rustls-pki-types", + "sha1", + "thiserror 1.0.69", + "utf-8", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "uluru" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c8a2469e56e6e5095c82ccd3afb98dad95f7af7929aab6d8ba8d6e0f73657da" +dependencies = [ + "arrayvec", +] + +[[package]] +name = "unicode-bom" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eec5d1121208364f6793f7d2e222bf75a915c19557537745b195b253dd64217" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "unsigned-varint" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" + +[[package]] +name = "unsigned-varint" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", + "serde_derive", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "webpki-roots" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75726053136156d419e285b9b7eddaaea9e3fea6ce32eed44a89901f0bd98de1" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4714fd92cf900833d49538023a9b3915155210801d1c1169eba513b2addefd71" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zlib-rs" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5431d5661c32445236631278f27946e444ddafe4684cac70b185272d4f9c52d5" + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/knot2/crates/knot-lfs/fuzz/Cargo.toml b/knot2/crates/knot-lfs/fuzz/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-lfs/fuzz/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "knot-lfs-fuzz" +version = "0.0.0" +publish = false +edition = "2024" + +[package.metadata] +cargo-fuzz = true + +[workspace] + +[dependencies] +libfuzzer-sys = "0.4" + +[dependencies.knot-lfs] +path = ".." + +[[bin]] +name = "transfer" +path = "fuzz_targets/transfer.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "batch" +path = "fuzz_targets/batch.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "pointer" +path = "fuzz_targets/pointer.rs" +test = false +doc = false +bench = false + +[patch.crates-io] +gix-pack = { path = "../../../third_party/gix-pack" } diff --git a/knot2/crates/knot-lfs/src/admission.rs b/knot2/crates/knot-lfs/src/admission.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-lfs/src/admission.rs @@ -0,0 +1,160 @@ +use knot_resource::{DiskGovernor, DiskReservation, ReserveError}; + +use crate::{ClaimedSize, FreeSpaceFloor, LfsError, LfsSize, LfsStorePath}; + +pub trait UploadAdmission: Send + Sync { + fn admit(&self, declared: ClaimedSize) -> Result; + + fn max_object(&self) -> LfsSize; +} + +pub struct UploadPermit { + _reservation: Option, +} + +impl UploadPermit { + fn unreserved() -> Self { + Self { _reservation: None } + } +} + +pub struct StoreAdmission { + root: LfsStorePath, + max_object: LfsSize, + floor: FreeSpaceFloor, + governor: DiskGovernor, +} + +impl StoreAdmission { + pub fn new(root: LfsStorePath, max_object: LfsSize, floor: FreeSpaceFloor) -> Self { + let governor = DiskGovernor::new(knot_resource::DiskFloorBytes::new(floor.get())); + Self { + root, + max_object, + floor, + governor, + } + } +} + +impl UploadAdmission for StoreAdmission { + fn admit(&self, declared: ClaimedSize) -> Result { + if declared.get() > self.max_object.get() { + tracing::warn!( + declared = declared.get(), + limit = self.max_object.get(), + "lfs upload denied by the object size limit" + ); + return Err(LfsError::SizeLimitExceeded { + declared, + limit: self.max_object, + }); + } + if self.floor.get() == 0 { + return Ok(UploadPermit::unreserved()); + } + match self.governor.reserve( + self.root.as_path(), + knot_resource::ReserveBytes::new(declared.get()), + ) { + Ok(reservation) => Ok(UploadPermit { + _reservation: Some(reservation), + }), + Err(ReserveError::BelowFloor { free, .. }) => { + tracing::warn!( + declared = declared.get(), + free = free.get(), + floor = self.floor.get(), + "lfs upload denied below the free-space floor" + ); + Err(LfsError::FreeSpaceDenied { + free: LfsSize::new(free.get()), + floor: self.floor, + }) + } + Err(ReserveError::Probe(source)) => Err(LfsError::Io { + op: "probe free space under", + path: self.root.as_path().to_path_buf(), + source, + }), + } + } + + fn max_object(&self) -> LfsSize { + self.max_object + } +} + +#[cfg(test)] +pub(crate) struct Unbounded; + +#[cfg(test)] +impl UploadAdmission for Unbounded { + fn admit(&self, _declared: ClaimedSize) -> Result { + Ok(UploadPermit::unreserved()) + } + + fn max_object(&self) -> LfsSize { + LfsSize::new(u64::MAX) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_size_limit_rejects_before_touching_the_disk() { + let gate = StoreAdmission::new( + LfsStorePath::new("/definitely/not/mounted"), + LfsSize::new(8), + FreeSpaceFloor::new(0), + ); + assert!(matches!( + gate.admit(ClaimedSize::new(9)), + Err(LfsError::SizeLimitExceeded { .. }) + )); + assert!(gate.admit(ClaimedSize::new(8)).is_ok()); + } + + #[test] + fn an_absurd_floor_denies_and_a_zero_floor_opts_out() { + let dir = tempfile::tempdir().unwrap(); + let strict = StoreAdmission::new( + LfsStorePath::new(dir.path()), + LfsSize::new(u64::MAX), + FreeSpaceFloor::new(u64::MAX), + ); + assert!(matches!( + strict.admit(ClaimedSize::new(1)), + Err(LfsError::FreeSpaceDenied { .. }) + )); + let opted_out = StoreAdmission::new( + LfsStorePath::new(dir.path()), + LfsSize::new(u64::MAX), + FreeSpaceFloor::new(0), + ); + assert!(opted_out.admit(ClaimedSize::new(u64::MAX)).is_ok()); + } + + #[test] + fn a_held_permit_reserves_against_the_next_admission() { + let dir = tempfile::tempdir().unwrap(); + let free = knot_resource::disk_free_bytes(dir.path()).unwrap(); + let gate = StoreAdmission::new( + LfsStorePath::new(dir.path()), + LfsSize::new(u64::MAX), + FreeSpaceFloor::new(free.get().saturating_sub(6_144)), + ); + let held = gate.admit(ClaimedSize::new(4_096)).unwrap(); + assert!( + matches!( + gate.admit(ClaimedSize::new(4_096)), + Err(LfsError::FreeSpaceDenied { .. }) + ), + "a second upload cannot pass the floor while the first is in flight" + ); + drop(held); + assert!(gate.admit(ClaimedSize::new(4_096)).is_ok()); + } +} diff --git a/knot2/crates/knot-lfs/src/batch.rs b/knot2/crates/knot-lfs/src/batch.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-lfs/src/batch.rs @@ -0,0 +1,222 @@ +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use url::Url; + +use knot_types::{HttpStatus, RefName}; + +use crate::{ClaimedSize, LfsOid}; + +pub const BATCH_MEDIA_TYPE: &str = "application/vnd.git-lfs+json"; +pub const HASH_ALGO: &str = "sha256"; +pub const BASIC_TRANSFER: &str = "basic"; +pub const MAX_BATCH_OBJECTS: usize = 1000; + +// Unknown adapters parse instead of failing the whole body, +// so that a client offering something we don't serve +// will at least get a 422 that shows a mismatch not a serde error! +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TransferAdapter { + Basic, + Other(String), +} + +impl TransferAdapter { + pub fn is_basic(&self) -> bool { + matches!(self, Self::Basic) + } + + pub fn as_str(&self) -> &str { + match self { + Self::Basic => BASIC_TRANSFER, + Self::Other(value) => value, + } + } +} + +impl Serialize for TransferAdapter { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> Deserialize<'de> for TransferAdapter { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + Ok(if raw == BASIC_TRANSFER { + Self::Basic + } else { + Self::Other(raw) + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum HashAlgo { + Sha256, + Other(String), +} + +impl HashAlgo { + pub fn is_sha256(&self) -> bool { + matches!(self, Self::Sha256) + } + + pub fn as_str(&self) -> &str { + match self { + Self::Sha256 => HASH_ALGO, + Self::Other(value) => value, + } + } +} + +impl Serialize for HashAlgo { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> Deserialize<'de> for HashAlgo { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + Ok(if raw == HASH_ALGO { + Self::Sha256 + } else { + Self::Other(raw) + }) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum BatchOperation { + Download, + Upload, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BatchRef { + pub name: RefName, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct BatchObject { + pub oid: LfsOid, + pub size: ClaimedSize, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BatchRequest { + pub operation: BatchOperation, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub transfers: Vec, + #[serde(default, rename = "ref", skip_serializing_if = "Option::is_none")] + pub reference: Option, + pub objects: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub hash_algo: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BatchAction { + pub href: Url, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BatchActions { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub download: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub upload: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BatchObjectError { + pub code: HttpStatus, + pub message: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BatchResponseObject { + pub oid: LfsOid, + pub size: ClaimedSize, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub authenticated: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub actions: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +fn basic_transfer() -> TransferAdapter { + TransferAdapter::Basic +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BatchResponse { + #[serde(default = "basic_transfer")] + pub transfer: TransferAdapter, + pub objects: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub hash_algo: Option, +} + +#[cfg(test)] +mod tests { + use super::*; + + const OID: &str = "6c17f2007cbe934aee6e309b28b2fba3c119d98be6ea4156da3aa3173456ad16"; + + #[test] + fn a_client_batch_request_round_trips() { + let body = format!( + r#"{{"operation":"download","transfers":["basic","ssh"],"ref":{{"name":"refs/heads/main"}},"objects":[{{"oid":"{OID}","size":42}}],"hash_algo":"sha256"}}"# + ); + let request: BatchRequest = serde_json::from_str(&body).unwrap(); + assert_eq!(request.operation, BatchOperation::Download); + assert_eq!(request.objects[0].oid.as_str(), OID); + assert_eq!(request.objects[0].size, ClaimedSize::new(42)); + assert_eq!( + request.reference.as_ref().unwrap().name.as_str(), + "refs/heads/main" + ); + } + + #[test] + fn a_hostile_oid_fails_the_whole_parse() { + let body = r#"{"operation":"download","objects":[{"oid":"../../etc/passwd","size":1}]}"#; + assert!(serde_json::from_str::(body).is_err()); + } + + #[test] + fn a_response_serializes_the_lfs_shape() { + let response = BatchResponse { + transfer: TransferAdapter::Basic, + objects: vec![BatchResponseObject { + oid: LfsOid::new(OID).unwrap(), + size: ClaimedSize::new(42), + authenticated: Some(true), + actions: Some(BatchActions { + download: Some(BatchAction { + href: Url::parse(&format!( + "https://nel.pet/did:plc:squid/media/info/lfs/objects/{OID}" + )) + .unwrap(), + }), + upload: None, + }), + error: None, + }], + hash_algo: Some(HashAlgo::Sha256), + }; + let json = serde_json::to_value(&response).unwrap(); + assert_eq!(json["transfer"], "basic"); + assert_eq!(json["objects"][0]["oid"], OID); + assert_eq!(json["objects"][0]["authenticated"], true); + assert!( + json["objects"][0]["actions"]["download"]["href"] + .as_str() + .unwrap() + .ends_with(OID) + ); + assert_eq!(json["objects"][0].get("error"), None); + } +} diff --git a/knot2/crates/knot-lfs/src/error.rs b/knot2/crates/knot-lfs/src/error.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-lfs/src/error.rs @@ -0,0 +1,51 @@ +use std::path::PathBuf; + +use crate::{ClaimedSize, FreeSpaceFloor, LfsOid, LfsSize}; + +#[derive(Debug, thiserror::Error)] +pub enum LfsError { + #[error("invalid LFS oid {value:?}")] + InvalidOid { value: String }, + #[error("unsafe repo DID {did:?}")] + UnsafeRepoDid { did: String }, + #[error("oid mismatch, declared {declared}, computed {computed}")] + HashMismatch { declared: LfsOid, computed: LfsOid }, + #[error("size mismatch, declared {declared}, received {received}")] + SizeMismatch { + declared: ClaimedSize, + received: LfsSize, + }, + #[error("object size {declared} exceeds limit {limit}")] + SizeLimitExceeded { + declared: ClaimedSize, + limit: LfsSize, + }, + #[error("free space {free} below floor {floor}")] + FreeSpaceDenied { + free: LfsSize, + floor: FreeSpaceFloor, + }, + #[error("object {oid} not found")] + NotFound { oid: LfsOid }, + #[error("protocol framing fault: {detail}")] + Framing { detail: String }, + #[error("too many {what} in one message, limit {limit}")] + TooMany { what: &'static str, limit: usize }, + #[error("transfer channel fault")] + Channel { + #[source] + source: std::io::Error, + }, + #[error("object body read failed")] + BodyRead { + #[source] + source: std::io::Error, + }, + #[error("{op} {path} failed")] + Io { + op: &'static str, + path: PathBuf, + #[source] + source: std::io::Error, + }, +} diff --git a/knot2/crates/knot-lfs/src/gc.rs b/knot2/crates/knot-lfs/src/gc.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-lfs/src/gc.rs @@ -0,0 +1,291 @@ +use std::collections::HashSet; +use std::time::{Duration, SystemTime}; + +use knot_git::{GitError, Haves, Repo, Wants}; +use knot_types::{Oid, RepoDid, UnixSeconds}; + +use crate::store::{DiskStore, Reclaimed, expired}; +use crate::{LfsError, LfsOid, LfsSize, scan_pointers}; + +#[derive(Debug, thiserror::Error)] +pub enum GcError { + #[error("git: {0}")] + Git(#[from] GitError), + #[error("store: {0}")] + Store(#[from] LfsError), +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct GcReport { + pub scanned: usize, + pub marked: usize, + pub swept: usize, + pub bytes: LfsSize, +} + +fn unix_seconds(now: SystemTime) -> UnixSeconds { + let secs = now + .duration_since(SystemTime::UNIX_EPOCH) + .map(|delta| delta.as_secs() as i64) + .unwrap_or(0); + UnixSeconds::new(secs) +} + +fn reachable_roots(repo: &Repo, floor: UnixSeconds) -> Result, GitError> { + let mut roots: HashSet = repo + .references()? + .into_iter() + .filter(|record| !knot_git::is_reserved(&record.name)) + .map(|record| record.target) + .collect(); + repo.reflog_updates_since(floor) + .into_iter() + .for_each(|update| { + roots.insert(update.new); + if let Some(old) = update.old { + roots.insert(old); + } + }); + Ok(roots + .into_iter() + .filter(|oid| repo.contains(*oid)) + .collect()) +} + +fn reachable_pointers(repo: &Repo, floor: UnixSeconds) -> Result, GitError> { + let roots = reachable_roots(repo, floor)?; + if roots.is_empty() { + return Ok(HashSet::new()); + } + Ok(scan_pointers(repo, Wants::new(&roots), Haves::new(&[]))? + .into_keys() + .collect()) +} + +pub fn collect_repo( + store: &DiskStore, + repo: &Repo, + did: &RepoDid, + grace: Duration, + now: SystemTime, +) -> Result { + let stored = store.enumerate(did)?; + if stored.is_empty() { + return Ok(GcReport::default()); + } + let floor = unix_seconds(now).saturating_sub_secs(grace.as_secs().min(i64::MAX as u64) as i64); + let reachable = reachable_pointers(repo, floor)?; + stored + .iter() + .filter(|object| !reachable.contains(&object.oid)) + .filter(|object| expired(now, object.mtime, grace)) + .map(|object| store.collect_expired(did, &object.oid, grace, now)) + .try_fold( + GcReport { + scanned: stored.len(), + ..GcReport::default() + }, + |mut report, outcome| { + report.marked += 1; + if let Reclaimed::Swept(size) = outcome? { + report.swept += 1; + report.bytes = report.bytes.saturating_add(size); + } + Ok::<_, GcError>(report) + }, + ) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use knot_git::{ + EntryKind, Identity, Layout, NewCommit, RefUpdate, Repo, StagedAction, StagedChange, + }; + use knot_types::{AuthorName, BranchName, Email, Oid, RefName, RepoDid}; + use sha2::{Digest, Sha256}; + + use super::*; + use crate::store::DiskStore; + use crate::{ClaimedSize, LfsOid, LfsSize, LfsStore, LfsStorePath}; + + const EMPTY_TREE: &str = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"; + const DAY: Duration = Duration::from_secs(86_400); + const MONTHS: Duration = Duration::from_secs(60 * 86_400); + + struct Fixture { + _scan: tempfile::TempDir, + _lfs: tempfile::TempDir, + store: DiskStore, + did: RepoDid, + repo: Repo, + } + + fn fixture() -> Fixture { + let scan = tempfile::tempdir().unwrap(); + let lfs = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()).with_default_branch(BranchName::new("main").unwrap()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let repo = layout.create(&did).unwrap(); + let store = DiskStore::open(LfsStorePath::new(lfs.path())).unwrap(); + Fixture { + _scan: scan, + _lfs: lfs, + store, + did, + repo, + } + } + + fn who(secs: i64) -> Identity { + Identity { + name: AuthorName::new("nel"), + email: Email::new("nel@oyster.cafe"), + time: knot_types::UnixSeconds::new(secs), + offset_seconds: 0, + } + } + + fn put_media(f: &Fixture, bytes: &[u8]) -> (LfsOid, LfsSize) { + let oid = LfsOid::from_digest(Sha256::digest(bytes).into()); + let bytes_len = bytes.len() as u64; + f.store + .put(&f.did, &oid, ClaimedSize::new(bytes_len), &mut &bytes[..]) + .unwrap(); + (oid, LfsSize::new(bytes_len)) + } + + fn empty_commit(f: &Fixture, message: &str, secs: i64) -> Oid { + f.repo + .write_commit(&NewCommit { + tree: Oid::from_hex(EMPTY_TREE).unwrap(), + parents: Vec::new(), + author: who(secs), + committer: who(secs), + message: message.to_string(), + extra_headers: Vec::new(), + }) + .unwrap() + } + + fn commit_pointer(f: &Fixture, name: &str, oid: &LfsOid, size: LfsSize) -> Oid { + let pointer = + format!("version https://git-lfs.github.com/spec/v1\noid sha256:{oid}\nsize {size}\n") + .into_bytes(); + let tree = f + .repo + .write_staged_tree( + Oid::from_hex(EMPTY_TREE).unwrap(), + &[StagedChange { + path: knot_types::RepoPath::new("clip.bin").unwrap(), + action: StagedAction::Put { + content: pointer, + kind: EntryKind::Blob, + }, + }], + ) + .unwrap(); + let tip = f + .repo + .write_commit(&NewCommit { + tree, + parents: Vec::new(), + author: who(1_700_000_000), + committer: who(1_700_000_000), + message: "add media".to_string(), + extra_headers: Vec::new(), + }) + .unwrap(); + f.repo + .update_ref(&RefUpdate::Create { + name: RefName::new(name).unwrap(), + new: tip, + }) + .unwrap(); + tip + } + + fn age(f: &Fixture, oid: &LfsOid, past: Duration) { + let path = f.store.object_file(&f.did, oid).unwrap().unwrap().1; + std::fs::OpenOptions::new() + .write(true) + .open(&path) + .unwrap() + .set_modified(SystemTime::now() - past) + .unwrap(); + } + + fn collect(f: &Fixture, grace: Duration) -> GcReport { + collect_repo(&f.store, &f.repo, &f.did, grace, SystemTime::now()).unwrap() + } + + #[test] + fn a_reachable_object_is_never_swept_regardless_of_retention_source() { + let f = fixture(); + + let (live, live_size) = put_media(&f, b"referenced media"); + commit_pointer(&f, "refs/heads/keep", &live, live_size); + age(&f, &live, MONTHS); + + let (fresh, _) = put_media(&f, b"just uploaded, pointer still in flight"); + + let (forced, forced_size) = put_media(&f, b"orphaned by a force push"); + let old = commit_pointer(&f, "refs/heads/main", &forced, forced_size); + let replacement = empty_commit(&f, "drop media", 1_700_000_100); + f.repo + .update_ref(&RefUpdate::Update { + name: RefName::new("refs/heads/main").unwrap(), + old, + new: replacement, + }) + .unwrap(); + age(&f, &forced, MONTHS); + + let report = collect(&f, Duration::from_secs(14 * 86_400)); + assert_eq!(report.marked, 0, "no reachable object is ever marked"); + assert_eq!(report.swept, 0); + [&live, &fresh, &forced].iter().for_each(|oid| { + assert!( + f.store.probe(&f.did, oid).unwrap().is_some(), + "a live ref, the grace window, and the reflog old tip each keep their object" + ); + }); + } + + #[test] + fn an_unreferenced_expired_object_is_swept_even_after_its_branch_is_gone() { + let f = fixture(); + + let (orphan, orphan_size) = put_media(&f, b"orphaned media"); + age(&f, &orphan, MONTHS); + + let (dropped, dropped_size) = put_media(&f, b"lived on a branch that was deleted"); + let tip = commit_pointer(&f, "refs/heads/topic", &dropped, dropped_size); + f.repo + .update_ref(&RefUpdate::Delete { + name: RefName::new("refs/heads/topic").unwrap(), + old: tip, + }) + .unwrap(); + age(&f, &dropped, MONTHS); + + let report = collect(&f, DAY); + assert_eq!(report.marked, 2); + assert_eq!( + report.swept, 2, + "a plain orphan and a deleted-branch orphan are both swept once the mtime grace expires" + ); + assert_eq!(report.bytes, orphan_size.saturating_add(dropped_size)); + assert_eq!(f.store.probe(&f.did, &orphan).unwrap(), None); + assert_eq!(f.store.probe(&f.did, &dropped).unwrap(), None); + } + + #[test] + fn a_repo_with_no_stored_objects_never_walks_git() { + let f = fixture(); + let orphan_pointer = LfsOid::from_digest(Sha256::digest(b"pointer without bytes").into()); + commit_pointer(&f, "refs/heads/main", &orphan_pointer, LfsSize::new(21)); + assert_eq!(collect(&f, DAY), GcReport::default()); + } +} diff --git a/knot2/crates/knot-lfs/src/lib.rs b/knot2/crates/knot-lfs/src/lib.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-lfs/src/lib.rs @@ -0,0 +1,77 @@ +mod admission; +mod batch; +mod error; +mod gc; +mod pointer; +mod scan; +mod store; +mod transfer; +mod types; + +pub use admission::{StoreAdmission, UploadAdmission, UploadPermit}; +pub use batch::{ + BASIC_TRANSFER, BATCH_MEDIA_TYPE, BatchAction, BatchActions, BatchObject, BatchObjectError, + BatchOperation, BatchRef, BatchRequest, BatchResponse, BatchResponseObject, HASH_ALGO, + HashAlgo, MAX_BATCH_OBJECTS, TransferAdapter, +}; +pub use error::LfsError; +pub use gc::{GcError, GcReport, collect_repo}; +pub use pointer::{LfsPointer, POINTER_MAX_BYTES, parse_pointer}; +pub use scan::scan_pointers; +pub use store::{ + DiskStore, LfsHandle, LfsStore, MemoryStore, OrphanSweep, Reclaimed, StoredObject, +}; +pub use transfer::{TransferOp, serve_transfer}; +pub use types::{ClaimedSize, FreeSpaceFloor, LfsOid, LfsSize, LfsStorePath, ObjectRelPath}; + +#[doc(hidden)] +pub mod fuzz { + use knot_types::RepoDid; + + use crate::{ + BatchRequest, FreeSpaceFloor, LfsSize, LfsStore, LfsStorePath, MAX_BATCH_OBJECTS, + MemoryStore, StoreAdmission, TransferOp, parse_pointer, serve_transfer, + }; + + pub fn transfer(data: &[u8]) { + let admission = StoreAdmission::new( + LfsStorePath::new("/"), + LfsSize::new(u64::MAX), + FreeSpaceFloor::new(0), + ); + let repo = RepoDid::new("did:plc:squid").expect("static DID is valid"); + let messages = &knot_messages::default_catalog().lfs; + [TransferOp::Upload, TransferOp::Download] + .iter() + .for_each(|op| { + let store = MemoryStore::new(); + let _ = serve_transfer( + &store, + &admission, + &repo, + *op, + messages, + data, + std::io::sink(), + ); + }); + } + + pub fn batch(data: &[u8]) { + let Ok(request) = serde_json::from_slice::(data) else { + return; + }; + if request.objects.len() > MAX_BATCH_OBJECTS { + return; + } + let repo = RepoDid::new("did:plc:squid").expect("static DID is valid"); + let store = MemoryStore::new(); + request.objects.iter().for_each(|object| { + let _ = store.probe(&repo, &object.oid); + }); + } + + pub fn pointer(data: &[u8]) { + let _ = parse_pointer(data); + } +} diff --git a/knot2/crates/knot-lfs/src/pointer.rs b/knot2/crates/knot-lfs/src/pointer.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-lfs/src/pointer.rs @@ -0,0 +1,88 @@ +use crate::{ClaimedSize, LfsOid}; + +pub const POINTER_MAX_BYTES: u64 = 1024; + +const SPEC_URLS: [&str; 2] = [ + "https://git-lfs.github.com/spec/v1", + "https://hawser.github.com/spec/v1", +]; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LfsPointer { + pub oid: LfsOid, + pub size: ClaimedSize, +} + +pub fn parse_pointer(blob: &[u8]) -> Option { + if blob.len() as u64 > POINTER_MAX_BYTES { + return None; + } + let text = std::str::from_utf8(blob).ok()?; + let mut lines = text.lines(); + let spec = lines.next()?.strip_prefix("version ")?; + SPEC_URLS.contains(&spec).then_some(())?; + let fields: Vec<(&str, &str)> = lines.filter_map(|line| line.split_once(' ')).collect(); + let value_of = |key: &str| { + fields + .iter() + .find_map(|(name, value)| (*name == key).then_some(*value)) + }; + let oid = LfsOid::new(value_of("oid")?.strip_prefix("sha256:")?).ok()?; + let size = ClaimedSize::new(value_of("size")?.parse().ok()?); + Some(LfsPointer { oid, size }) +} + +#[cfg(test)] +mod tests { + use super::*; + + const OID: &str = "6c17f2007cbe934aee6e309b28b2fba3c119d98be6ea4156da3aa3173456ad16"; + + fn pointer_text() -> String { + format!("version https://git-lfs.github.com/spec/v1\noid sha256:{OID}\nsize 12345\n") + } + + #[test] + fn valid_variants_beyond_the_canonical_form_still_parse() { + let hawser = pointer_text().replace("git-lfs.github.com", "hawser.github.com"); + assert!( + parse_pointer(hawser.as_bytes()).is_some(), + "the legacy hawser spec still counts" + ); + let extra = format!( + "version https://git-lfs.github.com/spec/v1\nname media/clip.mp4\noid sha256:{OID}\nsize 7\n" + ); + assert!( + parse_pointer(extra.as_bytes()).is_some(), + "extra sorted keys are tolerated" + ); + } + + #[test] + fn non_pointers_are_rejected() { + let cases: Vec> = vec![ + b"".to_vec(), + b"plain source file\n".to_vec(), + format!("oid sha256:{OID}\nsize 7\n").into_bytes(), + format!("version https://oyster.cafe/spec/v1\noid sha256:{OID}\nsize 7\n").into_bytes(), + pointer_text().replace("sha256:", "sha512:").into_bytes(), + pointer_text() + .replace("size 12345", "size lots") + .into_bytes(), + format!("version https://git-lfs.github.com/spec/v1\noid sha256:{OID}\n").into_bytes(), + b"version https://git-lfs.github.com/spec/v1\nsize 7\n".to_vec(), + [pointer_text().into_bytes(), vec![0xff, 0xfe]].concat(), + [ + pointer_text().into_bytes(), + vec![b' '; POINTER_MAX_BYTES as usize], + ] + .concat(), + ]; + cases.iter().enumerate().for_each(|(index, blob)| { + assert!( + parse_pointer(blob).is_none(), + "case {index} parsed as a pointer" + ); + }); + } +} diff --git a/knot2/crates/knot-lfs/src/scan.rs b/knot2/crates/knot-lfs/src/scan.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-lfs/src/scan.rs @@ -0,0 +1,32 @@ +use std::collections::BTreeMap; + +use knot_git::{Filter, GitError, Haves, PackBudget, Repo, Wants}; + +use crate::pointer::{POINTER_MAX_BYTES, parse_pointer}; +use crate::{ClaimedSize, LfsOid}; + +pub fn scan_pointers( + repo: &Repo, + wants: Wants<'_>, + haves: Haves<'_>, +) -> Result, GitError> { + let selection = repo.select_pack_objects_filtered( + wants, + haves, + Filter::BlobLimit(POINTER_MAX_BYTES + 1), + PackBudget::unbounded(), + )?; + selection + .send + .iter() + .filter_map(|oid| match repo.blob_size(*oid) { + Ok(_) => Some(repo.read_blob(*oid)), + Err(GitError::ObjectType { .. }) => None, + Err(fault) => Some(Err(fault)), + }) + .filter_map(|blob| match blob { + Ok(bytes) => parse_pointer(&bytes).map(|pointer| Ok((pointer.oid, pointer.size))), + Err(fault) => Some(Err(fault)), + }) + .collect() +} diff --git a/knot2/crates/knot-lfs/src/store.rs b/knot2/crates/knot-lfs/src/store.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-lfs/src/store.rs @@ -0,0 +1,829 @@ +use std::collections::{HashMap, HashSet}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; +use std::time::{Duration, SystemTime}; + +use knot_types::RepoDid; +use sha2::{Digest, Sha256}; + +use crate::types::RepoPrefix; +use crate::{ClaimedSize, FreeSpaceFloor, LfsError, LfsOid, LfsSize, LfsStorePath, ObjectRelPath}; + +pub trait LfsStore: Send + Sync { + fn put( + &self, + repo: &RepoDid, + oid: &LfsOid, + size: ClaimedSize, + body: &mut dyn Read, + ) -> Result<(), LfsError>; + + fn read(&self, repo: &RepoDid, oid: &LfsOid) -> Result, LfsError>; + + fn probe(&self, repo: &RepoDid, oid: &LfsOid) -> Result, LfsError>; + + fn touch(&self, repo: &RepoDid, oid: &LfsOid) -> Result, LfsError>; +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StoredObject { + pub oid: LfsOid, + pub size: LfsSize, + pub mtime: SystemTime, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Reclaimed { + Swept(LfsSize), + Spared, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct OrphanSweep { + pub prefixes: usize, + pub objects: usize, + pub bytes: LfsSize, +} + +pub(crate) fn expired(now: SystemTime, mtime: SystemTime, grace: Duration) -> bool { + now.duration_since(mtime) + .map(|age| age >= grace) + .unwrap_or(false) +} + +const COPY_CHUNK: usize = 64 * 1024; + +fn io_at(op: &'static str, path: &Path) -> impl FnOnce(std::io::Error) -> LfsError { + let path = path.to_path_buf(); + move |source| LfsError::Io { op, path, source } +} + +pub(crate) fn for_each_chunk( + body: &mut dyn Read, + mut step: impl FnMut(&[u8]) -> Result<(), LfsError>, +) -> Result<(), LfsError> { + let mut buffer = vec![0u8; COPY_CHUNK]; + std::iter::from_fn(|| match body.read(&mut buffer) { + Ok(0) => None, + Ok(count) => Some(step(&buffer[..count])), + Err(source) if source.kind() == std::io::ErrorKind::Interrupted => Some(Ok(())), + Err(source) => Some(Err(LfsError::BodyRead { source })), + }) + .try_for_each(std::convert::identity) +} + +fn write_verified( + declared: &LfsOid, + size: ClaimedSize, + body: &mut dyn Read, + mut sink: impl FnMut(&[u8]) -> Result<(), LfsError>, +) -> Result<(), LfsError> { + let mut hasher = Sha256::new(); + let mut received: u64 = 0; + for_each_chunk(body, |chunk| { + received += chunk.len() as u64; + if received > size.get() { + return Err(LfsError::SizeMismatch { + declared: size, + received: LfsSize::new(received), + }); + } + hasher.update(chunk); + sink(chunk) + })?; + if received != size.get() { + return Err(LfsError::SizeMismatch { + declared: size, + received: LfsSize::new(received), + }); + } + let computed = LfsOid::from_digest(hasher.finalize().into()); + match computed == *declared { + true => Ok(()), + false => Err(LfsError::HashMismatch { + declared: declared.clone(), + computed, + }), + } +} + +fn fsync_dir(path: &Path) -> Result<(), LfsError> { + std::fs::File::open(path) + .and_then(|dir| dir.sync_all()) + .map_err(io_at("sync dir", path)) +} + +fn fsync_chain(root: &Path, leaf: &Path) -> Result<(), LfsError> { + leaf.ancestors() + .take_while(|dir| dir.starts_with(root)) + .try_for_each(fsync_dir) +} + +const INCOMING_DIR: &str = ".incoming"; +const OID_LOCK_STRIPES: usize = 64; + +fn set_mtime_now(path: &Path) -> Result<(), LfsError> { + std::fs::OpenOptions::new() + .write(true) + .open(path) + .and_then(|file| file.set_modified(SystemTime::now())) + .map_err(io_at("touch mtime", path)) +} + +fn subdirs(path: &Path) -> Result, LfsError> { + match std::fs::read_dir(path) { + Ok(entries) => entries + .map(|entry| { + entry + .map(|entry| entry.path()) + .map_err(io_at("read dir", path)) + }) + .filter(|entry| entry.as_ref().map(|path| path.is_dir()).unwrap_or(true)) + .collect(), + Err(source) if source.kind() == std::io::ErrorKind::NotFound => Ok(Vec::new()), + Err(source) => Err(io_at("read dir", path)(source)), + } +} + +fn stored_object(path: &Path) -> Result, LfsError> { + let Some(oid) = path + .file_name() + .and_then(|name| name.to_str()) + .and_then(|name| LfsOid::new(name).ok()) + else { + return Ok(None); + }; + let meta = std::fs::metadata(path).map_err(io_at("stat", path))?; + let mtime = meta.modified().map_err(io_at("read mtime", path))?; + Ok(Some(StoredObject { + oid, + size: LfsSize::new(meta.len()), + mtime, + })) +} + +fn is_object_file(path: &Path) -> bool { + path.is_file() + && path + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| LfsOid::new(name).is_ok()) +} + +fn is_shard_nibble(path: &Path) -> bool { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| { + name.len() == 2 + && name + .bytes() + .all(|byte| matches!(byte, b'0'..=b'9' | b'a'..=b'f')) + }) +} + +fn is_object_leaf(dir: &Path) -> bool { + is_shard_nibble(dir) && dir.parent().is_some_and(is_shard_nibble) +} + +fn discover_prefixes(dir: &Path) -> Result, LfsError> { + let entries = match std::fs::read_dir(dir) { + Ok(entries) => entries + .map(|entry| { + entry + .map(|entry| entry.path()) + .map_err(io_at("read dir", dir)) + }) + .collect::, _>>()?, + Err(source) if source.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(source) => return Err(io_at("read dir", dir)(source)), + }; + if is_object_leaf(dir) && entries.iter().any(|path| is_object_file(path)) { + return Ok(dir + .parent() + .and_then(|shard| shard.parent()) + .map(Path::to_path_buf) + .into_iter() + .collect()); + } + entries + .iter() + .filter(|path| path.is_dir()) + .map(|sub| discover_prefixes(sub)) + .collect::, _>>() + .map(|nested| nested.into_iter().flatten().collect()) +} + +fn enumerate_prefix(prefix: &Path) -> Result, LfsError> { + subdirs(prefix)? + .iter() + .map(|shard| subdirs(shard)) + .collect::, _>>()? + .into_iter() + .flatten() + .map(|nibble| match std::fs::read_dir(&nibble) { + Ok(entries) => entries + .map(|entry| { + entry + .map(|entry| entry.path()) + .map_err(io_at("read dir", &nibble)) + }) + .collect::, _>>(), + Err(source) if source.kind() == std::io::ErrorKind::NotFound => Ok(Vec::new()), + Err(source) => Err(io_at("read dir", &nibble)(source)), + }) + .collect::, _>>()? + .into_iter() + .flatten() + .filter_map(|file| stored_object(&file).transpose()) + .collect() +} + +pub struct DiskStore { + root: LfsStorePath, + locks: Box<[Mutex<()>]>, +} + +impl DiskStore { + pub fn open(root: LfsStorePath) -> Result { + let incoming = root.as_path().join(INCOMING_DIR); + std::fs::create_dir_all(&incoming).map_err(io_at("create dir", &incoming))?; + std::fs::read_dir(&incoming) + .map_err(io_at("read dir", &incoming))? + .try_for_each(|entry| { + let path = entry.map_err(io_at("read dir", &incoming))?.path(); + match std::fs::remove_file(&path) { + Ok(()) => Ok(()), + Err(source) if source.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(source) => Err(io_at("remove abandoned upload", &path)(source)), + } + })?; + let locks = std::iter::repeat_with(|| Mutex::new(())) + .take(OID_LOCK_STRIPES) + .collect(); + Ok(Self { root, locks }) + } + + fn object_path(&self, repo: &RepoDid, oid: &LfsOid) -> Result { + Ok(self.root.object_path(&ObjectRelPath::new(repo, oid)?)) + } + + fn oid_lock(&self, oid: &LfsOid) -> &Mutex<()> { + let stripe = u8::from_str_radix(&oid.as_str()[0..2], 16).unwrap_or(0) as usize; + &self.locks[stripe % OID_LOCK_STRIPES] + } +} + +impl LfsStore for DiskStore { + fn put( + &self, + repo: &RepoDid, + oid: &LfsOid, + size: ClaimedSize, + body: &mut dyn Read, + ) -> Result<(), LfsError> { + let target = self.object_path(repo, oid)?; + let incoming = self.root.as_path().join(INCOMING_DIR); + let mut temp = tempfile::Builder::new() + .prefix("put-") + .tempfile_in(&incoming) + .map_err(io_at("create temp under", &incoming))?; + let temp_path = temp.path().to_path_buf(); + write_verified(oid, size, body, |chunk| { + temp.as_file_mut() + .write_all(chunk) + .map_err(io_at("write", &temp_path)) + })?; + temp.as_file() + .sync_all() + .map_err(io_at("sync", &temp_path))?; + let parent = target + .parent() + .expect("object path always has a shard parent"); + let _guard = self + .oid_lock(oid) + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + std::fs::create_dir_all(parent).map_err(io_at("create dir", parent))?; + temp.persist(&target).map_err(|fault| LfsError::Io { + op: "rename into", + path: target.clone(), + source: fault.error, + })?; + fsync_chain(self.root.as_path(), parent) + } + + fn read(&self, repo: &RepoDid, oid: &LfsOid) -> Result, LfsError> { + let path = self.object_path(repo, oid)?; + match std::fs::File::open(&path) { + Ok(file) => Ok(Box::new(file)), + Err(source) if source.kind() == std::io::ErrorKind::NotFound => { + Err(LfsError::NotFound { oid: oid.clone() }) + } + Err(source) => Err(io_at("open", &path)(source)), + } + } + + fn probe(&self, repo: &RepoDid, oid: &LfsOid) -> Result, LfsError> { + Ok(self.object_file(repo, oid)?.map(|(size, _)| size)) + } + + fn touch(&self, repo: &RepoDid, oid: &LfsOid) -> Result, LfsError> { + let path = self.object_path(repo, oid)?; + let _guard = self + .oid_lock(oid) + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + match std::fs::metadata(&path) { + Ok(meta) => { + set_mtime_now(&path)?; + Ok(Some(LfsSize::new(meta.len()))) + } + Err(source) if source.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(source) => Err(io_at("stat", &path)(source)), + } + } +} + +impl DiskStore { + pub fn object_file( + &self, + repo: &RepoDid, + oid: &LfsOid, + ) -> Result, LfsError> { + let path = self.object_path(repo, oid)?; + match std::fs::metadata(&path) { + Ok(meta) => Ok(Some((LfsSize::new(meta.len()), path))), + Err(source) if source.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(source) => Err(io_at("stat", &path)(source)), + } + } + + pub fn probe_ready(&self) -> Result<(), LfsError> { + let incoming = self.root.as_path().join(INCOMING_DIR); + tempfile::tempfile_in(&incoming) + .map(|_| ()) + .map_err(io_at("probe writability under", &incoming)) + } + + pub fn remove_repo(&self, repo: &RepoDid) -> Result<(), LfsError> { + let prefix = self.root.as_path().join(RepoPrefix::new(repo)?.as_path()); + match std::fs::remove_dir_all(&prefix) { + Ok(()) => Ok(()), + Err(source) if source.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(source) => Err(io_at("remove repo prefix", &prefix)(source)), + } + } + + pub fn enumerate(&self, repo: &RepoDid) -> Result, LfsError> { + let prefix = self.root.as_path().join(RepoPrefix::new(repo)?.as_path()); + enumerate_prefix(&prefix) + } + + pub fn collect_expired( + &self, + repo: &RepoDid, + oid: &LfsOid, + grace: Duration, + now: SystemTime, + ) -> Result { + let path = self.object_path(repo, oid)?; + let _guard = self + .oid_lock(oid) + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let meta = match std::fs::metadata(&path) { + Ok(meta) => meta, + Err(source) if source.kind() == std::io::ErrorKind::NotFound => { + return Ok(Reclaimed::Spared); + } + Err(source) => return Err(io_at("stat", &path)(source)), + }; + let mtime = meta.modified().map_err(io_at("read mtime", &path))?; + if !expired(now, mtime, grace) { + return Ok(Reclaimed::Spared); + } + match std::fs::remove_file(&path) { + Ok(()) => Ok(Reclaimed::Swept(LfsSize::new(meta.len()))), + Err(source) if source.kind() == std::io::ErrorKind::NotFound => Ok(Reclaimed::Spared), + Err(source) => Err(io_at("remove object", &path)(source)), + } + } + + pub fn sweep_orphans( + &self, + hosted: &HashSet, + grace: Duration, + now: SystemTime, + ) -> Result { + let root = self.root.as_path(); + let expected: HashSet = hosted + .iter() + .filter_map(|repo| RepoPrefix::new(repo).ok()) + .map(|prefix| prefix.as_path().to_path_buf()) + .collect(); + let orphans: HashSet = discover_prefixes(root)? + .into_iter() + .filter(|prefix| { + prefix + .strip_prefix(root) + .ok() + .filter(|rel| matches!(rel.components().count(), 2 | 3)) + .map(|rel| !expected.contains(rel)) + .unwrap_or(false) + }) + .collect(); + orphans + .iter() + .map(|prefix| self.reclaim_orphan(prefix, grace, now)) + .try_fold(OrphanSweep::default(), |acc, outcome| { + let outcome = outcome?; + Ok(OrphanSweep { + prefixes: acc.prefixes + outcome.prefixes, + objects: acc.objects + outcome.objects, + bytes: acc.bytes.saturating_add(outcome.bytes), + }) + }) + } + + fn reclaim_orphan( + &self, + prefix: &Path, + grace: Duration, + now: SystemTime, + ) -> Result { + let objects = enumerate_prefix(prefix)?; + let live = objects + .iter() + .any(|object| !expired(now, object.mtime, grace)); + if live { + return Ok(OrphanSweep::default()); + } + match std::fs::remove_dir_all(prefix) { + Ok(()) => Ok(OrphanSweep { + prefixes: 1, + objects: objects.len(), + bytes: objects + .iter() + .map(|object| object.size) + .fold(LfsSize::new(0), LfsSize::saturating_add), + }), + Err(source) if source.kind() == std::io::ErrorKind::NotFound => { + Ok(OrphanSweep::default()) + } + Err(source) => Err(io_at("remove orphan prefix", prefix)(source)), + } + } +} + +#[derive(Clone)] +pub struct LfsHandle { + pub store: std::sync::Arc, + pub admission: std::sync::Arc, +} + +impl LfsHandle { + pub fn open( + root: LfsStorePath, + max_object: LfsSize, + free_space_floor: FreeSpaceFloor, + ) -> Result { + let admission = crate::StoreAdmission::new(root.clone(), max_object, free_space_floor); + Ok(Self { + store: std::sync::Arc::new(DiskStore::open(root)?), + admission: std::sync::Arc::new(admission), + }) + } +} + +#[derive(Default)] +pub struct MemoryStore { + objects: Mutex>>, +} + +impl MemoryStore { + pub fn new() -> Self { + Self::default() + } + + fn locked(&self) -> std::sync::MutexGuard<'_, HashMap>> { + self.objects.lock().expect("lfs memory store lock poisoned") + } +} + +impl LfsStore for MemoryStore { + fn put( + &self, + repo: &RepoDid, + oid: &LfsOid, + size: ClaimedSize, + body: &mut dyn Read, + ) -> Result<(), LfsError> { + let rel = ObjectRelPath::new(repo, oid)?; + let mut bytes = Vec::new(); + write_verified(oid, size, body, |chunk| { + bytes.extend_from_slice(chunk); + Ok(()) + })?; + self.locked().insert(rel, bytes); + Ok(()) + } + + fn read(&self, repo: &RepoDid, oid: &LfsOid) -> Result, LfsError> { + let rel = ObjectRelPath::new(repo, oid)?; + self.locked() + .get(&rel) + .cloned() + .map(|bytes| Box::new(std::io::Cursor::new(bytes)) as Box) + .ok_or_else(|| LfsError::NotFound { oid: oid.clone() }) + } + + fn probe(&self, repo: &RepoDid, oid: &LfsOid) -> Result, LfsError> { + let rel = ObjectRelPath::new(repo, oid)?; + Ok(self + .locked() + .get(&rel) + .map(|bytes| LfsSize::new(bytes.len() as u64))) + } + + fn touch(&self, repo: &RepoDid, oid: &LfsOid) -> Result, LfsError> { + self.probe(repo, oid) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const MONTH: Duration = Duration::from_secs(30 * 86_400); + const GRACE: Duration = Duration::from_secs(14 * 86_400); + + fn oid_of(bytes: &[u8]) -> LfsOid { + LfsOid::from_digest(Sha256::digest(bytes).into()) + } + + fn disk() -> (DiskStore, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let store = DiskStore::open(LfsStorePath::new(dir.path())).unwrap(); + (store, dir) + } + + fn seed(store: &DiskStore, repo: &RepoDid, body: &[u8]) -> (LfsOid, LfsSize) { + let oid = oid_of(body); + let bytes = body.len() as u64; + store + .put(repo, &oid, ClaimedSize::new(bytes), &mut &body[..]) + .unwrap(); + (oid, LfsSize::new(bytes)) + } + + fn backdate(store: &DiskStore, repo: &RepoDid, oid: &LfsOid, past: Duration) { + let path = store.object_file(repo, oid).unwrap().unwrap().1; + std::fs::OpenOptions::new() + .write(true) + .open(&path) + .unwrap() + .set_modified(SystemTime::now() - past) + .unwrap(); + } + + fn read_back(store: &dyn LfsStore, repo: &RepoDid, oid: &LfsOid) -> Vec { + let mut out = Vec::new(); + store + .read(repo, oid) + .unwrap() + .read_to_end(&mut out) + .unwrap(); + out + } + + fn store_contract(store: &dyn LfsStore) { + let repo = RepoDid::new("did:plc:squid").unwrap(); + let body: &[u8] = b"lfs media bytes for the round trip"; + let oid = oid_of(body); + let size = LfsSize::new(body.len() as u64); + + assert_eq!(store.probe(&repo, &oid).unwrap(), None); + assert!(matches!( + store.read(&repo, &oid), + Err(LfsError::NotFound { .. }) + )); + + let claim = ClaimedSize::new(size.get()); + store.put(&repo, &oid, claim, &mut &body[..]).unwrap(); + store.put(&repo, &oid, claim, &mut &body[..]).unwrap(); + assert_eq!( + store.probe(&repo, &oid).unwrap(), + Some(size), + "a re-put of identical bytes is idempotent" + ); + + let other = RepoDid::new("did:plc:limpet").unwrap(); + assert_eq!(store.probe(&other, &oid).unwrap(), None); + assert!(matches!( + store.read(&other, &oid), + Err(LfsError::NotFound { .. }) + )); + } + + #[test] + fn every_store_honors_the_contract() { + store_contract(&MemoryStore::new()); + let (store, _dir) = disk(); + store_contract(&store); + } + + #[test] + fn a_body_longer_than_its_declared_size_errors_before_the_end() { + let store = MemoryStore::new(); + let repo = RepoDid::new("did:plc:squid").unwrap(); + let mut endless = std::io::repeat(0x5a); + assert!(matches!( + store.put( + &repo, + &oid_of(b"whatever"), + ClaimedSize::new(8), + &mut endless + ), + Err(LfsError::SizeMismatch { .. }) + )); + } + + #[test] + fn disk_writes_are_sharded_and_the_boot_sweep_clears_only_temp_files() { + let (store, dir) = disk(); + let repo = RepoDid::new("did:plc:squid").unwrap(); + let (oid, size) = seed(&store, &repo, b"sharded placement"); + let sharded = dir + .path() + .join("plc/sq/uid") + .join(&oid.as_str()[0..2]) + .join(&oid.as_str()[2..4]) + .join(oid.as_str()); + assert_eq!(std::fs::read(&sharded).unwrap(), b"sharded placement"); + + let method = RepoDid::new("did:incoming:squid").unwrap(); + seed( + &store, + &method, + b"a method named incoming mustn't alias the temp dir", + ); + + let tampered = oid_of(b"a different object"); + assert!(matches!( + store.put(&repo, &tampered, ClaimedSize::new(4), &mut &b"nope"[..]), + Err(LfsError::HashMismatch { .. }) + )); + assert_eq!(store.probe(&repo, &tampered).unwrap(), None); + let incoming = dir.path().join(INCOMING_DIR); + assert!( + std::fs::read_dir(&incoming).unwrap().next().is_none(), + "a failed put leaves no files in the incoming dir" + ); + + std::fs::write(incoming.join("put-torn4321"), b"partial bytes from a crash").unwrap(); + let store = DiskStore::open(LfsStorePath::new(dir.path())).unwrap(); + assert!( + std::fs::read_dir(&incoming).unwrap().next().is_none(), + "the boot sweep clears abandoned uploads" + ); + assert_eq!(store.probe(&repo, &oid).unwrap(), Some(size)); + assert_eq!(read_back(&store, &repo, &oid), b"sharded placement"); + } + + #[test] + fn remove_repo_reclaims_the_prefix_and_spares_shard_neighbors() { + let (store, _dir) = disk(); + let doomed = RepoDid::new("did:plc:squid").unwrap(); + let neighbor = RepoDid::new("did:plc:squirrel").unwrap(); + let (oid, size) = seed(&store, &doomed, b"prefix removal"); + seed(&store, &neighbor, b"prefix removal"); + store.remove_repo(&doomed).unwrap(); + assert_eq!(store.probe(&doomed, &oid).unwrap(), None); + assert_eq!(store.probe(&neighbor, &oid).unwrap(), Some(size)); + store.remove_repo(&doomed).unwrap(); + } + + #[test] + fn collect_touch_and_enumerate_govern_the_sweep_per_object() { + let (store, _dir) = disk(); + let repo = RepoDid::new("did:plc:squid").unwrap(); + assert!( + store + .enumerate(&RepoDid::new("did:plc:limpet").unwrap()) + .unwrap() + .is_empty(), + "a missing prefix enumerates to nothing" + ); + + let (stale, stale_size) = seed(&store, &repo, b"long unreferenced"); + let (fresh, _) = seed(&store, &repo, b"still within grace"); + let (vouched, vouched_size) = seed(&store, &repo, b"vouched for moments before the sweep"); + backdate(&store, &repo, &stale, MONTH); + backdate(&store, &repo, &vouched, MONTH); + let now = SystemTime::now(); + + let listed: HashSet = store + .enumerate(&repo) + .unwrap() + .into_iter() + .map(|object| object.oid) + .collect(); + assert_eq!( + listed, + HashSet::from([stale.clone(), fresh.clone(), vouched.clone()]) + ); + + assert_eq!( + store.collect_expired(&repo, &fresh, GRACE, now).unwrap(), + Reclaimed::Spared, + "a fresh object is inside its grace window" + ); + assert_eq!( + store.collect_expired(&repo, &stale, GRACE, now).unwrap(), + Reclaimed::Swept(stale_size) + ); + assert_eq!(store.probe(&repo, &stale).unwrap(), None); + assert_eq!( + store.collect_expired(&repo, &stale, GRACE, now).unwrap(), + Reclaimed::Spared, + "collecting an already-gone object is a no-op" + ); + + assert_eq!(store.touch(&repo, &vouched).unwrap(), Some(vouched_size)); + assert_eq!( + store + .collect_expired(&repo, &vouched, GRACE, SystemTime::now()) + .unwrap(), + Reclaimed::Spared, + "the touch bumped the mtime inside the grace window" + ); + assert_eq!(store.probe(&repo, &vouched).unwrap(), Some(vouched_size)); + } + + #[test] + fn the_orphan_sweep_reclaims_unregistered_prefixes_and_spares_every_other_class() { + let (store, dir) = disk(); + let hosted = RepoDid::new("did:plc:squid").unwrap(); + let orphan = RepoDid::new("did:plc:limpet").unwrap(); + let fresh = RepoDid::new("did:plc:cuttle").unwrap(); + let short_hosted = RepoDid::new("did:web:ab").unwrap(); + let short_orphan = RepoDid::new("did:web:cd").unwrap(); + assert_eq!( + RepoPrefix::new(&short_hosted) + .unwrap() + .as_path() + .components() + .count(), + 2, + "a short method-specific-id shards to a two-component prefix" + ); + + let (kept, kept_size) = seed(&store, &hosted, b"belongs to a live repo"); + let (doomed, doomed_size) = seed(&store, &orphan, b"repo was deleted"); + let (spared, spared_size) = seed(&store, &fresh, b"deleted repo, but only just"); + let (short_kept, short_kept_size) = seed(&store, &short_hosted, b"live short-did object"); + let (short_doomed, short_doomed_size) = + seed(&store, &short_orphan, b"orphaned short-did media"); + [ + (&hosted, &kept), + (&orphan, &doomed), + (&short_hosted, &short_kept), + (&short_orphan, &short_doomed), + ] + .iter() + .for_each(|(did, oid)| backdate(&store, did, oid, MONTH)); + + std::fs::write( + dir.path().join(doomed.as_str()), + b"stray at the wrong depth", + ) + .unwrap(); + + let registry = HashSet::from([hosted.clone(), short_hosted.clone()]); + let sweep = store + .sweep_orphans(®istry, GRACE, SystemTime::now()) + .unwrap(); + + assert_eq!(sweep.prefixes, 2, "both past-grace orphans are reclaimed"); + assert_eq!(sweep.objects, 2, "one object under each reclaimed prefix"); + assert_eq!(sweep.bytes, doomed_size.saturating_add(short_doomed_size)); + assert_eq!(store.probe(&orphan, &doomed).unwrap(), None); + assert_eq!(store.probe(&short_orphan, &short_doomed).unwrap(), None); + assert_eq!( + store.probe(&hosted, &kept).unwrap(), + Some(kept_size), + "a hosted prefix is never an orphan" + ); + assert_eq!( + store.probe(&short_hosted, &short_kept).unwrap(), + Some(short_kept_size), + "a hosted repo shallower than the oid shards survives too" + ); + assert_eq!( + store.probe(&fresh, &spared).unwrap(), + Some(spared_size), + "a fresh orphan is held by the grace window" + ); + } +} diff --git a/knot2/crates/knot-lfs/src/transfer.rs b/knot2/crates/knot-lfs/src/transfer.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-lfs/src/transfer.rs @@ -0,0 +1,1175 @@ +use std::io::{Read, Write}; + +use gix_packetline::PacketLineRef; +use gix_packetline::blocking_io::{StreamingPeekableIter, encode}; +use knot_messages::{ + AlgorithmKey, CommandKey, DeclaredComputedKey, DeclaredLimitKey, DeclaredReceivedKey, + DetailKey, FreeFloorKey, LfsMessages, OidKey, ValueKey, VersionKey, WhatLimitKey, +}; +use knot_types::{HttpStatus, RepoDid}; + +use crate::store::for_each_chunk; +use crate::{ + BatchObject, ClaimedSize, LfsError, LfsOid, LfsSize, LfsStore, MAX_BATCH_OBJECTS, + UploadAdmission, +}; + +pub const CAPABILITY_VERSION: &str = "version=1"; +const PKT_DATA_MAX: usize = 65516; +const MAX_MESSAGE_ARGS: usize = 64; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TransferOp { + Upload, + Download, +} + +impl TransferOp { + pub fn parse(token: &str) -> Option { + match token { + "upload" => Some(Self::Upload), + "download" => Some(Self::Download), + _ => None, + } + } +} + +#[allow(clippy::too_many_arguments)] +pub fn serve_transfer( + store: &dyn LfsStore, + admission: &dyn UploadAdmission, + repo: &RepoDid, + op: TransferOp, + messages: &LfsMessages, + input: impl Read, + mut output: impl Write, +) -> Result<(), LfsError> { + write_text(&mut output, CAPABILITY_VERSION)?; + write_flush(&mut output)?; + let mut session = Session { + store, + admission, + repo, + op, + messages, + lines: StreamingPeekableIter::new(input, &[], false), + out: output, + }; + let mut done = false; + std::iter::from_fn(|| { + (!done).then(|| { + session.step().map(|flow| match flow { + Flow::Continue => (), + Flow::Quit => done = true, + }) + }) + }) + .try_for_each(std::convert::identity) +} + +enum Flow { + Continue, + Quit, +} + +enum Pkt { + Data(Vec), + Flush, + Delim, + Eof, +} + +enum Ended { + Flush, + Delim, +} + +fn next_pkt(lines: &mut StreamingPeekableIter) -> Result { + match lines.read_line() { + None => Ok(Pkt::Eof), + Some(Err(source)) if source.kind() == std::io::ErrorKind::UnexpectedEof => Ok(Pkt::Eof), + Some(Err(source)) => Err(LfsError::Channel { source }), + Some(Ok(Err(fault))) => Err(LfsError::Framing { + detail: fault.to_string(), + }), + Some(Ok(Ok(PacketLineRef::Data(payload)))) => Ok(Pkt::Data(payload.to_vec())), + Some(Ok(Ok(PacketLineRef::Flush))) => Ok(Pkt::Flush), + Some(Ok(Ok(PacketLineRef::Delimiter))) => Ok(Pkt::Delim), + Some(Ok(Ok(PacketLineRef::ResponseEnd))) => Err(LfsError::Framing { + detail: "unexpected response-end packet".to_string(), + }), + } +} + +fn text_of(payload: Vec) -> Result { + String::from_utf8(payload) + .map(|line| line.trim_end_matches('\n').to_string()) + .map_err(|_| LfsError::Framing { + detail: "non-utf8 text packet".to_string(), + }) +} + +fn fault_text(fault: &LfsError, messages: &LfsMessages) -> String { + match fault { + LfsError::InvalidOid { value } => messages + .invalid_oid + .line(|ValueKey::Value| format!("{value:?}")), + LfsError::HashMismatch { declared, computed } => { + messages.hash_mismatch.line(|key| match key { + DeclaredComputedKey::Declared => declared.to_string(), + DeclaredComputedKey::Computed => computed.to_string(), + }) + } + LfsError::SizeMismatch { declared, received } => { + messages.size_mismatch.line(|key| match key { + DeclaredReceivedKey::Declared => declared.to_string(), + DeclaredReceivedKey::Received => received.to_string(), + }) + } + LfsError::SizeLimitExceeded { declared, limit } => { + messages.size_limit_exceeded.line(|key| match key { + DeclaredLimitKey::Declared => declared.to_string(), + DeclaredLimitKey::Limit => limit.to_string(), + }) + } + LfsError::FreeSpaceDenied { free, floor } => { + messages.free_space_denied.line(|key| match key { + FreeFloorKey::Free => free.to_string(), + FreeFloorKey::Floor => floor.to_string(), + }) + } + LfsError::NotFound { oid } => messages.not_found.line(|OidKey::Oid| oid.to_string()), + LfsError::Framing { detail } => messages.framing.line(|DetailKey::Detail| detail.clone()), + LfsError::TooMany { what, limit } => messages.too_many.line(|key| match key { + WhatLimitKey::What => what.to_string(), + WhatLimitKey::Limit => limit.to_string(), + }), + other => other.to_string(), + } +} + +fn status_of(fault: &LfsError) -> HttpStatus { + HttpStatus::new(match fault { + LfsError::NotFound { .. } => 404, + LfsError::InvalidOid { .. } + | LfsError::HashMismatch { .. } + | LfsError::SizeMismatch { .. } + | LfsError::Framing { .. } => 400, + LfsError::SizeLimitExceeded { .. } => 413, + LfsError::FreeSpaceDenied { .. } => 429, + _ => 500, + }) +} + +fn write_text(out: &mut impl Write, line: &str) -> Result<(), LfsError> { + encode::data_to_write(format!("{line}\n").as_bytes(), &mut *out) + .map(|_| ()) + .map_err(|source| LfsError::Channel { source }) +} + +fn write_flush(out: &mut impl Write) -> Result<(), LfsError> { + encode::flush_to_write(&mut *out) + .and_then(|_| out.flush().map(|()| 0)) + .map(|_| ()) + .map_err(|source| LfsError::Channel { source }) +} + +fn write_delim(out: &mut impl Write) -> Result<(), LfsError> { + encode::delim_to_write(&mut *out) + .map(|_| ()) + .map_err(|source| LfsError::Channel { source }) +} + +fn arg_value<'a>(args: &'a [String], key: &str) -> Option<&'a str> { + args.iter().find_map(|arg| { + arg.strip_prefix(key) + .and_then(|rest| rest.strip_prefix('=')) + }) +} + +struct Session<'a, R: Read, W: Write> { + store: &'a dyn LfsStore, + admission: &'a dyn UploadAdmission, + repo: &'a RepoDid, + op: TransferOp, + messages: &'a LfsMessages, + lines: StreamingPeekableIter, + out: W, +} + +impl Session<'_, R, W> { + fn step(&mut self) -> Result { + match next_pkt(&mut self.lines)? { + Pkt::Eof => Ok(Flow::Quit), + Pkt::Flush | Pkt::Delim => Err(LfsError::Framing { + detail: "expected a command packet".to_string(), + }), + Pkt::Data(payload) => self.dispatch(text_of(payload)?), + } + } + + fn dispatch(&mut self, command: String) -> Result { + let (verb, rest) = command + .split_once(' ') + .map_or((command.as_str(), ""), |(verb, rest)| (verb, rest)); + match verb { + "version" => self.handle_version(rest), + "batch" => self.handle_batch(), + "put-object" => self.handle_put(rest), + "verify-object" => self.handle_verify(rest), + "get-object" => self.handle_get(rest), + "quit" => { + self.drain_message()?; + self.respond_ok(&[])?; + Ok(Flow::Quit) + } + _ => { + self.drain_message()?; + let message = self + .messages + .unknown_command + .line(|CommandKey::Command| format!("{verb:?}")); + self.respond_error(HttpStatus::new(400), &message)?; + Ok(Flow::Continue) + } + } + } + + fn read_args(&mut self) -> Result<(Vec, Ended), LfsError> { + let mut args = Vec::new(); + std::iter::from_fn(|| Some(next_pkt(&mut self.lines))) + .find_map(|pkt| match pkt { + Err(fault) => Some(Err(fault)), + Ok(Pkt::Data(payload)) => match text_of(payload) { + Ok(_) if args.len() >= MAX_MESSAGE_ARGS => Some(Err(LfsError::TooMany { + what: "arguments", + limit: MAX_MESSAGE_ARGS, + })), + Ok(line) => { + args.push(line); + None + } + Err(fault) => Some(Err(fault)), + }, + Ok(Pkt::Flush) => Some(Ok(Ended::Flush)), + Ok(Pkt::Delim) => Some(Ok(Ended::Delim)), + Ok(Pkt::Eof) => Some(Err(LfsError::Framing { + detail: "message truncated before flush".to_string(), + })), + }) + .expect("an endless packet iterator always yields a terminator") + .map(|ended| (args, ended)) + } + + fn drain_budget(&self) -> LfsSize { + self.admission.max_object() + } + + fn drain_to_flush(&mut self, limit: LfsSize) -> Result<(), LfsError> { + let mut discarded = LfsSize::new(0); + std::iter::from_fn(|| Some(next_pkt(&mut self.lines))) + .find_map(|pkt| match pkt { + Err(fault) => Some(Err(fault)), + Ok(Pkt::Flush) => Some(Ok(())), + Ok(Pkt::Eof) => Some(Err(LfsError::Framing { + detail: "message truncated before flush".to_string(), + })), + Ok(Pkt::Data(payload)) => { + discarded = discarded.saturating_add(LfsSize::new(payload.len() as u64)); + match discarded > limit { + true => Some(Err(LfsError::Framing { + detail: "message body exceeds the drain bound".to_string(), + })), + false => None, + } + } + Ok(Pkt::Delim) => None, + }) + .expect("an endless packet iterator always yields a terminator") + } + + fn drain_message(&mut self) -> Result<(), LfsError> { + match self.read_args()?.1 { + Ended::Flush => Ok(()), + Ended::Delim => self.drain_to_flush(self.drain_budget()), + } + } + + fn respond_ok(&mut self, args: &[String]) -> Result<(), LfsError> { + write_text(&mut self.out, "status 200")?; + args.iter() + .try_for_each(|arg| write_text(&mut self.out, arg))?; + write_flush(&mut self.out) + } + + fn respond_error(&mut self, code: HttpStatus, message: &str) -> Result<(), LfsError> { + write_text(&mut self.out, &format!("status {:03}", code.get()))?; + write_delim(&mut self.out)?; + write_text(&mut self.out, &format!("error: {message}"))?; + write_flush(&mut self.out) + } + + fn respond_fault(&mut self, fault: &LfsError) -> Result<(), LfsError> { + let message = match fault { + LfsError::Io { .. } => { + tracing::warn!(repo = self.repo.as_str(), %fault, "lfs store fault"); + "internal storage fault".to_string() + } + other => fault_text(other, self.messages), + }; + self.respond_error(status_of(fault), &message) + } + + fn handle_version(&mut self, rest: &str) -> Result { + self.drain_message()?; + match rest.trim() { + "1" => self.respond_ok(&[])?, + other => { + let message = self + .messages + .unsupported_version + .line(|VersionKey::Version| format!("{other:?}")); + self.respond_error(HttpStatus::new(400), &message)?; + } + } + Ok(Flow::Continue) + } + + fn handle_batch(&mut self) -> Result { + let (args, ended) = self.read_args()?; + if let Some(algo) = arg_value(&args, "hash-algo") + && algo != crate::HASH_ALGO + { + if matches!(ended, Ended::Delim) { + self.drain_to_flush(self.drain_budget())?; + } + let message = self + .messages + .unsupported_hash + .line(|AlgorithmKey::Algorithm| format!("{algo:?}")); + self.respond_error(HttpStatus::new(400), &message)?; + return Ok(Flow::Continue); + } + let items = match ended { + Ended::Flush => Ok(Vec::new()), + Ended::Delim => self.read_batch_items(), + }; + let items = match items { + Ok(items) => items, + Err(fault @ (LfsError::InvalidOid { .. } | LfsError::Framing { .. })) => { + self.drain_to_flush(self.drain_budget())?; + self.respond_fault(&fault)?; + return Ok(Flow::Continue); + } + Err(fault) => return Err(fault), + }; + let lines: Result, LfsError> = + items.iter().map(|item| self.batch_line(item)).collect(); + match lines { + Ok(lines) => { + write_text(&mut self.out, "status 200")?; + write_text(&mut self.out, &format!("hash-algo={}", crate::HASH_ALGO))?; + write_delim(&mut self.out)?; + lines + .iter() + .try_for_each(|line| write_text(&mut self.out, line))?; + write_flush(&mut self.out)?; + } + Err(fault) => self.respond_fault(&fault)?, + } + Ok(Flow::Continue) + } + + fn read_batch_items(&mut self) -> Result, LfsError> { + let mut items = Vec::new(); + std::iter::from_fn(|| Some(next_pkt(&mut self.lines))) + .find_map(|pkt| match pkt { + Err(fault) => Some(Err(fault)), + Ok(Pkt::Data(payload)) => match text_of(payload).and_then(parse_batch_item) { + Ok(_) if items.len() >= MAX_BATCH_OBJECTS => Some(Err(LfsError::TooMany { + what: "batch items", + limit: MAX_BATCH_OBJECTS, + })), + Ok(item) => { + items.push(item); + None + } + Err(fault) => Some(Err(fault)), + }, + Ok(Pkt::Flush) => Some(Ok(())), + Ok(Pkt::Delim | Pkt::Eof) => Some(Err(LfsError::Framing { + detail: "batch items truncated before flush".to_string(), + })), + }) + .expect("an endless packet iterator always yields a terminator") + .map(|()| items) + } + + fn batch_line(&self, item: &BatchObject) -> Result { + let stored = match self.op { + TransferOp::Upload => self.store.touch(self.repo, &item.oid)?, + TransferOp::Download => self.store.probe(self.repo, &item.oid)?, + }; + let (size, action) = match (self.op, stored) { + (TransferOp::Upload, Some(_)) => (item.size.get(), "noop"), + (TransferOp::Upload, None) => (item.size.get(), "upload"), + (TransferOp::Download, Some(actual)) => (actual.get(), "download"), + (TransferOp::Download, None) => (item.size.get(), "download"), + }; + Ok(format!("{} {} {}", item.oid, size, action)) + } + + fn handle_put(&mut self, rest: &str) -> Result { + if self.op != TransferOp::Upload { + self.drain_message()?; + self.respond_error(HttpStatus::new(403), &self.messages.put_on_download.text())?; + return Ok(Flow::Continue); + } + let (args, ended) = self.read_args()?; + let checked = LfsOid::new(rest).and_then(|oid| { + let declared = arg_value(&args, "size") + .and_then(|value| value.parse().ok()) + .map(ClaimedSize::new) + .ok_or_else(|| LfsError::Framing { + detail: "put-object requires a size argument".to_string(), + })?; + let permit = self.admission.admit(declared)?; + Ok((oid, declared, permit)) + }); + let (oid, declared, permit) = match checked { + Ok(admitted) => admitted, + Err(fault) => { + if matches!(ended, Ended::Delim) { + self.drain_to_flush(self.drain_budget())?; + } + self.respond_fault(&fault)?; + return Ok(Flow::Continue); + } + }; + if matches!(ended, Ended::Flush) { + self.respond_error(HttpStatus::new(400), &self.messages.put_no_body.text())?; + return Ok(Flow::Continue); + } + let mut body = PktBody { + lines: &mut self.lines, + buffer: Vec::new(), + offset: 0, + done: false, + }; + let stored = self.store.put(self.repo, &oid, declared, &mut body); + drop(permit); + let synced = body.done; + if !synced { + self.drain_to_flush(self.drain_budget())?; + } + match stored { + Ok(()) => { + tracing::info!( + repo = self.repo.as_str(), + oid = oid.as_str(), + size = declared.get(), + "lfs object received over ssh" + ); + self.respond_ok(&[])? + } + Err(fault) => self.respond_fault(&fault)?, + } + Ok(Flow::Continue) + } + + fn handle_verify(&mut self, rest: &str) -> Result { + if self.op != TransferOp::Upload { + self.drain_message()?; + self.respond_error( + HttpStatus::new(403), + &self.messages.verify_on_download.text(), + )?; + return Ok(Flow::Continue); + } + let (args, ended) = self.read_args()?; + if matches!(ended, Ended::Delim) { + self.drain_to_flush(self.drain_budget())?; + } + let declared = arg_value(&args, "size") + .and_then(|value| value.parse().ok()) + .map(ClaimedSize::new); + let verdict = LfsOid::new(rest).and_then(|oid| { + match (self.store.touch(self.repo, &oid)?, declared) { + (None, _) => Err(LfsError::NotFound { oid }), + (Some(actual), Some(declared)) if !declared.matches(actual) => { + Err(LfsError::SizeMismatch { + declared, + received: actual, + }) + } + (Some(_), _) => Ok(()), + } + }); + match verdict { + Ok(()) => self.respond_ok(&[])?, + Err(fault) => self.respond_fault(&fault)?, + } + Ok(Flow::Continue) + } + + fn handle_get(&mut self, rest: &str) -> Result { + if self.op != TransferOp::Download { + self.drain_message()?; + self.respond_error(HttpStatus::new(403), &self.messages.get_on_upload.text())?; + return Ok(Flow::Continue); + } + self.drain_message()?; + let opened = LfsOid::new(rest).and_then(|oid| { + let size = self + .store + .probe(self.repo, &oid)? + .ok_or(LfsError::NotFound { oid: oid.clone() })?; + let body = self.store.read(self.repo, &oid)?; + Ok((size, body)) + }); + let (size, mut body) = match opened { + Ok(found) => found, + Err(fault) => { + self.respond_fault(&fault)?; + return Ok(Flow::Continue); + } + }; + write_text(&mut self.out, "status 200")?; + write_text(&mut self.out, &format!("size={size}"))?; + write_delim(&mut self.out)?; + let out = &mut self.out; + for_each_chunk(&mut body, |chunk| { + chunk.chunks(PKT_DATA_MAX).try_for_each(|piece| { + encode::data_to_write(piece, &mut *out) + .map(|_| ()) + .map_err(|source| LfsError::Channel { source }) + }) + })?; + write_flush(&mut self.out)?; + tracing::info!( + repo = self.repo.as_str(), + oid = rest, + size = size.get(), + "lfs object served over ssh" + ); + Ok(Flow::Continue) + } +} + +fn parse_batch_item(line: String) -> Result { + let mut tokens = line.split(' '); + let oid = LfsOid::new(tokens.next().unwrap_or_default())?; + let size = tokens + .next() + .and_then(|token| token.parse().ok()) + .map(ClaimedSize::new) + .ok_or_else(|| LfsError::Framing { + detail: format!("malformed batch item {line:?}"), + })?; + Ok(BatchObject { oid, size }) +} + +struct PktBody<'a, R: Read> { + lines: &'a mut StreamingPeekableIter, + buffer: Vec, + offset: usize, + done: bool, +} + +impl Read for PktBody<'_, R> { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + if self.done { + return Ok(0); + } + if self.offset >= self.buffer.len() { + match next_pkt(self.lines).map_err(std::io::Error::other)? { + Pkt::Data(payload) => { + self.buffer = payload; + self.offset = 0; + } + Pkt::Flush => { + self.done = true; + return Ok(0); + } + Pkt::Delim => { + return Err(std::io::Error::other("unexpected delimiter in object body")); + } + Pkt::Eof => return Err(std::io::Error::other("object body truncated")), + } + } + let take = out.len().min(self.buffer.len() - self.offset); + out[..take].copy_from_slice(&self.buffer[self.offset..self.offset + take]); + self.offset += take; + Ok(take) + } +} + +#[cfg(test)] +mod tests { + use sha2::{Digest, Sha256}; + + use super::*; + use crate::admission::Unbounded; + use crate::{FreeSpaceFloor, MemoryStore}; + + fn oid_of(bytes: &[u8]) -> LfsOid { + LfsOid::from_digest(Sha256::digest(bytes).into()) + } + + fn repo() -> RepoDid { + RepoDid::new("did:plc:squid").unwrap() + } + + fn put_text(buf: &mut Vec, line: &str) { + encode::data_to_write(format!("{line}\n").as_bytes(), &mut *buf).unwrap(); + } + + fn msg(buf: &mut Vec, command: &str, args: &[&str]) { + put_text(buf, command); + args.iter().for_each(|arg| put_text(buf, arg)); + encode::flush_to_write(&mut *buf).unwrap(); + } + + fn msg_lines(buf: &mut Vec, command: &str, args: &[&str], lines: &[String]) { + put_text(buf, command); + args.iter().for_each(|arg| put_text(buf, arg)); + encode::delim_to_write(&mut *buf).unwrap(); + lines.iter().for_each(|line| put_text(buf, line)); + encode::flush_to_write(&mut *buf).unwrap(); + } + + fn msg_data(buf: &mut Vec, command: &str, args: &[&str], data: &[u8]) { + put_text(buf, command); + args.iter().for_each(|arg| put_text(buf, arg)); + encode::delim_to_write(&mut *buf).unwrap(); + data.chunks(PKT_DATA_MAX).for_each(|chunk| { + encode::data_to_write(chunk, &mut *buf).unwrap(); + }); + encode::flush_to_write(&mut *buf).unwrap(); + } + + #[derive(Debug, PartialEq, Eq, Clone)] + enum Out { + Line(String), + Bin(Vec), + Delim, + Flush, + } + + fn parse_out(bytes: &[u8]) -> Vec { + let mut lines = StreamingPeekableIter::new(bytes, &[], false); + std::iter::from_fn(|| { + lines.read_line().and_then(|pkt| { + if matches!(&pkt, Err(fault) if fault.kind() == std::io::ErrorKind::UnexpectedEof) { + return None; + } + Some( + match pkt.expect("readable output").expect("well-formed output") { + PacketLineRef::Data(payload) => std::str::from_utf8(payload) + .ok() + .filter(|text| text.ends_with('\n')) + .map(|text| Out::Line(text.trim_end_matches('\n').to_string())) + .unwrap_or_else(|| Out::Bin(payload.to_vec())), + PacketLineRef::Flush => Out::Flush, + PacketLineRef::Delimiter => Out::Delim, + PacketLineRef::ResponseEnd => panic!("server never writes response-end"), + }, + ) + }) + }) + .collect() + } + + fn line(text: &str) -> Out { + Out::Line(text.to_string()) + } + + fn run(op: TransferOp, store: &dyn LfsStore, script: &[u8]) -> Vec { + let mut output = Vec::new(); + serve_transfer( + store, + &Unbounded, + &repo(), + op, + &knot_messages::default_catalog().lfs, + script, + &mut output, + ) + .unwrap(); + parse_out(&output) + } + + fn responses(out: &[Out]) -> Vec> { + out.split_inclusive(|item| *item == Out::Flush) + .map(<[Out]>::to_vec) + .collect() + } + + #[test] + fn an_upload_session_lands_and_verifies_an_object() { + let store = MemoryStore::new(); + let seeded: &[u8] = b"already on the server"; + let seeded_oid = oid_of(seeded); + store + .put( + &repo(), + &seeded_oid, + ClaimedSize::new(seeded.len() as u64), + &mut &seeded[..], + ) + .unwrap(); + + let fresh: &[u8] = b"\xffnew binary media\x00with raw bytes"; + let fresh_oid = oid_of(fresh); + let fresh_len = fresh.len(); + + let mut script = Vec::new(); + msg(&mut script, "version 1", &[]); + msg_lines( + &mut script, + "batch", + &[ + "transfer=ssh", + "hash-algo=sha256", + "refname=refs/heads/main", + ], + &[ + format!("{seeded_oid} {} extra=ignored", seeded.len()), + format!("{fresh_oid} {fresh_len}"), + ], + ); + msg_data( + &mut script, + &format!("put-object {fresh_oid}"), + &[&format!("size={fresh_len}")], + fresh, + ); + msg( + &mut script, + &format!("verify-object {fresh_oid}"), + &[&format!("size={fresh_len}")], + ); + msg(&mut script, "quit", &[]); + + let out = run(TransferOp::Upload, &store, &script); + let turns = responses(&out); + assert_eq!(turns[0], vec![line(CAPABILITY_VERSION), Out::Flush]); + assert_eq!(turns[1], vec![line("status 200"), Out::Flush]); + assert_eq!( + turns[2], + vec![ + line("status 200"), + line("hash-algo=sha256"), + Out::Delim, + line(&format!("{seeded_oid} {} noop", seeded.len())), + line(&format!("{fresh_oid} {fresh_len} upload")), + Out::Flush, + ] + ); + assert_eq!(turns[3], vec![line("status 200"), Out::Flush]); + assert_eq!(turns[4], vec![line("status 200"), Out::Flush]); + assert_eq!(turns[5], vec![line("status 200"), Out::Flush]); + assert_eq!( + store.probe(&repo(), &fresh_oid).unwrap(), + Some(LfsSize::new(fresh_len as u64)) + ); + } + + #[test] + fn a_download_session_streams_bytes_and_404s_the_missing() { + let store = MemoryStore::new(); + let media: &[u8] = b"\xff\x00streamable media"; + let media_oid = oid_of(media); + let absent = oid_of(b"never uploaded"); + store + .put( + &repo(), + &media_oid, + ClaimedSize::new(media.len() as u64), + &mut &media[..], + ) + .unwrap(); + + let mut script = Vec::new(); + msg_lines( + &mut script, + "batch", + &["transfer=ssh", "hash-algo=sha256"], + &[format!("{media_oid} 1"), format!("{absent} 9")], + ); + msg(&mut script, &format!("get-object {media_oid}"), &[]); + msg(&mut script, &format!("get-object {absent}"), &[]); + msg(&mut script, "quit", &[]); + + let out = run(TransferOp::Download, &store, &script); + let turns = responses(&out); + assert_eq!( + turns[1], + vec![ + line("status 200"), + line("hash-algo=sha256"), + Out::Delim, + line(&format!("{media_oid} {} download", media.len())), + line(&format!("{absent} 9 download")), + Out::Flush, + ] + ); + assert_eq!( + turns[2], + vec![ + line("status 200"), + line(&format!("size={}", media.len())), + Out::Delim, + Out::Bin(media.to_vec()), + Out::Flush, + ] + ); + assert_eq!(turns[3][0], line("status 404")); + assert_eq!(turns[4], vec![line("status 200"), Out::Flush]); + } + + #[test] + fn the_channel_mode_gates_every_write_and_read_verb() { + let store = MemoryStore::new(); + let oid = oid_of(b"whatever"); + + let mut download_script = Vec::new(); + msg_data( + &mut download_script, + &format!("put-object {oid}"), + &["size=3"], + b"abc", + ); + msg( + &mut download_script, + &format!("verify-object {oid}"), + &["size=3"], + ); + let out = run(TransferOp::Download, &store, &download_script); + let turns = responses(&out); + assert_eq!(turns[1][0], line("status 403")); + assert_eq!(turns[2][0], line("status 403")); + assert_eq!(store.probe(&repo(), &oid).unwrap(), None); + + let mut upload_script = Vec::new(); + msg(&mut upload_script, &format!("get-object {oid}"), &[]); + let out = run(TransferOp::Upload, &store, &upload_script); + assert_eq!(responses(&out)[1][0], line("status 403")); + } + + #[test] + fn admission_rejections_surface_as_typed_statuses() { + struct Deny(LfsSize); + impl UploadAdmission for Deny { + fn admit(&self, declared: ClaimedSize) -> Result { + match declared.get() > self.0.get() { + true => Err(LfsError::SizeLimitExceeded { + declared, + limit: self.0, + }), + false => Err(LfsError::FreeSpaceDenied { + free: LfsSize::new(1), + floor: FreeSpaceFloor::new(2), + }), + } + } + + fn max_object(&self) -> LfsSize { + self.0 + } + } + let store = MemoryStore::new(); + let body: &[u8] = b"denied"; + let oid = oid_of(body); + let mut script = Vec::new(); + msg_data( + &mut script, + &format!("put-object {oid}"), + &["size=999"], + body, + ); + msg_data(&mut script, &format!("put-object {oid}"), &["size=6"], body); + let mut output = Vec::new(); + serve_transfer( + &store, + &Deny(LfsSize::new(10)), + &repo(), + TransferOp::Upload, + &knot_messages::default_catalog().lfs, + &script[..], + &mut output, + ) + .unwrap(); + let turns = responses(&parse_out(&output)); + assert_eq!(turns[1][0], line("status 413")); + assert_eq!(turns[2][0], line("status 429")); + assert_eq!(store.probe(&repo(), &oid).unwrap(), None); + } + + #[test] + fn tampered_and_truncated_uploads_fail_closed_and_the_session_survives() { + let store = MemoryStore::new(); + let body: &[u8] = b"the true bytes"; + let liar = oid_of(b"some other bytes"); + let valid = oid_of(body); + + let mut script = Vec::new(); + msg_data( + &mut script, + &format!("put-object {liar}"), + &[&format!("size={}", body.len())], + body, + ); + msg_data( + &mut script, + &format!("put-object {valid}"), + &[&format!("size={}", body.len() + 5)], + body, + ); + msg( + &mut script, + &format!("verify-object {valid}"), + &[&format!("size={}", body.len())], + ); + msg_data( + &mut script, + &format!("put-object {valid}"), + &[&format!("size={}", body.len())], + body, + ); + msg(&mut script, "quit", &[]); + + let out = run(TransferOp::Upload, &store, &script); + let turns = responses(&out); + assert_eq!(turns[1][0], line("status 400"), "hash mismatch"); + assert_eq!(turns[2][0], line("status 400"), "size mismatch"); + assert_eq!(turns[3][0], line("status 404"), "nothing landed"); + assert_eq!( + turns[4][0], + line("status 200"), + "retry with matching bytes succeeds" + ); + assert_eq!( + store.probe(&repo(), &valid).unwrap(), + Some(LfsSize::new(body.len() as u64)) + ); + assert_eq!(store.probe(&repo(), &liar).unwrap(), None); + } + + #[test] + fn hostile_commands_get_clean_rejections_and_never_a_panic() { + let store = MemoryStore::new(); + let mut script = Vec::new(); + msg(&mut script, "version 9", &[]); + msg(&mut script, "steal-the-objects now", &[]); + msg_lines( + &mut script, + "batch", + &["hash-algo=sha256"], + &["../../../etc/passwd0000000000000000000000000000000000000000 5".to_string()], + ); + msg_lines( + &mut script, + "batch", + &["hash-algo=sha1"], + &[format!("{} 5", oid_of(b"x"))], + ); + msg(&mut script, "quit", &[]); + + let out = run(TransferOp::Upload, &store, &script); + let turns = responses(&out); + assert_eq!(turns[1][0], line("status 400"), "unsupported version"); + assert_eq!(turns[2][0], line("status 400"), "unknown command"); + assert_eq!(turns[3][0], line("status 400"), "traversal oid"); + assert_eq!(turns[4][0], line("status 400"), "foreign hash algo"); + assert_eq!(turns[5][0], line("status 200"), "quit still answers"); + } + + #[test] + fn floods_kill_the_session_instead_of_accumulating() { + let store = MemoryStore::new(); + let flooded_batch: Vec = (0..MAX_BATCH_OBJECTS + 1) + .map(|index| format!("{} 1", oid_of(index.to_string().as_bytes()))) + .collect(); + let mut script = Vec::new(); + msg_lines(&mut script, "batch", &["hash-algo=sha256"], &flooded_batch); + let mut output = Vec::new(); + let verdict = serve_transfer( + &store, + &Unbounded, + &repo(), + TransferOp::Upload, + &knot_messages::default_catalog().lfs, + &script[..], + &mut output, + ); + assert!(matches!( + verdict, + Err(LfsError::TooMany { + what: "batch items", + .. + }) + )); + + let flooded_args: Vec<&str> = std::iter::repeat_n("size=1", MAX_MESSAGE_ARGS + 1).collect(); + let mut script = Vec::new(); + msg( + &mut script, + &format!("put-object {}", oid_of(b"flooded")), + &flooded_args, + ); + let mut output = Vec::new(); + let verdict = serve_transfer( + &store, + &Unbounded, + &repo(), + TransferOp::Upload, + &knot_messages::default_catalog().lfs, + &script[..], + &mut output, + ); + assert!(matches!( + verdict, + Err(LfsError::TooMany { + what: "arguments", + .. + }) + )); + } + + #[test] + fn malformed_and_oversized_put_streams_end_the_session() { + let store = MemoryStore::new(); + + let mut sink = Vec::new(); + let garbage = serve_transfer( + &store, + &Unbounded, + &repo(), + TransferOp::Upload, + &knot_messages::default_catalog().lfs, + &b"zzzz not pkt-line at all"[..], + &mut sink, + ); + assert!( + matches!(garbage, Err(LfsError::Framing { .. })), + "raw garbage on the wire is a framing fault, not a hang" + ); + + let admission = crate::StoreAdmission::new( + crate::LfsStorePath::new("/"), + LfsSize::new(10), + FreeSpaceFloor::new(0), + ); + let oid = oid_of(b"whatever"); + let mut script = Vec::new(); + put_text(&mut script, &format!("put-object {oid}")); + put_text(&mut script, "size=1"); + encode::delim_to_write(&mut script).unwrap(); + (0..3).for_each(|_| { + encode::data_to_write(&[0u8; 100][..], &mut script).unwrap(); + }); + encode::flush_to_write(&mut script).unwrap(); + let mut output = Vec::new(); + let overshoot = serve_transfer( + &store, + &admission, + &repo(), + TransferOp::Upload, + &knot_messages::default_catalog().lfs, + &script[..], + &mut output, + ); + assert!( + matches!(overshoot, Err(LfsError::Framing { .. })), + "a body overshooting the object size limit ends the session instead of draining unbounded bytes" + ); + assert_eq!(store.probe(&repo(), &oid).unwrap(), None); + } + + #[test] + fn store_faults_reach_the_client_without_the_path() { + struct BrokenStore; + impl LfsStore for BrokenStore { + fn put( + &self, + _repo: &RepoDid, + _oid: &LfsOid, + _size: ClaimedSize, + _body: &mut dyn std::io::Read, + ) -> Result<(), LfsError> { + // for now!!!! + unreachable!("this session never puts") + } + + fn read( + &self, + _repo: &RepoDid, + _oid: &LfsOid, + ) -> Result, LfsError> { + unreachable!("this session never reads") + } + + fn probe(&self, _repo: &RepoDid, _oid: &LfsOid) -> Result, LfsError> { + Err(LfsError::Io { + op: "stat", + path: "/srv/secret-lfs-root/plc/sq/uid".into(), + source: std::io::Error::other("disk fell off"), + }) + } + + fn touch(&self, repo: &RepoDid, oid: &LfsOid) -> Result, LfsError> { + self.probe(repo, oid) + } + } + + let mut script = Vec::new(); + msg_lines( + &mut script, + "batch", + &["hash-algo=sha256"], + &[format!("{} 5", oid_of(b"whatever"))], + ); + let out = run(TransferOp::Upload, &BrokenStore, &script); + let turns = responses(&out); + assert_eq!(turns[1][0], line("status 500")); + assert!(turns[1].contains(&line("error: internal storage fault"))); + turns[1].iter().for_each(|item| { + if let Out::Line(text) = item { + assert!(!text.contains("secret-lfs-root"), "leaked path in {text:?}"); + } + }); + } + + #[test] + fn a_large_object_round_trips_across_many_packets() { + let store = MemoryStore::new(); + let media: Vec = (0..500_000u32).map(|n| (n % 251) as u8).collect(); + let oid = oid_of(&media); + + let mut script = Vec::new(); + msg_data( + &mut script, + &format!("put-object {oid}"), + &[&format!("size={}", media.len())], + &media, + ); + let out = run(TransferOp::Upload, &store, &script); + assert_eq!(responses(&out)[1][0], line("status 200")); + + let mut fetch = Vec::new(); + msg(&mut fetch, &format!("get-object {oid}"), &[]); + let out = run(TransferOp::Download, &store, &fetch); + let body: Vec = out + .iter() + .skip_while(|item| **item != Out::Delim) + .filter_map(|item| match item { + Out::Bin(chunk) => Some(chunk.clone()), + Out::Line(text) => Some(format!("{text}\n").into_bytes()), + _ => None, + }) + .flatten() + .collect(); + assert_eq!(body, media); + } +} diff --git a/knot2/crates/knot-lfs/src/types.rs b/knot2/crates/knot-lfs/src/types.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-lfs/src/types.rs @@ -0,0 +1,264 @@ +use std::fmt; +use std::path::{Path, PathBuf}; +use std::str::FromStr; + +use knot_types::RepoDid; + +use crate::LfsError; + +const OID_HEX_LEN: usize = 64; + +#[derive( + Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize, serde::Deserialize, +)] +#[serde(try_from = "String", into = "String")] +pub struct LfsOid(String); + +impl LfsOid { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + let valid = value.len() == OID_HEX_LEN + && value + .bytes() + .all(|byte| matches!(byte, b'0'..=b'9' | b'a'..=b'f')); + match valid { + true => Ok(Self(value)), + false => Err(LfsError::InvalidOid { value }), + } + } + + pub fn from_digest(digest: [u8; 32]) -> Self { + Self(knot_types::lowercase_hex(&digest)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl FromStr for LfsOid { + type Err = LfsError; + + fn from_str(s: &str) -> Result { + Self::new(s) + } +} + +impl TryFrom for LfsOid { + type Error = LfsError; + + fn try_from(value: String) -> Result { + Self::new(value) + } +} + +impl From for String { + fn from(oid: LfsOid) -> Self { + oid.0 + } +} + +impl fmt::Display for LfsOid { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl AsRef for LfsOid { + fn as_ref(&self) -> &str { + &self.0 + } +} + +#[derive( + Debug, + Clone, + Copy, + Default, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + serde::Serialize, + serde::Deserialize, +)] +#[serde(transparent)] +pub struct LfsSize(u64); + +impl LfsSize { + pub const fn new(bytes: u64) -> Self { + Self(bytes) + } + + pub const fn get(self) -> u64 { + self.0 + } + + pub const fn saturating_add(self, other: LfsSize) -> LfsSize { + LfsSize(self.0.saturating_add(other.0)) + } +} + +impl fmt::Display for LfsSize { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +#[derive( + Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize, serde::Deserialize, +)] +#[serde(transparent)] +// The mass that the client says the incoming object has. +// Admission will reserve disk space +// using this before the obj bytes arrive, +// and the count of what actually showed up in the end is `LfsSize`. +pub struct ClaimedSize(u64); + +impl ClaimedSize { + pub const fn new(bytes: u64) -> Self { + Self(bytes) + } + + pub const fn get(self) -> u64 { + self.0 + } + + pub const fn matches(self, actual: LfsSize) -> bool { + self.0 == actual.get() + } +} + +impl fmt::Display for ClaimedSize { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct FreeSpaceFloor(u64); + +impl FreeSpaceFloor { + pub const fn new(bytes: u64) -> Self { + Self(bytes) + } + + pub const fn get(self) -> u64 { + self.0 + } +} + +impl fmt::Display for FreeSpaceFloor { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct RepoPrefix(PathBuf); + +impl RepoPrefix { + pub fn new(repo: &RepoDid) -> Result { + knot_git::repo_shard(repo) + .map(Self) + .map_err(|_| LfsError::UnsafeRepoDid { + did: repo.as_str().to_string(), + }) + } + + pub fn as_path(&self) -> &Path { + &self.0 + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct ObjectRelPath(PathBuf); + +impl ObjectRelPath { + pub fn new(repo: &RepoDid, oid: &LfsOid) -> Result { + let prefix = RepoPrefix::new(repo)?; + let hex = oid.as_str(); + Ok(Self(prefix.0.join(&hex[0..2]).join(&hex[2..4]).join(hex))) + } + + pub fn as_path(&self) -> &Path { + &self.0 + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LfsStorePath(PathBuf); + +impl LfsStorePath { + pub fn new(root: impl Into) -> Self { + Self(root.into()) + } + + pub fn as_path(&self) -> &Path { + &self.0 + } + + pub fn object_path(&self, rel: &ObjectRelPath) -> PathBuf { + self.0.join(rel.as_path()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const SAMPLE_OID: &str = "6c17f2007cbe934aee6e309b28b2fba3c119d98be6ea4156da3aa3173456ad16"; + + #[test] + fn oid_accepts_lowercase_hex_and_round_trips_a_digest() { + let oid = LfsOid::new(SAMPLE_OID).unwrap(); + assert_eq!(oid.as_str(), SAMPLE_OID); + assert_eq!(oid.to_string(), SAMPLE_OID); + assert_eq!(SAMPLE_OID.parse::().unwrap(), oid); + + let from_digest = LfsOid::from_digest([0xab; 32]); + assert_eq!(from_digest.as_str().len(), 64); + assert_eq!(LfsOid::new(from_digest.as_str()).unwrap(), from_digest); + } + + #[test] + fn oid_rejects_everything_else() { + let hostile = [ + "", + "abc123", + &SAMPLE_OID[..63], + &format!("{SAMPLE_OID}0"), + &SAMPLE_OID.to_uppercase(), + &format!("{}g", &SAMPLE_OID[..63]), + "../../../../../../etc/passwd0000000000000000000000000000000000000", + "..%2f..%2f..%2f..%2f..%2fetc%2fpasswd0000000000000000000000000000", + &format!("{}\0", &SAMPLE_OID[..63]), + ]; + hostile.iter().for_each(|value| { + assert!( + matches!(LfsOid::new(*value), Err(LfsError::InvalidOid { .. })), + "accepted {value:?}" + ); + }); + } + + #[test] + fn object_paths_shard_by_repo_then_oid_and_stay_under_the_root() { + let repo = RepoDid::new("did:plc:squid").unwrap(); + let oid = LfsOid::new(SAMPLE_OID).unwrap(); + let rel = ObjectRelPath::new(&repo, &oid).unwrap(); + assert_eq!( + rel.as_path(), + Path::new("plc/sq/uid/6c/17").join(SAMPLE_OID) + ); + + let root = LfsStorePath::new("/srv/lfs"); + let path = root.object_path(&rel); + assert!(path.starts_with(root.as_path())); + assert!( + path.components() + .all(|part| !matches!(part, std::path::Component::ParentDir)) + ); + } +} diff --git a/knot2/crates/knot-lfs/tests/chaos.rs b/knot2/crates/knot-lfs/tests/chaos.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-lfs/tests/chaos.rs @@ -0,0 +1,315 @@ +mod common; + +use std::path::Path; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant, SystemTime}; + +use common::{backdate, incompressible, object_path, oid_of, pointer_blob}; +use knot_git::{EntryKind, Identity, Layout, NewCommit, RefUpdate, StagedAction, StagedChange}; +use knot_lfs::{ClaimedSize, DiskStore, LfsOid, LfsSize, LfsStore, LfsStorePath, collect_repo}; +use knot_types::{AuthorName, BranchName, Email, Oid, RefName, RepoDid, UnixSeconds}; + +const DID: &str = "did:plc:squid"; +const EMPTY_TREE: &str = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"; +const PUT_BYTES: usize = 48 * 1024 * 1024; +const REFERENCED: usize = 24; +const UNREFERENCED: usize = 320; +const GRACE: Duration = Duration::from_secs(86_400); +const BACKDATE: Duration = Duration::from_secs(60 * 86_400); + +fn did() -> RepoDid { + RepoDid::new(DID).unwrap() +} + +fn spawn_worker(role: &str, envs: &[(&str, &Path)]) -> std::process::Child { + let mut command = Command::new(std::env::current_exe().unwrap()); + command + .args(["--exact", &format!("chaos_{role}_worker"), "--nocapture"]) + .env("KNOT_CHAOS_ROLE", role) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + envs.iter().for_each(|(key, value)| { + command.env(key, value); + }); + command.spawn().expect("spawn chaos worker") +} + +fn kill_after(mut child: std::process::Child, delay: Duration) { + std::thread::sleep(delay); + let _ = child.kill(); + child.wait().unwrap(); +} + +fn delays(full: Duration) -> Vec { + let fractions = [0.20, 0.35, 0.45, 0.55, 0.65, 0.75, 0.85, 0.95]; + std::iter::once(Duration::from_millis(1)) + .chain(std::iter::once(Duration::from_millis(3))) + .chain(fractions.iter().map(|fraction| full.mul_f64(*fraction))) + .chain(std::iter::once(full.mul_f64(2.0))) + .collect() +} + +#[test] +fn chaos_put_worker() { + if std::env::var("KNOT_CHAOS_ROLE").as_deref() != Ok("put") { + return; + } + let store_dir = std::env::var("KNOT_CHAOS_STORE").unwrap(); + let store = DiskStore::open(LfsStorePath::new(&store_dir)).unwrap(); + let body = incompressible(PUT_BYTES, 0x2545_f491_4f6c_dd1d); + let oid = oid_of(&body); + let _ = store.put( + &did(), + &oid, + ClaimedSize::new(body.len() as u64), + &mut &body[..], + ); +} + +#[test] +fn kill9_during_put_object_never_leaves_a_torn_object() { + let body = incompressible(PUT_BYTES, 0x2545_f491_4f6c_dd1d); + let oid = oid_of(&body); + let scratch = tempfile::tempdir().unwrap(); + + let warm = scratch.path().join("warm"); + std::fs::create_dir_all(&warm).unwrap(); + let started = Instant::now(); + spawn_worker("put", &[("KNOT_CHAOS_STORE", &warm)]) + .wait() + .unwrap(); + let full = started.elapsed(); + assert!( + object_path(&warm, &oid).is_file(), + "an uninterrupted put stores the object" + ); + + let landed: Vec = delays(full) + .iter() + .enumerate() + .map(|(trial, delay)| { + let store_dir = scratch.path().join(format!("store-{trial}")); + std::fs::create_dir_all(&store_dir).unwrap(); + kill_after( + spawn_worker("put", &[("KNOT_CHAOS_STORE", &store_dir)]), + *delay, + ); + + let final_path = object_path(&store_dir, &oid); + let present = final_path.is_file(); + if present { + let bytes = std::fs::read(&final_path).unwrap(); + assert_eq!( + bytes.len(), + PUT_BYTES, + "trial {trial}: a visible object is never truncated" + ); + assert_eq!( + oid_of(&bytes), + oid, + "trial {trial}: a visible object always hashes to its oid" + ); + } + + let store = DiskStore::open(LfsStorePath::new(&store_dir)).unwrap(); + let incoming: Vec<_> = std::fs::read_dir(store_dir.join(".incoming")) + .unwrap() + .collect(); + assert!( + incoming.is_empty(), + "trial {trial}: boot sweep clears abandoned uploads, found {incoming:?}" + ); + assert_eq!( + store.probe(&did(), &oid).unwrap().is_some(), + present, + "trial {trial}: the boot sweep never deletes a stored object" + ); + present + }) + .collect(); + + assert!( + landed.iter().any(|present| !present), + "some trial must be killed before the rename, or the kill delays are all too long" + ); + assert!( + landed.iter().any(|present| *present), + "some trial must complete, or the kill delays are all too short" + ); +} + +fn identity() -> Identity { + Identity { + name: AuthorName::new("nel"), + email: Email::new("nel@oyster.cafe"), + time: UnixSeconds::new(1_700_000_000), + offset_seconds: 0, + } +} + +struct GcFixture { + referenced: Vec<(LfsOid, Vec)>, + unreferenced: Vec, +} + +fn build_gc_fixture(scan: &Path, store_dir: &Path) -> GcFixture { + let layout = Layout::new(scan).with_default_branch(BranchName::new("main").unwrap()); + let repo = layout.create(&did()).unwrap(); + let store = DiskStore::open(LfsStorePath::new(store_dir)).unwrap(); + + let referenced: Vec<(LfsOid, Vec)> = (0..REFERENCED) + .map(|index| { + let body = incompressible(2048, 0x9e37_79b9_7f4a_7c15 ^ index as u64); + let oid = oid_of(&body); + store + .put( + &did(), + &oid, + ClaimedSize::new(body.len() as u64), + &mut &body[..], + ) + .unwrap(); + backdate(&object_path(store_dir, &oid), BACKDATE); + (oid, body) + }) + .collect(); + + let unreferenced: Vec = (0..UNREFERENCED) + .map(|index| { + let body = incompressible(512, 0xdead_beef_cafe_f00d ^ index as u64); + let oid = oid_of(&body); + store + .put( + &did(), + &oid, + ClaimedSize::new(body.len() as u64), + &mut &body[..], + ) + .unwrap(); + backdate(&object_path(store_dir, &oid), BACKDATE); + oid + }) + .collect(); + + let changes: Vec = referenced + .iter() + .enumerate() + .map(|(index, (oid, body))| StagedChange { + path: knot_types::RepoPath::new(format!("media/clip{index}.bin")).unwrap(), + action: StagedAction::Put { + content: pointer_blob(oid, LfsSize::new(body.len() as u64)), + kind: EntryKind::Blob, + }, + }) + .collect(); + let empty = Oid::from_hex(EMPTY_TREE).unwrap(); + let tree = repo.write_staged_tree(empty, &changes).unwrap(); + let tip = repo + .write_commit(&NewCommit { + tree, + parents: Vec::new(), + author: identity(), + committer: identity(), + message: "add media".to_string(), + extra_headers: Vec::new(), + }) + .unwrap(); + repo.update_ref(&RefUpdate::Create { + name: RefName::new("refs/heads/main").unwrap(), + new: tip, + }) + .unwrap(); + + GcFixture { + referenced, + unreferenced, + } +} + +fn run_gc(scan: &Path, store_dir: &Path) { + let layout = Layout::new(scan); + let repo = layout.open(&did()).unwrap(); + let store = DiskStore::open(LfsStorePath::new(store_dir)).unwrap(); + let _ = collect_repo(&store, &repo, &did(), GRACE, SystemTime::now()); +} + +#[test] +fn chaos_gc_worker() { + if std::env::var("KNOT_CHAOS_ROLE").as_deref() != Ok("gc") { + return; + } + let scan = std::env::var("KNOT_CHAOS_SCAN").unwrap(); + let store_dir = std::env::var("KNOT_CHAOS_STORE").unwrap(); + run_gc(Path::new(&scan), Path::new(&store_dir)); +} + +#[test] +fn kill9_during_gc_never_loses_a_referenced_object() { + let scratch = tempfile::tempdir().unwrap(); + + let warm_scan = scratch.path().join("warm-scan"); + let warm_store = scratch.path().join("warm-store"); + let warm_fixture = build_gc_fixture(&warm_scan, &warm_store); + let started = Instant::now(); + spawn_worker( + "gc", + &[ + ("KNOT_CHAOS_SCAN", &warm_scan), + ("KNOT_CHAOS_STORE", &warm_store), + ], + ) + .wait() + .unwrap(); + let full = started.elapsed(); + let warm_disk = DiskStore::open(LfsStorePath::new(&warm_store)).unwrap(); + warm_fixture.referenced.iter().for_each(|(oid, _)| { + assert!( + warm_disk.probe(&did(), oid).unwrap().is_some(), + "an uninterrupted gc keeps every referenced object" + ); + }); + warm_fixture.unreferenced.iter().for_each(|oid| { + assert_eq!( + warm_disk.probe(&did(), oid).unwrap(), + None, + "an uninterrupted gc reclaims every expired orphan" + ); + }); + + delays(full).iter().enumerate().for_each(|(trial, delay)| { + let scan = scratch.path().join(format!("scan-{trial}")); + let store_dir = scratch.path().join(format!("store-{trial}")); + let fixture = build_gc_fixture(&scan, &store_dir); + kill_after( + spawn_worker( + "gc", + &[("KNOT_CHAOS_SCAN", &scan), ("KNOT_CHAOS_STORE", &store_dir)], + ), + *delay, + ); + + let store = DiskStore::open(LfsStorePath::new(&store_dir)).unwrap(); + fixture.referenced.iter().for_each(|(oid, body)| { + assert_eq!( + store.probe(&did(), oid).unwrap(), + Some(LfsSize::new(body.len() as u64)), + "trial {trial}: a referenced object remains stored after a killed sweep" + ); + }); + + run_gc(&scan, &store_dir); + fixture.referenced.iter().for_each(|(oid, _)| { + assert!( + store.probe(&did(), oid).unwrap().is_some(), + "trial {trial}: a referenced object remains stored after the self-heal pass" + ); + }); + fixture.unreferenced.iter().for_each(|oid| { + assert_eq!( + store.probe(&did(), oid).unwrap(), + None, + "trial {trial}: the self-heal pass finishes the interrupted reclaim" + ); + }); + }); +} diff --git a/knot2/crates/knot-lfs/tests/fuzz_smoke.rs b/knot2/crates/knot-lfs/tests/fuzz_smoke.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-lfs/tests/fuzz_smoke.rs @@ -0,0 +1,20 @@ +use proptest::prelude::*; + +proptest! { + #![proptest_config(ProptestConfig::with_cases(1024))] + + #[test] + fn the_transfer_engine_never_panics(data in proptest::collection::vec(any::(), 0..4096)) { + knot_lfs::fuzz::transfer(&data); + } + + #[test] + fn the_batch_json_parser_never_panics(data in proptest::collection::vec(any::(), 0..4096)) { + knot_lfs::fuzz::batch(&data); + } + + #[test] + fn the_pointer_parser_never_panics(data in proptest::collection::vec(any::(), 0..4096)) { + knot_lfs::fuzz::pointer(&data); + } +} diff --git a/knot2/crates/knot-lfs/tests/properties.rs b/knot2/crates/knot-lfs/tests/properties.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-lfs/tests/properties.rs @@ -0,0 +1,117 @@ +mod common; + +use common::{oid_of, repo}; +use knot_lfs::{ + ClaimedSize, DiskStore, LfsError, LfsOid, LfsSize, LfsStore, LfsStorePath, MemoryStore, +}; +use proptest::prelude::*; + +fn round_trip(store: &dyn LfsStore, body: &[u8]) -> Result<(), TestCaseError> { + let repo = repo(); + let oid = oid_of(body); + let bytes = body.len() as u64; + store + .put(&repo, &oid, ClaimedSize::new(bytes), &mut &body[..]) + .expect("put with matching size and oid succeeds"); + prop_assert_eq!( + store.probe(&repo, &oid).expect("probe"), + Some(LfsSize::new(bytes)) + ); + let mut out = Vec::new(); + store + .read(&repo, &oid) + .expect("stored object opens") + .read_to_end(&mut out) + .expect("stored object reads"); + prop_assert_eq!(out, body); + Ok(()) +} + +#[derive(Debug, Clone)] +enum Tamper { + Flip { at: usize, xor: u8 }, + Truncate { keep: usize }, + Extend { extra: Vec }, +} + +fn tampered(body: &[u8], tamper: &Tamper) -> Vec { + match tamper { + Tamper::Flip { at, xor } => { + let mut bytes = body.to_vec(); + bytes[at % body.len()] ^= xor; + bytes + } + Tamper::Truncate { keep } => body[..keep % body.len()].to_vec(), + Tamper::Extend { extra } => [body, extra].concat(), + } +} + +fn tamper_strategy() -> impl Strategy { + prop_oneof![ + (any::(), 1u8..).prop_map(|(at, xor)| Tamper::Flip { at, xor }), + any::().prop_map(|keep| Tamper::Truncate { keep }), + proptest::collection::vec(any::(), 1..64).prop_map(|extra| Tamper::Extend { extra }), + ] +} + +fn rejects_tampering( + store: &dyn LfsStore, + body: &[u8], + tamper: &Tamper, +) -> Result<(), TestCaseError> { + let repo = repo(); + let oid = oid_of(body); + let size = ClaimedSize::new(body.len() as u64); + let forged = tampered(body, tamper); + let verdict = store.put(&repo, &oid, size, &mut &forged[..]); + prop_assert!( + matches!( + verdict, + Err(LfsError::HashMismatch { .. } | LfsError::SizeMismatch { .. }) + ), + "a tampered body must fail the verifier, got {verdict:?}" + ); + prop_assert_eq!(store.probe(&repo, &oid).expect("probe"), None); + Ok(()) +} + +fn disk_store() -> (DiskStore, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let store = DiskStore::open(LfsStorePath::new(dir.path())).expect("store opens"); + (store, dir) +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(64))] + + #[test] + fn any_body_round_trips_through_both_stores( + body in proptest::collection::vec(any::(), 0..4096) + ) { + round_trip(&MemoryStore::new(), &body)?; + let (store, _dir) = disk_store(); + round_trip(&store, &body)?; + } + + #[test] + fn the_verifier_rejects_any_tampered_or_truncated_body( + body in proptest::collection::vec(any::(), 1..2048), + tamper in tamper_strategy(), + ) { + prop_assume!(!matches!(&tamper, Tamper::Flip { xor: 0, .. })); + rejects_tampering(&MemoryStore::new(), &body, &tamper)?; + let (store, _dir) = disk_store(); + rejects_tampering(&store, &body, &tamper)?; + } + + #[test] + fn a_pointer_file_round_trips(digest in any::<[u8; 32]>(), size in any::()) { + let oid = LfsOid::from_digest(digest); + let text = format!( + "version https://git-lfs.github.com/spec/v1\noid sha256:{oid}\nsize {size}\n" + ); + let parsed = knot_lfs::parse_pointer(text.as_bytes()).expect("a spec pointer parses"); + prop_assert_eq!(parsed.oid, oid); + prop_assert_eq!(parsed.size, ClaimedSize::new(size)); + } +} diff --git a/knot2/crates/knot-lfs/tests/race.rs b/knot2/crates/knot-lfs/tests/race.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-lfs/tests/race.rs @@ -0,0 +1,127 @@ +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, SystemTime}; + +mod common; + +use common::{oid_of, repo}; +use knot_lfs::{ClaimedSize, DiskStore, LfsOid, LfsSize, LfsStore, LfsStorePath, Reclaimed}; + +const ROUNDS: usize = 400; +const GRACE: Duration = Duration::from_secs(14 * 86_400); +const BACKDATE: Duration = Duration::from_secs(60 * 86_400); + +#[derive(Clone, Copy)] +enum Bias { + TouchFirst, + SweepFirst, + Simultaneous, +} + +impl Bias { + fn of_round(round: usize) -> Self { + match round % 3 { + 0 => Self::TouchFirst, + 1 => Self::SweepFirst, + _ => Self::Simultaneous, + } + } +} + +fn seed_expired(store: &DiskStore, round: usize) -> LfsOid { + let body = format!("past-grace media, round {round}").into_bytes(); + let oid = oid_of(&body); + store + .put( + &repo(), + &oid, + ClaimedSize::new(body.len() as u64), + &mut &body[..], + ) + .unwrap(); + let path = store.object_file(&repo(), &oid).unwrap().unwrap().1; + std::fs::OpenOptions::new() + .write(true) + .open(path) + .unwrap() + .set_modified(SystemTime::now() - BACKDATE) + .unwrap(); + oid +} + +#[test] +fn a_mention_concurrent_with_the_sweep_never_yields_a_dangling_pointer() { + let dir = tempfile::tempdir().unwrap(); + let store = DiskStore::open(LfsStorePath::new(dir.path())).unwrap(); + + let outcomes: Vec<(Option, Reclaimed)> = (0..ROUNDS) + .map(|round| { + let oid = seed_expired(&store, round); + let bias = Bias::of_round(round); + let go = AtomicBool::new(false); + let touch_started = AtomicBool::new(false); + let sweep_started = AtomicBool::new(false); + let (vouched, reclaimed) = std::thread::scope(|scope| { + let toucher = scope.spawn(|| { + while !go.load(Ordering::Acquire) { + std::hint::spin_loop(); + } + if matches!(bias, Bias::SweepFirst) { + while !sweep_started.load(Ordering::Acquire) { + std::hint::spin_loop(); + } + } + touch_started.store(true, Ordering::Release); + store.touch(&repo(), &oid).unwrap() + }); + let sweeper = scope.spawn(|| { + while !go.load(Ordering::Acquire) { + std::hint::spin_loop(); + } + if matches!(bias, Bias::TouchFirst) { + while !touch_started.load(Ordering::Acquire) { + std::hint::spin_loop(); + } + } + sweep_started.store(true, Ordering::Release); + store + .collect_expired(&repo(), &oid, GRACE, SystemTime::now()) + .unwrap() + }); + go.store(true, Ordering::Release); + (toucher.join().unwrap(), sweeper.join().unwrap()) + }); + + if let Some(size) = vouched { + assert!( + matches!(reclaimed, Reclaimed::Spared), + "round {round}: the sweep deleted an object the server just reported stored" + ); + assert_eq!( + store.probe(&repo(), &oid).unwrap(), + Some(size), + "round {round}: an object reported stored must remain readable" + ); + } else { + assert!( + matches!(reclaimed, Reclaimed::Swept(_)), + "round {round}: a touch that reports missing means the sweeper unlinked first" + ); + assert_eq!( + store.probe(&repo(), &oid).unwrap(), + None, + "round {round}: a swept object reports missing" + ); + } + (vouched, reclaimed) + }) + .collect(); + + assert!( + outcomes.iter().any(|(vouched, _)| vouched.is_some()), + "some round must complete the touch first, or the interleaving never varied" + ); + assert!( + outcomes.iter().any(|(vouched, _)| vouched.is_none()), + "some round must complete the sweep first, or the interleaving never varied" + ); +} diff --git a/knot2/crates/knot-lfs/tests/soak.rs b/knot2/crates/knot-lfs/tests/soak.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-lfs/tests/soak.rs @@ -0,0 +1,160 @@ +mod common; + +use std::io::Write; + +use common::{ + GROWTH_SLACK, PEAK_CEILING, download_script, incompressible, oid_of, repo, rss_bytes, + upload_script, +}; +use knot_lfs::{ + ClaimedSize, DiskStore, FreeSpaceFloor, LfsOid, LfsSize, LfsStore, LfsStorePath, + StoreAdmission, TransferOp, serve_transfer, +}; + +const OBJECT_BYTES: usize = 8 * 1024 * 1024; +const SEEDS: usize = 4; +const WRITERS: u64 = 4; +const READERS: usize = 4; +const ROUNDS: u64 = 3; + +struct CountingSink(u64); + +impl Write for CountingSink { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0 += buf.len() as u64; + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +#[test] +fn sustained_concurrent_transfers_stay_bounded_and_leak_nothing() { + let dir = tempfile::tempdir().unwrap(); + let store = DiskStore::open(LfsStorePath::new(dir.path())).unwrap(); + let admission = StoreAdmission::new( + LfsStorePath::new(dir.path()), + LfsSize::new(u64::MAX), + FreeSpaceFloor::new(0), + ); + + let seeded: Vec = (0..SEEDS) + .map(|seed| { + let body = incompressible(OBJECT_BYTES, 0x5eed_0000 + seed as u64); + let oid = oid_of(&body); + store + .put( + &repo(), + &oid, + ClaimedSize::new(body.len() as u64), + &mut &body[..], + ) + .unwrap(); + oid + }) + .collect(); + + let storm = |round: u64| { + std::thread::scope(|scope| { + let writers: Vec<_> = (0..WRITERS) + .map(|writer| { + let store = &store; + let admission = &admission; + scope.spawn(move || { + let body = + incompressible(OBJECT_BYTES, 0xfeed_0000 + round * WRITERS + writer); + let (oid, script) = upload_script(&body); + let mut out = Vec::new(); + serve_transfer( + store, + admission, + &repo(), + TransferOp::Upload, + &knot_messages::default_catalog().lfs, + &script[..], + &mut out, + ) + .unwrap(); + let replies = String::from_utf8_lossy(&out); + assert!( + !replies.contains("status 4") && !replies.contains("status 5"), + "round {round} writer {writer}: upload session failed:\n{replies}" + ); + oid + }) + }) + .collect(); + let readers: Vec<_> = (0..READERS) + .map(|reader| { + let store = &store; + let admission = &admission; + let oid = seeded[reader % SEEDS].clone(); + scope.spawn(move || { + let script = download_script(&oid); + let mut sink = CountingSink(0); + serve_transfer( + store, + admission, + &repo(), + TransferOp::Download, + &knot_messages::default_catalog().lfs, + &script[..], + &mut sink, + ) + .unwrap(); + assert!( + sink.0 >= OBJECT_BYTES as u64, + "round {round} reader {reader}: streamed {} bytes", + sink.0 + ); + }) + }) + .collect(); + let uploaded: Vec = writers + .into_iter() + .map(|writer| writer.join().unwrap()) + .collect(); + readers.into_iter().for_each(|reader| { + reader.join().unwrap(); + }); + uploaded + }) + }; + + let mut uploaded = storm(0); + let settled = rss_bytes(); + + let peaks: Vec = (1..ROUNDS) + .map(|round| { + uploaded.extend(storm(round)); + rss_bytes() + }) + .collect(); + + let peak = peaks.iter().copied().max().unwrap_or(settled); + assert!( + peak < PEAK_CEILING, + "concurrent transfers peaked at {peak} bytes, ceiling {PEAK_CEILING}" + ); + let last = *peaks.last().unwrap_or(&settled); + assert!( + last <= settled + GROWTH_SLACK, + "rss grew from {settled} to {last} across rounds, transfers are leaking" + ); + + uploaded.iter().chain(seeded.iter()).for_each(|oid| { + assert!( + store.probe(&repo(), oid).unwrap().is_some(), + "object {oid} must be readable after the concurrent rounds" + ); + }); + let leftover: Vec<_> = std::fs::read_dir(dir.path().join(".incoming")) + .unwrap() + .collect(); + assert!( + leftover.is_empty(), + "temporary upload files leaked: {leftover:?}" + ); +} diff --git a/knot2/crates/knot-maintenance/src/bitmap.rs b/knot2/crates/knot-maintenance/src/bitmap.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-maintenance/src/bitmap.rs @@ -0,0 +1,71 @@ +use std::path::Path; + +use knot_git::Repo; + +use crate::MaintError; +use crate::fsio::{self, MIDX_SIDECAR_PREFIX, PackStem}; + +pub fn exists(objects_dir: &Path) -> bool { + fsio::pack_idx_paths(objects_dir) + .iter() + .any(|idx| idx.with_extension("bitmap").exists()) +} + +pub fn refresh(repo: &Repo, objects_dir: &Path) -> Result { + let idxs = fsio::pack_idx_paths(objects_dir); + match idxs.as_slice() { + [only] => { + let wrote = knot_git::write_bitmap(repo, only) + .map_err(|error| MaintError::Pack(error.to_string()))?; + let stem = PackStem::of(only); + prune_sidecars(objects_dir, stem.as_ref()); + Ok(wrote) + } + [] => { + prune_sidecars(objects_dir, None); + Ok(false) + } + _ => { + let wrote = knot_git::write_midx_bitmap(repo) + .map_err(|error| MaintError::Pack(error.to_string()))?; + let keep = current_midx_stem(objects_dir); + prune_sidecars(objects_dir, keep.as_ref()); + Ok(wrote) + } + } +} + +fn current_midx_stem(objects_dir: &Path) -> Option { + let path = objects_dir.join("pack").join("multi-pack-index"); + let file = + gix_pack::multi_index::File::at(path, Some(crate::midx::MIDX_ALLOC_LIMIT_BYTES)).ok()?; + Some(PackStem::midx_sidecar( + &file.checksum().to_hex().to_string(), + )) +} + +fn prune_sidecars(objects_dir: &Path, keep_stem: Option<&PackStem>) { + let pack_dir = objects_dir.join("pack"); + let Ok(entries) = std::fs::read_dir(&pack_dir) else { + return; + }; + entries + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| is_bitmap_sidecar(path)) + .filter(|path| PackStem::of(path).as_ref() != keep_stem) + .for_each(|path| { + let _ = std::fs::remove_file(path); + }); +} + +fn is_bitmap_sidecar(path: &Path) -> bool { + match path.extension().and_then(|ext| ext.to_str()) { + Some("bitmap") => true, + Some("rev") => path + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with(MIDX_SIDECAR_PREFIX)), + _ => false, + } +} diff --git a/knot2/crates/knot-maintenance/src/commitgraph.rs b/knot2/crates/knot-maintenance/src/commitgraph.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-maintenance/src/commitgraph.rs @@ -0,0 +1,795 @@ +use std::collections::{HashMap, HashSet}; + +use gix::ObjectId; +use gix::objs::tree::EntryKind as TreeEntryKind; +use gix::objs::{CommitRef, Kind, TagRefIter, TreeRef}; +use gix::prelude::FindExt; +use knot_git::Repo; +use knot_types::UnixSeconds; + +use crate::MaintError; +use crate::fsio; + +const GRAPH_PARENT_NONE: u32 = 0x7000_0000; +const GRAPH_EXTRA_EDGES_NEEDED: u32 = 0x8000_0000; +const GRAPH_LAST_EDGE: u32 = 0x8000_0000; +const GRAPH_GENERATION_MAX: u32 = 0x3FFF_FFFF; +const MAX_PEEL_DEPTH: usize = 32; + +const CORRECTED_OFFSET_OVERFLOW: u32 = 0x8000_0000; +const CORRECTED_OFFSET_MAX: u64 = (1 << 31) - 1; + +const BLOOM_HASH_VERSION: u32 = 2; +const BLOOM_NUM_HASHES: u32 = 7; +const BLOOM_BITS_PER_ENTRY: u32 = 10; +const BLOOM_MAX_CHANGED_PATHS: usize = 512; +const BLOOM_SEED0: u32 = 0x293a_e76f; +const BLOOM_SEED1: u32 = 0x7e64_6e2c; +const MURMUR_C1: u32 = 0xcc9e_2d51; +const MURMUR_C2: u32 = 0x1b87_3593; +const MURMUR_N: u32 = 0xe654_6b64; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +struct Generation(u32); + +impl Generation { + const fn new(value: u32) -> Self { + Self(value) + } + + const fn get(self) -> u32 { + self.0 + } + + const fn succ(self) -> Self { + Self(self.0.saturating_add(1)) + } +} + +knot_types::scalar_newtype! { + struct GraphPosition(u32); +} + +#[derive(Debug, Clone, Copy)] +struct ParentField(u32); + +impl ParentField { + const NONE: Self = Self(GRAPH_PARENT_NONE); + + fn from_position(position: Option) -> Self { + position.map_or(Self::NONE, |position| Self(position.get())) + } + + fn extra_edges(index: usize) -> Self { + Self(GRAPH_EXTRA_EDGES_NEEDED | index as u32) + } + + fn to_be_bytes(self) -> [u8; 4] { + self.0.to_be_bytes() + } +} + +#[derive(Debug, Clone, Copy)] +struct ParentFields { + first: ParentField, + second: ParentField, +} + +#[derive(Debug, Clone, Copy)] +struct EdgeField(u32); + +impl EdgeField { + fn new(parent: ParentField, last: bool) -> Self { + Self(parent.0 | if last { GRAPH_LAST_EDGE } else { 0 }) + } + + fn to_be_bytes(self) -> [u8; 4] { + self.0.to_be_bytes() + } +} + +knot_types::scalar_newtype! { + struct CorrectedDate(u64); +} + +struct ChangedPathFilter(Vec); + +impl ChangedPathFilter { + fn bytes(&self) -> &[u8] { + &self.0 + } + + fn len_bytes(&self) -> usize { + self.0.len() + } +} + +struct CommitMeta { + tree: ObjectId, + parents: Vec, + seconds: UnixSeconds, +} + +pub fn graph_path(repo: &Repo) -> std::path::PathBuf { + repo.objects_dir().join("info").join("commit-graph") +} + +pub fn exists(repo: &Repo) -> bool { + graph_path(repo).exists() +} + +pub fn write(repo: &Repo) -> Result { + let kind = repo.object_format().kind(); + let Some(commits) = collect(repo, kind)? else { + return Ok(false); + }; + if commits.is_empty() { + return Ok(false); + } + let blooms = changed_path_filters(repo, &commits, kind)?; + let bytes = serialize(&commits, &blooms, kind); + let path = graph_path(repo); + let info_dir = repo.objects_dir().join("info"); + std::fs::create_dir_all(&info_dir).map_err(|error| fsio::io_error(&info_dir, error))?; + clear_chain(&info_dir)?; + knot_resource::atomic_write_bytes(&path, &bytes, knot_resource::FileMode::Inherited)?; + Ok(true) +} + +pub fn remove(repo: &Repo) -> Result<(), MaintError> { + let path = graph_path(repo); + match std::fs::remove_file(&path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(fsio::io_error(&path, error)), + } + clear_chain(&repo.objects_dir().join("info")) +} + +fn clear_chain(info_dir: &std::path::Path) -> Result<(), MaintError> { + let chain_dir = info_dir.join("commit-graphs"); + match std::fs::remove_dir_all(&chain_dir) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(fsio::io_error(&chain_dir, error)), + } +} + +fn collect( + repo: &Repo, + kind: gix::hash::Kind, +) -> Result>, MaintError> { + let odb = &repo.git().objects; + let mut stack: Vec = repo + .references()? + .into_iter() + .filter_map(|record| peel_to_commit(odb, record.target.object_id(), kind, MAX_PEEL_DEPTH)) + .collect(); + let mut commits: HashMap = HashMap::new(); + while let Some(oid) = stack.pop() { + if commits.contains_key(&oid) { + continue; + } + let mut buf = Vec::new(); + let data = match odb.find(&oid, &mut buf) { + Ok(data) => data, + Err(_) => return Ok(None), + }; + if data.kind != Kind::Commit { + return Ok(None); + } + let commit = CommitRef::from_bytes(data.data, kind) + .map_err(|error| MaintError::CommitGraph(error.to_string()))?; + let tree = commit.tree(); + let parents: Vec = commit.parents().collect(); + let seconds = UnixSeconds::new(commit.committer().map(|sig| sig.seconds()).unwrap_or(0)); + parents.iter().for_each(|parent| stack.push(*parent)); + commits.insert( + oid, + CommitMeta { + tree, + parents, + seconds, + }, + ); + } + Ok(Some(commits)) +} + +fn peel_to_commit( + odb: &gix::odb::Handle, + oid: ObjectId, + kind: gix::hash::Kind, + depth: usize, +) -> Option { + if depth == 0 { + return None; + } + let mut buf = Vec::new(); + let data = odb.find(&oid, &mut buf).ok()?; + match data.kind { + Kind::Commit => Some(oid), + Kind::Tag => { + let target = TagRefIter::from_bytes(data.data, kind).target_id().ok()?; + peel_to_commit(odb, target, kind, depth - 1) + } + _ => None, + } +} + +fn serialize( + commits: &HashMap, + blooms: &HashMap, + kind: gix::hash::Kind, +) -> Vec { + let hash_len = match kind { + gix::hash::Kind::Sha256 => 32, + _ => 20, + }; + let mut oids: Vec = commits.keys().copied().collect(); + oids.sort(); + let position: HashMap = oids + .iter() + .enumerate() + .map(|(index, oid)| (*oid, GraphPosition::new(index as u32))) + .collect(); + let generations = generations(commits, &oids); + let corrected = corrected_dates(commits, &oids); + + let mut edges: Vec = Vec::new(); + let mut cdat: Vec = Vec::with_capacity(oids.len() * (hash_len + 16)); + oids.iter().for_each(|oid| { + let meta = &commits[oid]; + cdat.extend_from_slice(meta.tree.as_slice()); + let parents = parent_fields(&meta.parents, &position, &mut edges); + cdat.extend_from_slice(&parents.first.to_be_bytes()); + cdat.extend_from_slice(&parents.second.to_be_bytes()); + let generation = generations + .get(oid) + .copied() + .unwrap_or(Generation::new(0)) + .get() as u64; + let date = (meta.seconds.get().max(0) as u64) & 0x3_FFFF_FFFF; + let packed = (generation << 34) | date; + cdat.extend_from_slice(&packed.to_be_bytes()); + }); + + let (gda2, overflow) = oids.iter().fold( + (Vec::with_capacity(oids.len() * 4), Vec::::new()), + |(mut bytes, mut ovf), oid| { + let date = commits[oid].seconds.get().max(0) as u64; + let offset = corrected + .get(oid) + .copied() + .unwrap_or(CorrectedDate::new(date)) + .get() + .saturating_sub(date); + let packed = if offset > CORRECTED_OFFSET_MAX { + let index = ovf.len() as u32; + ovf.push(offset); + CORRECTED_OFFSET_OVERFLOW | index + } else { + offset as u32 + }; + bytes.extend_from_slice(&packed.to_be_bytes()); + (bytes, ovf) + }, + ); + let gdo2: Vec = overflow + .iter() + .flat_map(|value| value.to_be_bytes()) + .collect(); + + let bidx: Vec = oids + .iter() + .scan(0u32, |acc, oid| { + *acc = acc.saturating_add(filter_for(blooms, oid).len_bytes() as u32); + Some(acc.to_be_bytes()) + }) + .flatten() + .collect(); + let bdat: Vec = [BLOOM_HASH_VERSION, BLOOM_NUM_HASHES, BLOOM_BITS_PER_ENTRY] + .iter() + .flat_map(|value| value.to_be_bytes()) + .chain( + oids.iter() + .flat_map(|oid| filter_for(blooms, oid).bytes().iter().copied()), + ) + .collect(); + + let oidf = fanout(&oids); + let mut oidl: Vec = Vec::with_capacity(oids.len() * hash_len); + oids.iter() + .for_each(|oid| oidl.extend_from_slice(oid.as_slice())); + let mut edge_bytes: Vec = Vec::with_capacity(edges.len() * 4); + edges + .iter() + .for_each(|edge| edge_bytes.extend_from_slice(&edge.to_be_bytes())); + + let mut chunks: Vec<(&[u8; 4], Vec)> = vec![ + (b"OIDF", oidf), + (b"OIDL", oidl), + (b"CDAT", cdat), + (b"GDA2", gda2), + ]; + if !gdo2.is_empty() { + chunks.push((b"GDO2", gdo2)); + } + if !edge_bytes.is_empty() { + chunks.push((b"EDGE", edge_bytes)); + } + chunks.push((b"BIDX", bidx)); + chunks.push((b"BDAT", bdat)); + + assemble(chunks, kind) +} + +fn filter_for<'a>( + blooms: &'a HashMap, + oid: &ObjectId, +) -> &'a ChangedPathFilter { + static EMPTY: ChangedPathFilter = ChangedPathFilter(Vec::new()); + blooms.get(oid).unwrap_or(&EMPTY) +} + +fn parent_fields( + parents: &[ObjectId], + position: &HashMap, + edges: &mut Vec, +) -> ParentFields { + let pos = |oid: &ObjectId| ParentField::from_position(position.get(oid).copied()); + match parents { + [] => ParentFields { + first: ParentField::NONE, + second: ParentField::NONE, + }, + [first] => ParentFields { + first: pos(first), + second: ParentField::NONE, + }, + [first, second] => ParentFields { + first: pos(first), + second: pos(second), + }, + [first, rest @ ..] => { + let edge_index = edges.len(); + let last = rest.len() - 1; + rest.iter().enumerate().for_each(|(index, parent)| { + edges.push(EdgeField::new(pos(parent), index == last)); + }); + ParentFields { + first: pos(first), + second: ParentField::extra_edges(edge_index), + } + } + } +} + +fn fanout(oids: &[ObjectId]) -> Vec { + let mut buckets = [0u32; 256]; + oids.iter() + .for_each(|oid| buckets[oid.as_slice()[0] as usize] += 1); + (1..256).for_each(|index| buckets[index] += buckets[index - 1]); + buckets + .iter() + .flat_map(|count| count.to_be_bytes()) + .collect() +} + +fn resolve_topo( + commits: &HashMap, + oids: &[ObjectId], + transform: impl Fn(&[V], &CommitMeta) -> V, +) -> HashMap { + let mut value: HashMap = HashMap::new(); + oids.iter().for_each(|root| { + if value.contains_key(root) { + return; + } + let mut stack = vec![*root]; + while let Some(top) = stack.last().copied() { + if value.contains_key(&top) { + stack.pop(); + continue; + } + let parents = &commits[&top].parents; + let unresolved: Vec = parents + .iter() + .filter(|parent| commits.contains_key(*parent) && !value.contains_key(*parent)) + .copied() + .collect(); + if unresolved.is_empty() { + let resolved: Vec = parents + .iter() + .filter_map(|parent| value.get(parent)) + .copied() + .collect(); + let computed = transform(&resolved, &commits[&top]); + value.insert(top, computed); + stack.pop(); + } else { + unresolved.into_iter().for_each(|parent| stack.push(parent)); + } + } + }); + value +} + +fn generations( + commits: &HashMap, + oids: &[ObjectId], +) -> HashMap { + resolve_topo(commits, oids, |parents: &[Generation], _meta| { + parents + .iter() + .copied() + .max() + .unwrap_or(Generation::new(0)) + .succ() + .min(Generation::new(GRAPH_GENERATION_MAX)) + }) +} + +fn corrected_dates( + commits: &HashMap, + oids: &[ObjectId], +) -> HashMap { + resolve_topo(commits, oids, |parents: &[CorrectedDate], meta| { + let max_parent = parents.iter().map(|date| date.get()).max().unwrap_or(0); + let date = meta.seconds.get().max(0) as u64; + let base = if date > max_parent { + date - 1 + } else { + max_parent + }; + CorrectedDate::new(base + 1) + }) +} + +fn tuned(handle: &gix::odb::Handle) -> gix::odb::Handle { + let mut odb = handle.clone(); + odb.refresh_never(); + odb.prevent_pack_unload(); + odb +} + +fn one_filter( + odb: &gix::odb::Handle, + commits: &HashMap, + meta: &CommitMeta, + kind: gix::hash::Kind, +) -> Result { + let parent_tree = meta + .parents + .first() + .and_then(|parent| commits.get(parent)) + .map(|found| found.tree); + let changed = diff_trees(odb, parent_tree, Some(meta.tree), kind)?; + Ok(build_filter(&changed)) +} + +fn changed_path_filters( + repo: &Repo, + commits: &HashMap, + kind: gix::hash::Kind, +) -> Result, MaintError> { + let entries: Vec<(&ObjectId, &CommitMeta)> = commits.iter().collect(); + let path = repo.path().to_owned(); + let produced = knot_resource::map_chunks(&entries, |batch| { + let local = Repo::open(&path)?; + let odb = tuned(&local.git().objects); + batch + .iter() + .map(|(oid, meta)| Ok((**oid, one_filter(&odb, commits, meta, kind)?))) + .collect::, MaintError>>() + })?; + Ok(produced.into_iter().collect()) +} + +fn tree_entries( + odb: &gix::odb::Handle, + oid: Option, + kind: gix::hash::Kind, +) -> Result, (TreeEntryKind, ObjectId)>, MaintError> { + let Some(oid) = oid else { + return Ok(HashMap::new()); + }; + if oid == ObjectId::empty_tree(kind) { + return Ok(HashMap::new()); + } + let mut buf = Vec::new(); + let data = odb + .find(&oid, &mut buf) + .map_err(|error| MaintError::CommitGraph(error.to_string()))?; + if data.kind != Kind::Tree { + return Ok(HashMap::new()); + } + let tree = TreeRef::from_bytes(data.data, kind) + .map_err(|error| MaintError::CommitGraph(error.to_string()))?; + Ok(tree + .entries + .into_iter() + .map(|entry| { + ( + entry.filename.to_vec(), + (entry.mode.kind(), entry.oid.to_owned()), + ) + }) + .collect()) +} + +fn diff_trees( + odb: &gix::odb::Handle, + parent: Option, + commit: Option, + kind: gix::hash::Kind, +) -> Result>, MaintError> { + let is_tree = |kind: &TreeEntryKind| matches!(kind, TreeEntryKind::Tree); + let mut out: Vec> = Vec::new(); + let mut stack: Vec<(Option, Option, Vec)> = + vec![(parent, commit, Vec::new())]; + while let Some((parent, commit, prefix)) = stack.pop() { + let parent_entries = tree_entries(odb, parent, kind)?; + let commit_entries = tree_entries(odb, commit, kind)?; + let names: HashSet<&Vec> = parent_entries.keys().chain(commit_entries.keys()).collect(); + names.into_iter().for_each(|name| { + let full: Vec = prefix.iter().copied().chain(name.iter().copied()).collect(); + let subprefix = + || -> Vec { full.iter().copied().chain(std::iter::once(b'/')).collect() }; + match (parent_entries.get(name), commit_entries.get(name)) { + (None, Some((ck, co))) => { + if is_tree(ck) { + stack.push((None, Some(*co), subprefix())); + } else { + out.push(full); + } + } + (Some((pk, po)), None) => { + if is_tree(pk) { + stack.push((Some(*po), None, subprefix())); + } else { + out.push(full); + } + } + (Some((pk, po)), Some((ck, co))) => match (is_tree(pk), is_tree(ck)) { + (true, true) => { + if po != co { + stack.push((Some(*po), Some(*co), subprefix())); + } + } + (false, false) => { + if po != co || pk != ck { + out.push(full); + } + } + (true, false) => { + stack.push((Some(*po), None, subprefix())); + out.push(full); + } + (false, true) => { + stack.push((None, Some(*co), subprefix())); + out.push(full); + } + }, + (None, None) => {} + } + }); + } + Ok(out) +} + +fn build_filter(changed: &[Vec]) -> ChangedPathFilter { + if changed.len() > BLOOM_MAX_CHANGED_PATHS { + // `0xFF` = git's "too many changed paths" marker. + return ChangedPathFilter(vec![0xFF]); + } + let paths: HashSet> = changed.iter().flat_map(|path| prefixes(path)).collect(); + if paths.len() > BLOOM_MAX_CHANGED_PATHS { + return ChangedPathFilter(vec![0xFF]); + } + let bits = paths.len() * BLOOM_BITS_PER_ENTRY as usize; + let len_bytes = bits.div_ceil(8).max(1); + let modulus = (len_bytes * 8) as u64; + let data = paths.iter().fold(vec![0u8; len_bytes], |mut data, path| { + let hash0 = murmur3(BLOOM_SEED0, path); + let hash1 = murmur3(BLOOM_SEED1, path); + (0..BLOOM_NUM_HASHES).for_each(|index| { + let combined = hash0.wrapping_add(index.wrapping_mul(hash1)); + let position = (combined as u64) % modulus; + data[(position / 8) as usize] |= 1 << (position % 8); + }); + data + }); + ChangedPathFilter(data) +} + +fn prefixes(path: &[u8]) -> Vec> { + std::iter::once(path.to_vec()) + .chain( + path.iter() + .enumerate() + .filter(|(_, byte)| **byte == b'/') + .map(|(index, _)| path[..index].to_vec()), + ) + .collect() +} + +fn murmur3(seed: u32, data: &[u8]) -> u32 { + let body = data.len() / 4; + let mixed = (0..body).fold(seed, |seed, index| { + let base = index * 4; + let block = + u32::from_le_bytes([data[base], data[base + 1], data[base + 2], data[base + 3]]); + let block = block + .wrapping_mul(MURMUR_C1) + .rotate_left(15) + .wrapping_mul(MURMUR_C2); + (seed ^ block) + .rotate_left(13) + .wrapping_mul(5) + .wrapping_add(MURMUR_N) + }); + let tail = &data[body * 4..]; + let tail_key = tail.iter().enumerate().fold(0u32, |key, (index, byte)| { + key | ((*byte as u32) << (8 * index)) + }); + let mixed = if tail.is_empty() { + mixed + } else { + mixed + ^ tail_key + .wrapping_mul(MURMUR_C1) + .rotate_left(15) + .wrapping_mul(MURMUR_C2) + }; + let mixed = mixed ^ (data.len() as u32); + let mixed = (mixed ^ (mixed >> 16)).wrapping_mul(0x85eb_ca6b); + let mixed = (mixed ^ (mixed >> 13)).wrapping_mul(0xc2b2_ae35); + mixed ^ (mixed >> 16) +} + +fn assemble(chunks: Vec<(&[u8; 4], Vec)>, kind: gix::hash::Kind) -> Vec { + let num_chunks = chunks.len() as u8; + let table_len = (chunks.len() + 1) * 12; + let data_start = 8 + table_len; + let hash_version = match kind { + gix::hash::Kind::Sha256 => 2u8, + _ => 1u8, + }; + + let mut out: Vec = Vec::new(); + out.extend_from_slice(b"CGPH"); + out.push(1); + out.push(hash_version); + out.push(num_chunks); + out.push(0); + + let mut offset = data_start as u64; + chunks.iter().for_each(|(id, body)| { + out.extend_from_slice(*id); + out.extend_from_slice(&offset.to_be_bytes()); + offset += body.len() as u64; + }); + out.extend_from_slice(&[0, 0, 0, 0]); + out.extend_from_slice(&offset.to_be_bytes()); + + chunks + .iter() + .for_each(|(_, body)| out.extend_from_slice(body)); + + let mut hasher = gix_hash::hasher(kind); + hasher.update(&out); + let checksum = hasher + .try_finalize() + .expect("commit-graph checksum finalizes"); + out.extend_from_slice(checksum.as_slice()); + out +} + +#[cfg(test)] +mod tests { + use super::{GRAPH_GENERATION_MAX, Generation, build_filter, murmur3}; + + #[test] + fn succ_advances_by_one_and_orders_above_its_source() { + let base = Generation::new(7); + assert_eq!(base.succ(), Generation::new(8)); + assert!(base.succ() > base); + } + + #[test] + fn succ_saturates_at_the_numeric_ceiling() { + assert_eq!(Generation::new(u32::MAX).succ(), Generation::new(u32::MAX)); + } + + #[test] + fn a_child_sits_one_above_its_highest_parent() { + let parents = [Generation::new(2), Generation::new(5), Generation::new(3)]; + let child = parents.into_iter().max().unwrap().succ(); + assert_eq!(child, Generation::new(6)); + } + + #[test] + fn clamping_holds_the_value_at_the_format_maximum() { + let value = Generation::new(GRAPH_GENERATION_MAX) + .succ() + .min(Generation::new(GRAPH_GENERATION_MAX)); + assert_eq!(value, Generation::new(GRAPH_GENERATION_MAX)); + } + + #[test] + fn empty_change_set_is_a_single_zero_word() { + let filter = build_filter(&[]); + assert_eq!(filter.bytes(), &[0u8]); + } + + #[test] + fn overlarge_change_set_is_a_single_saturated_word() { + let many: Vec> = (0..600).map(|n| format!("p{n}").into_bytes()).collect(); + let filter = build_filter(&many); + assert_eq!(filter.bytes(), &[0xFFu8]); + } + + #[test] + fn murmur3_matches_known_vectors() { + assert_eq!(murmur3(0, b""), 0); + assert_eq!(murmur3(0, b"hello"), 0x248bfa47); + } + + #[test] + fn diff_trees_walks_a_deeply_nested_tree_without_overflowing_the_stack() { + use knot_git::{EntryKind, Repo, StagedAction, StagedChange}; + use knot_types::Oid; + + let depth = 4000usize; + let dir = tempfile::tempdir().unwrap(); + let git_dir = dir.path().join("deep.git"); + + let build_dir = git_dir.clone(); + let tree = std::thread::Builder::new() + .stack_size(64 * 1024 * 1024) + .spawn(move || { + let repo = Repo::create(&build_dir).unwrap(); + let kind = repo.object_format().kind(); + let base = Oid::from(gix::ObjectId::empty_tree(kind)); + let path = (0..depth) + .map(|_| "d") + .chain(std::iter::once("leaf.txt")) + .collect::>() + .join("/"); + repo.write_staged_tree( + base, + &[StagedChange { + path: knot_types::RepoPath::new(path).unwrap(), + action: StagedAction::Put { + content: b"leaf\n".to_vec(), + kind: EntryKind::Blob, + }, + }], + ) + .unwrap() + .object_id() + }) + .unwrap() + .join() + .unwrap(); + + let changed = std::thread::Builder::new() + .stack_size(256 * 1024) + .spawn(move || { + let repo = Repo::open(&git_dir).unwrap(); + let kind = repo.object_format().kind(); + super::diff_trees(&repo.git().objects, None, Some(tree), kind).unwrap() + }) + .unwrap() + .join() + .expect("diff_trees on a 256 KiB stack mustn't overflow on a 4000-deep tree"); + + assert_eq!(changed.len(), 1, "the single leaf is the only changed path"); + assert_eq!( + changed[0].iter().filter(|byte| **byte == b'/').count(), + depth, + "the changed path retains every nesting level" + ); + } +} diff --git a/knot2/crates/knot-maintenance/src/cruft.rs b/knot2/crates/knot-maintenance/src/cruft.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-maintenance/src/cruft.rs @@ -0,0 +1,529 @@ +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::AtomicBool; +use std::time::{Duration, SystemTime}; + +use gix::progress::Discard; +use knot_types::{Oid, UnixSeconds}; + +use crate::fsio::{self, PackStem}; +use crate::{FileCount, MaintError, ObjectCount, PruneReport}; + +const MTIMES_MAGIC: u32 = 0x4d54_4d45; +const MTIMES_VERSION: u32 = 1; + +pub fn run( + objects_dir: &Path, + kind: gix::hash::Kind, + reachable: &HashSet, + new_reachable_stem: Option<&PackStem>, + kept_large: &[PackStem], + loose: &[(Oid, PathBuf)], + grace: Duration, +) -> Result { + let pack_dir = objects_dir.join("pack"); + let idxs = fsio::pack_idx_paths(objects_dir); + let now = SystemTime::now(); + let now_mtime = PackedMtime::from_system_time(now); + + let kept_pack = + |stem: &PackStem| -> bool { Some(stem) == new_reachable_stem || kept_large.contains(stem) }; + + let kept_pack_oids: HashSet = idxs + .iter() + .filter(|idx| PackStem::of(idx).is_some_and(|stem| kept_pack(&stem))) + .filter_map(|idx| fsio::pack_oids(idx, kind)) + .flatten() + .collect(); + + let recorded = read_recorded_mtimes(&idxs, kind); + let mtime_of = mtime_index(&idxs, &pack_dir, kind, &recorded, loose); + + let unreachable = + unreachable_candidates(&idxs, kind, loose, reachable, &kept_pack_oids, &kept_pack); + let mut keep: Vec = unreachable + .into_iter() + .filter(|oid| !is_expired(mtime_of.get(oid), now, grace)) + .collect(); + keep.sort(); + + let cruft_stem = if keep.is_empty() { + None + } else { + let stem = write_cruft_pack(objects_dir, keep.clone(), kind)?; + if let Some(stem) = &stem { + write_mtimes(&pack_dir, stem, kind, |oid| { + mtime_of + .get(&oid) + .map(PackedMtime::from_unix) + .unwrap_or(now_mtime) + })?; + } + stem + }; + knot_resource::fsync_path(&pack_dir)?; + + let kept_oids: HashSet = kept_pack_oids + .iter() + .copied() + .chain(cruft_stem.as_ref().into_iter().flat_map(|stem| { + fsio::pack_oids(&stem.file(&pack_dir, "idx"), kind).unwrap_or_default() + })) + .collect(); + + let keep_covered = keep.iter().all(|oid| kept_oids.contains(oid)); + if !closure_is_covered(reachable, &kept_oids) || !keep_covered { + if let Some(stem) = cruft_stem.as_ref() { + fsio::remove_pack_files(&stem.file(&pack_dir, "idx"), &pack_dir); + } + return Ok(PruneReport::skipped()); + } + + let removed_packs = idxs + .iter() + .filter(|idx| { + PackStem::of(idx) + .is_some_and(|stem| !kept_pack(&stem) && Some(&stem) != cruft_stem.as_ref()) + }) + .filter(|idx| fsio::remove_pack_files(idx, &pack_dir)) + .count(); + let removed_loose = loose + .iter() + .filter(|(oid, _)| !reachable.contains(oid)) + .filter(|(_, path)| std::fs::remove_file(path).is_ok()) + .count(); + knot_resource::fsync_path(&pack_dir)?; + knot_resource::fsync_path(objects_dir)?; + + Ok(PruneReport { + removed: FileCount::new(removed_loose), + removed_packs: FileCount::new(removed_packs), + crufted: ObjectCount::new(keep.len()), + ran: true, + }) +} + +fn closure_is_covered(reachable: &HashSet, kept: &HashSet) -> bool { + reachable.is_subset(kept) +} + +fn unreachable_candidates bool>( + idxs: &[PathBuf], + kind: gix::hash::Kind, + loose: &[(Oid, PathBuf)], + reachable: &HashSet, + kept_pack_oids: &HashSet, + kept_pack: &K, +) -> Vec { + idxs.iter() + .filter(|idx| PackStem::of(idx).is_some_and(|stem| !kept_pack(&stem))) + .filter_map(|idx| fsio::pack_oids(idx, kind)) + .flatten() + .chain(loose.iter().map(|(oid, _)| *oid)) + .filter(|oid| !reachable.contains(oid)) + .filter(|oid| !kept_pack_oids.contains(oid)) + .collect::>() + .into_iter() + .collect() +} + +fn mtime_index( + idxs: &[PathBuf], + pack_dir: &Path, + kind: gix::hash::Kind, + recorded: &HashMap, + loose: &[(Oid, PathBuf)], +) -> HashMap { + let from_packs = idxs.iter().flat_map(|idx| { + let is_cruft = idx.with_extension("mtimes").exists(); + let pack_secs = fsio::pack_mtime(idx, pack_dir); + fsio::pack_oids(idx, kind) + .unwrap_or_default() + .into_iter() + .filter_map(move |oid| { + let secs = if is_cruft { + recorded.get(&oid).copied() + } else { + pack_secs + }; + secs.map(|secs| (oid, secs)) + }) + }); + let from_loose = loose + .iter() + .filter_map(|(oid, path)| loose_mtime(path).map(|secs| (*oid, secs))); + newest_by_oid(from_packs.chain(from_loose)) +} + +fn newest_by_oid(pairs: impl Iterator) -> HashMap { + pairs.fold(HashMap::new(), |mut acc, (oid, secs)| { + acc.entry(oid) + .and_modify(|current| { + if secs.get() > current.get() { + *current = secs; + } + }) + .or_insert(secs); + acc + }) +} + +fn read_recorded_mtimes(idxs: &[PathBuf], kind: gix::hash::Kind) -> HashMap { + idxs.iter() + .filter(|idx| idx.with_extension("mtimes").exists()) + .filter_map(|idx| { + let bytes = std::fs::read(idx.with_extension("mtimes")).ok()?; + let oids = fsio::pack_oids(idx, kind)?; + let checksum = gix_pack::data::File::at(idx.with_extension("pack"), kind) + .ok()? + .checksum(); + let table = validated_mtimes(&bytes, oids.len(), kind, checksum.as_slice())?; + Some( + oids.into_iter() + .zip(table) + .map(|(oid, mtime)| (oid, mtime.to_unix())) + .collect::>(), + ) + }) + .flatten() + .collect() +} + +fn validated_mtimes( + bytes: &[u8], + count: usize, + kind: gix::hash::Kind, + pack_checksum: &[u8], +) -> Option> { + let hash_len = kind.len_in_bytes(); + let header = 12usize; + let total = header + count * 4 + hash_len * 2; + if bytes.len() != total + || bytes[0..4] != MTIMES_MAGIC.to_be_bytes() + || bytes[4..8] != MTIMES_VERSION.to_be_bytes() + || bytes[8..12] != hash_id(kind).to_be_bytes() + { + return None; + } + let table_end = header + count * 4; + if bytes[table_end..table_end + hash_len] != *pack_checksum { + return None; + } + let mut hasher = gix_hash::hasher(kind); + hasher.update(&bytes[..total - hash_len]); + let digest = hasher.try_finalize().ok()?; + if digest.as_slice() != &bytes[total - hash_len..] { + return None; + } + Some( + (0..count) + .map(|index| { + let offset = header + index * 4; + PackedMtime(u32::from_be_bytes( + bytes[offset..offset + 4].try_into().unwrap(), + )) + }) + .collect(), + ) +} + +fn is_expired(mtime: Option<&UnixSeconds>, now: SystemTime, grace: Duration) -> bool { + let Some(mtime) = mtime else { + return false; + }; + let when = SystemTime::UNIX_EPOCH + Duration::from_secs(mtime.get().max(0) as u64); + now.duration_since(when) + .map(|age| age >= grace) + .unwrap_or(false) +} + +fn loose_mtime(path: &Path) -> Option { + let modified = path.metadata().ok()?.modified().ok()?; + Some(PackedMtime::from_system_time(modified).to_unix()) +} + +#[derive(Debug, Clone, Copy)] +struct PackedMtime(u32); + +impl PackedMtime { + fn from_unix(secs: &UnixSeconds) -> Self { + Self(secs.get().clamp(0, u32::MAX as i64) as u32) + } + + fn from_system_time(time: SystemTime) -> Self { + Self( + time.duration_since(SystemTime::UNIX_EPOCH) + .map(|delta| delta.as_secs().min(u32::MAX as u64) as u32) + .unwrap_or(0), + ) + } + + fn to_unix(self) -> UnixSeconds { + UnixSeconds::new(self.0 as i64) + } + + fn to_be_bytes(self) -> [u8; 4] { + self.0.to_be_bytes() + } +} + +fn hash_id(kind: gix::hash::Kind) -> u32 { + match kind { + gix::hash::Kind::Sha256 => 2, + _ => 1, + } +} + +fn write_cruft_pack( + objects_dir: &Path, + oids: Vec, + kind: gix::hash::Kind, +) -> Result, MaintError> { + let pack_dir = objects_dir.join("pack"); + std::fs::create_dir_all(&pack_dir).map_err(|error| fsio::io_error(&pack_dir, error))?; + knot_resource::clear_stale(&pack_dir, ".knot-cruft."); + let staging = pack_dir.join(format!( + ".knot-cruft.{}.pack", + knot_resource::staging_nonce() + )); + let outcome = stream_pack(objects_dir, oids, kind, &staging) + .and_then(|()| install_pack(&pack_dir, &staging, kind)); + let _ = std::fs::remove_file(&staging); + outcome +} + +fn stream_pack( + objects_dir: &Path, + oids: Vec, + kind: gix::hash::Kind, + staging: &Path, +) -> Result<(), MaintError> { + let file = std::fs::File::create(staging).map_err(|error| fsio::io_error(staging, error))?; + let mut writer = std::io::BufWriter::new(file); + knot_pack::write_pack(objects_dir, oids, None, &mut writer, kind) + .map_err(|error| MaintError::Pack(error.to_string()))?; + writer + .into_inner() + .map(|_| ()) + .map_err(|error| fsio::io_error(staging, error.into_error())) +} + +fn install_pack( + pack_dir: &Path, + staging: &Path, + kind: gix::hash::Kind, +) -> Result, MaintError> { + let file = std::fs::File::open(staging).map_err(|error| fsio::io_error(staging, error))?; + let mut reader = std::io::BufReader::new(file); + let outcome = gix_pack::Bundle::write_to_directory( + &mut reader, + Some(pack_dir), + &mut Discard, + &AtomicBool::new(false), + None::, + gix_pack::bundle::write::Options { + thread_limit: Some(1), + iteration_mode: gix_pack::data::input::Mode::Verify, + index_version: gix_pack::index::Version::default(), + object_hash: kind, + }, + ) + .map_err(|error| MaintError::Pack(error.to_string()))?; + + if let Some(keep) = &outcome.keep_path { + let _ = std::fs::remove_file(keep); + } + [&outcome.data_path, &outcome.index_path] + .into_iter() + .flatten() + .try_for_each(|path| knot_resource::fsync_path(path))?; + + Ok(outcome + .data_path + .as_ref() + .and_then(|path| PackStem::of(path))) +} + +fn write_mtimes( + pack_dir: &Path, + stem: &PackStem, + kind: gix::hash::Kind, + mtime_for: impl Fn(Oid) -> PackedMtime, +) -> Result<(), MaintError> { + let idx = stem.file(pack_dir, "idx"); + let pack = stem.file(pack_dir, "pack"); + let index = gix_pack::index::File::at(&idx, kind) + .map_err(|error| MaintError::Pack(format!("open cruft index: {error}")))?; + let checksum = gix_pack::data::File::at(&pack, kind) + .map_err(|error| MaintError::Pack(format!("open cruft pack: {error}")))? + .checksum(); + + let mut out = Vec::new(); + out.extend_from_slice(&MTIMES_MAGIC.to_be_bytes()); + out.extend_from_slice(&MTIMES_VERSION.to_be_bytes()); + out.extend_from_slice(&hash_id(kind).to_be_bytes()); + index + .iter() + .for_each(|entry| out.extend_from_slice(&mtime_for(Oid::from(entry.oid)).to_be_bytes())); + out.extend_from_slice(checksum.as_slice()); + let mut hasher = gix_hash::hasher(kind); + hasher.update(&out); + let digest = hasher + .try_finalize() + .map_err(|error| MaintError::Pack(format!("cruft mtimes checksum: {error}")))?; + out.extend_from_slice(digest.as_slice()); + + knot_resource::atomic_write_bytes( + &stem.file(pack_dir, "mtimes"), + &out, + knot_resource::FileMode::Inherited, + )?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn oid(byte: u8) -> Oid { + Oid::from_hex(&format!("{byte:02x}").repeat(20)).unwrap() + } + + #[test] + fn closure_covered_when_every_reachable_oid_survives() { + let reachable: HashSet = [oid(1), oid(2)].into_iter().collect(); + let kept: HashSet = [oid(1), oid(2), oid(3)].into_iter().collect(); + assert!(closure_is_covered(&reachable, &kept)); + } + + #[test] + fn closure_uncovered_when_a_reachable_oid_is_missing() { + let reachable: HashSet = [oid(1), oid(2)].into_iter().collect(); + let kept: HashSet = [oid(1)].into_iter().collect(); + assert!(!closure_is_covered(&reachable, &kept)); + } + + #[test] + fn unknown_mtime_is_never_expired() { + assert!(!is_expired(None, SystemTime::now(), Duration::ZERO)); + } + + #[test] + fn future_mtime_is_never_expired() { + let future = PackedMtime::from_system_time(SystemTime::now()) + .to_unix() + .saturating_add_secs(100_000); + assert!(!is_expired( + Some(&future), + SystemTime::now(), + Duration::from_secs(1) + )); + } + + #[test] + fn newest_mtime_wins_regardless_of_iteration_order() { + let shared = oid(7); + let old = UnixSeconds::new(1_000); + let fresh = UnixSeconds::new(2_000); + let forward = newest_by_oid([(shared, old), (shared, fresh)].into_iter()); + let reverse = newest_by_oid([(shared, fresh), (shared, old)].into_iter()); + assert_eq!(forward.get(&shared), Some(&fresh)); + assert_eq!(reverse.get(&shared), Some(&fresh)); + } + + #[test] + fn old_mtime_past_grace_is_expired() { + let old = UnixSeconds::new(1_000); + assert!(is_expired( + Some(&old), + SystemTime::now(), + Duration::from_secs(60) + )); + } + + #[test] + fn corrupt_mtimes_are_rejected_rather_than_trusted() { + let count = 3usize; + let hash_len = gix::hash::Kind::Sha1.len_in_bytes(); + let total = 12 + count * 4 + hash_len * 2; + let zero_checksum = vec![0u8; hash_len]; + let unsigned = vec![0u8; total]; + assert!( + validated_mtimes(&unsigned, count, gix::hash::Kind::Sha1, &zero_checksum).is_none(), + "a correctly-sized but unsigned mtimes table isn't trusted" + ); + let truncated = vec![0u8; total - 1]; + assert!( + validated_mtimes(&truncated, count, gix::hash::Kind::Sha1, &zero_checksum).is_none() + ); + let mut wrong_pack = vec![0u8; total]; + wrong_pack[0..4].copy_from_slice(&MTIMES_MAGIC.to_be_bytes()); + wrong_pack[4..8].copy_from_slice(&MTIMES_VERSION.to_be_bytes()); + wrong_pack[8..12].copy_from_slice(&hash_id(gix::hash::Kind::Sha1).to_be_bytes()); + let mismatched = vec![0xabu8; hash_len]; + assert!( + validated_mtimes(&wrong_pack, count, gix::hash::Kind::Sha1, &mismatched).is_none(), + "an mtimes table whose pack checksum names a different pack isn't trusted" + ); + } + + #[test] + fn run_fail_closes_when_survivors_miss_the_closure() { + use knot_git::{Layout, RefUpdate}; + use knot_types::{BranchName, RefName, RepoDid}; + + use crate::test_support::{commit_on, empty_tree}; + + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()).with_default_branch(BranchName::new("main").unwrap()); + let did = RepoDid::new("did:plc:limpet").unwrap(); + let repo = layout.create(&did).unwrap(); + let tip = commit_on(&repo, empty_tree(repo.object_format()), Vec::new(), "a"); + repo.update_ref(&RefUpdate::Create { + name: RefName::new("refs/heads/main").unwrap(), + new: tip, + }) + .unwrap(); + + let options = crate::Options { + repack_max_objects: crate::ObjectCount::new(1_000_000), + geometric_factor: crate::GeometricFactor::full_repack(), + prune_grace: crate::PruneGrace::from_secs(0), + reflog_floor: crate::ReflogRetention::from_secs(i64::MAX as u64 / 4), + commit_graph: false, + multi_pack_index: false, + bitmap: false, + }; + crate::run_repo(&repo, UnixSeconds::new(1_700_000_500), &options).unwrap(); + + let objects_dir = repo.objects_dir(); + let kind = repo.object_format().kind(); + let reachable: HashSet = repo + .select_pack_objects(knot_git::Wants::new(&[tip]), knot_git::Haves::new(&[])) + .unwrap() + .into_iter() + .collect(); + assert!(!reachable.is_empty()); + + let absent = PackStem::of(Path::new( + "pack-0000000000000000000000000000000000000000.idx", + )) + .unwrap(); + let report = run( + &objects_dir, + kind, + &reachable, + Some(&absent), + &[], + &[], + Duration::ZERO, + ) + .unwrap(); + assert!(!report.ran, "an uncovered closure fail-closes the prune"); + let reopened = knot_git::Repo::open(repo.git().git_dir()).unwrap(); + assert!( + reachable.iter().all(|oid| reopened.contains(*oid)), + "no reachable object is deleted when survivors don't cover the closure" + ); + } +} diff --git a/knot2/crates/knot-maintenance/src/fsio.rs b/knot2/crates/knot-maintenance/src/fsio.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-maintenance/src/fsio.rs @@ -0,0 +1,178 @@ +use std::path::{Path, PathBuf}; +use std::time::{Duration, SystemTime}; + +use knot_types::{Oid, UnixSeconds}; + +use crate::MaintError; + +pub(crate) const MIDX_SIDECAR_PREFIX: &str = "multi-pack-index-"; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +// Every file in a pack-set has only the one stem, +// so maintenance finds siblings by just swapping the file extension. +// Checking the stem once here is more efficient than +// checking it at each place that makes a sibling path. +pub struct PackStem(String); + +impl PackStem { + pub fn of(path: &Path) -> Option { + path.file_stem() + .and_then(|stem| stem.to_str()) + .filter(|stem| stem.starts_with("pack-") || stem.starts_with(MIDX_SIDECAR_PREFIX)) + .map(|stem| Self(stem.to_string())) + } + + pub(crate) fn midx_sidecar(checksum_hex: &str) -> Self { + Self(format!("{MIDX_SIDECAR_PREFIX}{checksum_hex}")) + } + + pub fn file(&self, pack_dir: &Path, extension: &str) -> PathBuf { + pack_dir.join(format!("{}.{extension}", self.0)) + } +} + +pub fn io_error(path: &Path, error: std::io::Error) -> MaintError { + MaintError::Io { + path: path.to_path_buf(), + message: error.to_string(), + } +} + +pub fn loose_objects(objects_dir: &Path) -> Vec<(Oid, PathBuf)> { + let Ok(shards) = std::fs::read_dir(objects_dir) else { + return Vec::new(); + }; + shards + .filter_map(Result::ok) + .filter(|shard| is_shard_name(&shard.file_name())) + .flat_map(|shard| loose_in_shard(&shard.path(), &shard.file_name())) + .collect() +} + +fn is_shard_name(name: &std::ffi::OsString) -> bool { + name.to_str() + .is_some_and(|text| text.len() == 2 && text.bytes().all(|byte| byte.is_ascii_hexdigit())) +} + +fn loose_in_shard(shard_path: &Path, shard_name: &std::ffi::OsString) -> Vec<(Oid, PathBuf)> { + let Some(prefix) = shard_name.to_str() else { + return Vec::new(); + }; + let Ok(entries) = std::fs::read_dir(shard_path) else { + return Vec::new(); + }; + entries + .filter_map(Result::ok) + .filter(|entry| entry.file_type().is_ok_and(|kind| kind.is_file())) + .filter_map(|entry| { + let name = entry.file_name(); + let rest = name.to_str()?; + let oid = Oid::from_hex(&format!("{prefix}{rest}")).ok()?; + Some((oid, entry.path())) + }) + .collect() +} + +pub fn has_loose_refs(git_dir: &Path) -> bool { + walkdir::WalkDir::new(git_dir.join("refs")) + .into_iter() + .filter_map(Result::ok) + .any(|entry| entry.file_type().is_file()) +} + +pub fn pack_idx_paths(objects_dir: &Path) -> Vec { + let pack_dir = objects_dir.join("pack"); + let Ok(entries) = std::fs::read_dir(&pack_dir) else { + return Vec::new(); + }; + entries + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| path.extension().is_some_and(|ext| ext == "idx")) + .collect() +} + +pub fn pack_oids(idx: &Path, kind: gix::hash::Kind) -> Option> { + let index = gix_pack::index::File::at(idx, kind).ok()?; + Some(index.iter().map(|entry| Oid::from(entry.oid)).collect()) +} + +pub fn pack_file(idx: &Path, pack_dir: &Path) -> Option { + let stem = idx.file_stem().and_then(|stem| stem.to_str())?; + Some(pack_dir.join(format!("{stem}.pack"))) +} + +pub fn remove_pack_files(idx: &Path, pack_dir: &Path) -> bool { + let Some(stem) = idx.file_stem().and_then(|stem| stem.to_str()) else { + return false; + }; + let idx_removed = std::fs::remove_file(idx).is_ok(); + let pack_removed = std::fs::remove_file(pack_dir.join(format!("{stem}.pack"))).is_ok(); + let _ = std::fs::remove_file(pack_dir.join(format!("{stem}.rev"))); + let _ = std::fs::remove_file(pack_dir.join(format!("{stem}.bitmap"))); + let _ = std::fs::remove_file(pack_dir.join(format!("{stem}.mtimes"))); + idx_removed || pack_removed +} + +pub fn pack_mtime(idx: &Path, pack_dir: &Path) -> Option { + let pack = pack_file(idx, pack_dir)?; + let modified = pack.metadata().ok()?.modified().ok()?; + let secs = modified + .duration_since(SystemTime::UNIX_EPOCH) + .ok()? + .as_secs(); + Some(UnixSeconds::new(secs as i64)) +} + +pub fn older_than(path: &Path, grace: Duration) -> bool { + path.metadata() + .and_then(|meta| meta.modified()) + .map(|modified| { + SystemTime::now() + .duration_since(modified) + .unwrap_or(Duration::ZERO) + >= grace + }) + .unwrap_or(false) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn touch(path: &Path) { + std::fs::write(path, b"x").unwrap(); + } + + #[test] + fn remove_pack_files_clears_idx_and_pack_together() { + let dir = tempfile::tempdir().unwrap(); + let pack_dir = dir.path(); + let idx = pack_dir.join("pack-scallop.idx"); + touch(&idx); + touch(&pack_dir.join("pack-scallop.pack")); + touch(&pack_dir.join("pack-scallop.mtimes")); + + assert!(remove_pack_files(&idx, pack_dir)); + assert!(!idx.exists()); + assert!(!pack_dir.join("pack-scallop.pack").exists()); + assert!(!pack_dir.join("pack-scallop.mtimes").exists()); + } + + #[test] + fn remove_pack_files_reclaims_an_orphan_idx_with_no_pack() { + let dir = tempfile::tempdir().unwrap(); + let pack_dir = dir.path(); + let idx = pack_dir.join("pack-whelk.idx"); + touch(&idx); + + assert!( + remove_pack_files(&idx, pack_dir), + "a lone idx left by a crash mid-deletion is still reclaimed" + ); + assert!( + !idx.exists(), + "the orphan idx no longer advertises phantom objects" + ); + } +} diff --git a/knot2/crates/knot-maintenance/src/lib.rs b/knot2/crates/knot-maintenance/src/lib.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-maintenance/src/lib.rs @@ -0,0 +1,413 @@ +use std::collections::HashSet; +use std::path::PathBuf; +use std::time::Duration; + +use knot_git::{PackRefsReport, ReflogReport, Repo}; +use knot_types::{Oid, UnixSeconds}; + +mod bitmap; +mod commitgraph; +mod cruft; +mod fsio; +mod midx; +mod prune; +mod repack; +mod scheduler; +#[cfg(test)] +mod test_support; + +pub use midx::MidxStatus; +pub use scheduler::{MaintenanceHandle, PushBytes, RepoSource, Scheduler}; + +pub const MIN_REFLOG_RETENTION_SECS: i64 = 30 * 24 * 60 * 60; + +#[derive(Debug, thiserror::Error)] +pub enum MaintError { + #[error("git: {0}")] + Git(#[from] knot_git::GitError), + #[error("pack: {0}")] + Pack(String), + #[error("io {path}: {message}")] + Io { path: PathBuf, message: String }, + #[error("commit-graph: {0}")] + CommitGraph(String), +} + +impl From for MaintError { + fn from(error: knot_resource::FsError) -> Self { + MaintError::Io { + path: error.path, + message: error.source.to_string(), + } + } +} + +pub use knot_types::ObjectCount; + +knot_types::scalar_newtype! { + pub struct FileCount(usize); +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct GeometricFactor(u64); + +impl GeometricFactor { + pub const fn new(value: u64) -> Self { + Self(if value < 2 { 2 } else { value }) + } + + pub const fn full_repack() -> Self { + Self(u64::MAX) + } + + pub const fn get(self) -> u64 { + self.0 + } +} + +#[derive(Debug, Clone, Copy)] +pub struct Options { + pub repack_max_objects: ObjectCount, + pub geometric_factor: GeometricFactor, + pub prune_grace: PruneGrace, + pub reflog_floor: ReflogRetention, + pub commit_graph: bool, + pub multi_pack_index: bool, + pub bitmap: bool, +} + +impl Options { + pub fn from_config(config: &knot_config::MaintenanceConfig) -> Self { + Self { + repack_max_objects: ObjectCount::new(config.repack_max_objects as usize), + geometric_factor: GeometricFactor::new(config.repack_geometric_factor), + prune_grace: PruneGrace::from_secs(config.prune_grace_secs), + reflog_floor: ReflogRetention::from_secs(config.reflog_expire_secs), + commit_graph: config.commit_graph, + multi_pack_index: config.multi_pack_index, + bitmap: config.bitmap, + } + } +} + +const LFS_GRACE_MIN: Duration = Duration::from_secs(86_400); + +#[derive(Debug, Clone, Copy)] +pub struct GcGrace(Duration); + +impl GcGrace { + pub const fn from_secs(secs: u64) -> Self { + Self(Duration::from_secs(secs)) + } +} + +#[derive(Debug, Clone, Copy)] +pub struct ReflogRetention(Duration); + +impl ReflogRetention { + pub const fn from_secs(secs: u64) -> Self { + Self(Duration::from_secs(secs)) + } + + pub const fn get(self) -> Duration { + self.0 + } +} + +#[derive(Debug, Clone, Copy)] +pub struct PruneGrace(Duration); + +impl PruneGrace { + pub const fn from_secs(secs: u64) -> Self { + Self(Duration::from_secs(secs)) + } + + pub const fn get(self) -> Duration { + self.0 + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct LfsGrace(Duration); + +impl LfsGrace { + pub const fn get(self) -> Duration { + self.0 + } +} + +#[derive(Debug, Clone, Copy)] +pub struct SweepInterval(Duration); + +impl SweepInterval { + pub const fn new(interval: Duration) -> Self { + Self(interval) + } + + pub const fn get(self) -> Duration { + self.0 + } +} + +pub fn lfs_grace(gc_grace: GcGrace, reflog_retention: ReflogRetention) -> LfsGrace { + let ceiling = reflog_retention + .0 + .max(Duration::from_secs(MIN_REFLOG_RETENTION_SECS as u64)) + .max(LFS_GRACE_MIN); + LfsGrace(gc_grace.0.clamp(LFS_GRACE_MIN, ceiling)) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RepackStatus { + Repacked, + Clean, + SkippedTooLarge, + ClosureFailed, + NothingReachable, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RepackReport { + pub status: RepackStatus, + pub packed_objects: ObjectCount, + pub removed_loose: FileCount, + pub removed_packs: FileCount, +} + +impl RepackReport { + fn skipped(status: RepackStatus) -> Self { + Self { + status, + packed_objects: ObjectCount::new(0), + removed_loose: FileCount::new(0), + removed_packs: FileCount::new(0), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PruneReport { + pub removed: FileCount, + pub removed_packs: FileCount, + pub crufted: ObjectCount, + pub ran: bool, +} + +impl PruneReport { + fn skipped() -> Self { + Self { + removed: FileCount::new(0), + removed_packs: FileCount::new(0), + crufted: ObjectCount::new(0), + ran: false, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Report { + pub packed_refs: PackRefsReport, + pub reflog: ReflogReport, + pub commit_graph: bool, + pub repack: RepackReport, + pub prune: PruneReport, + pub multi_pack_index: MidxStatus, + pub bitmap: bool, +} + +impl Report { + fn noop() -> Self { + Self { + packed_refs: PackRefsReport { packed: 0 }, + reflog: ReflogReport { + files: 0, + dropped: 0, + }, + commit_graph: false, + repack: RepackReport::skipped(RepackStatus::Clean), + prune: PruneReport::skipped(), + multi_pack_index: MidxStatus::Absent, + bitmap: false, + } + } +} + +pub fn run_repo( + repo: &Repo, + now_seconds: UnixSeconds, + opts: &Options, +) -> Result { + let objects_dir = repo.objects_dir(); + let kind = repo.object_format().kind(); + let loose = fsio::loose_objects(&objects_dir); + let pack_count = fsio::pack_idx_paths(&objects_dir).len(); + let loose_refs = fsio::has_loose_refs(repo.git().git_dir()); + + let graph_pending = opts.commit_graph && pack_count >= 1 && !commitgraph::exists(repo); + let bitmap_pending = opts.bitmap && pack_count == 1 && !bitmap::exists(&objects_dir); + if !opts.commit_graph { + commitgraph::remove(repo)?; + } + if loose.is_empty() && pack_count <= 1 && !loose_refs && !graph_pending && !bitmap_pending { + return Ok(Report::noop()); + } + + let packed_refs = if loose_refs { + repo.pack_refs()? + } else { + PackRefsReport { packed: 0 } + }; + let floor_secs = (opts.reflog_floor.get().as_secs() as i64).max(MIN_REFLOG_RETENTION_SECS); + let reflog = repo.expire_reflogs(now_seconds.saturating_sub_secs(floor_secs))?; + + let commit_graph = if opts.commit_graph { + commitgraph::write(repo)? + } else { + false + }; + + let retention_floor = now_seconds.saturating_sub_secs(floor_secs); + let (repack, reachable, roots, new_stem, kept_large) = if loose.is_empty() && pack_count <= 1 { + ( + RepackReport::skipped(RepackStatus::Clean), + None, + HashSet::new(), + None, + Vec::new(), + ) + } else { + let roots = collect_roots(repo, retention_floor)?; + let (report, reachable, new_stem, kept_large) = repack::run( + repo, + &objects_dir, + kind, + roots.iter().copied().collect(), + opts.repack_max_objects, + opts.geometric_factor, + &loose, + )?; + (report, reachable, roots, new_stem, kept_large) + }; + + let prune = match &reachable { + Some(set) => repo.with_ref_lock(|| { + let current = collect_roots(repo, retention_floor)?; + if current != roots { + return Ok(PruneReport::skipped()); + } + if repack.status == RepackStatus::Repacked { + midx::clear(&objects_dir)?; + cruft::run( + &objects_dir, + kind, + set, + new_stem.as_ref(), + &kept_large, + &loose, + opts.prune_grace.get(), + ) + } else { + prune::run(&objects_dir, set, &loose, opts.prune_grace.get()) + } + })?, + None => PruneReport::skipped(), + }; + + let multi_pack_index = if opts.multi_pack_index { + midx::write(repo)? + } else { + MidxStatus::Absent + }; + + let bitmap = if opts.bitmap { + bitmap::refresh(repo, &objects_dir)? + } else { + false + }; + + Ok(Report { + packed_refs, + reflog, + commit_graph, + repack, + prune, + multi_pack_index, + bitmap, + }) +} + +fn collect_roots(repo: &Repo, retention_floor: UnixSeconds) -> Result, MaintError> { + let mut roots: HashSet = repo + .references()? + .into_iter() + .map(|record| record.target) + .collect(); + repo.reflog_updates_since(retention_floor) + .into_iter() + .for_each(|update| { + roots.insert(update.new); + if let Some(old) = update.old { + roots.insert(old); + } + }); + Ok(roots + .into_iter() + .filter(|oid| repo.contains(*oid)) + .collect()) +} + +#[cfg(test)] +mod tests { + use super::{GcGrace, LFS_GRACE_MIN, MIN_REFLOG_RETENTION_SECS, ReflogRetention, lfs_grace}; + + #[test] + fn the_default_grace_is_not_clamped_by_the_coupling() { + let fourteen_days = 14 * 86_400; + let ninety_days = 90 * 86_400; + assert_eq!( + lfs_grace( + GcGrace::from_secs(fourteen_days), + ReflogRetention::from_secs(ninety_days) + ) + .get() + .as_secs(), + fourteen_days, + "the 14-day default is within the window and never clamped" + ); + } + + #[test] + fn a_small_grace_is_clamped_to_the_hard_minimum() { + assert_eq!( + lfs_grace( + GcGrace::from_secs(0), + ReflogRetention::from_secs(90 * 86_400) + ) + .get(), + LFS_GRACE_MIN + ); + assert_eq!( + lfs_grace( + GcGrace::from_secs(60), + ReflogRetention::from_secs(90 * 86_400) + ) + .get(), + LFS_GRACE_MIN + ); + } + + #[test] + fn grace_never_exceeds_the_reflog_retention() { + let short_reflog = MIN_REFLOG_RETENTION_SECS as u64; + assert_eq!( + lfs_grace( + GcGrace::from_secs(u64::MAX), + ReflogRetention::from_secs(short_reflog) + ) + .get() + .as_secs(), + short_reflog, + "a grace above the reflog retention is clamped to it" + ); + } +} diff --git a/knot2/crates/knot-maintenance/src/midx.rs b/knot2/crates/knot-maintenance/src/midx.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-maintenance/src/midx.rs @@ -0,0 +1,230 @@ +use std::path::{Path, PathBuf}; +use std::sync::atomic::AtomicBool; + +use gix::progress::Discard; +use knot_git::Repo; + +use crate::fsio; +use crate::{FileCount, MaintError}; + +const FILE_NAME: &str = "multi-pack-index"; + +pub(crate) const MIDX_ALLOC_LIMIT_BYTES: usize = 16 * 1024 * 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MidxStatus { + Written(FileCount), + Removed, + Absent, +} + +pub(crate) fn clear(objects_dir: &Path) -> Result<(), MaintError> { + let pack_dir = objects_dir.join("pack"); + knot_resource::clear_temps(&pack_dir, FILE_NAME); + knot_resource::fsync_path(&pack_dir).map_err(Into::into) +} + +pub fn write(repo: &Repo) -> Result { + let objects_dir = repo.objects_dir(); + let kind = repo.object_format().kind(); + let target = objects_dir.join("pack").join(FILE_NAME); + let idx_paths = fsio::pack_idx_paths(&objects_dir); + match FileCount::new(idx_paths.len()) { + count if count.get() >= 2 => { + write_atomic(idx_paths, kind, &target)?; + Ok(MidxStatus::Written(count)) + } + _ => remove_if_present(&target), + } +} + +fn write_atomic( + idx_paths: Vec, + kind: gix::hash::Kind, + target: &Path, +) -> Result<(), MaintError> { + knot_resource::atomic_write(target, knot_resource::FileMode::Inherited, |file| { + let mut writer = std::io::BufWriter::new(file); + gix_pack::multi_index::write_from_index_paths( + idx_paths, + &mut writer, + &mut Discard, + &AtomicBool::new(false), + gix_pack::multi_index::write::Options { object_hash: kind }, + ) + .map_err(|e| MaintError::Pack(e.to_string()))?; + std::io::Write::flush(&mut writer).map_err(|e| fsio::io_error(target, e)) + }) +} + +fn remove_if_present(target: &Path) -> Result { + match std::fs::remove_file(target) { + Ok(()) => { + if let Some(dir) = target.parent() { + knot_resource::fsync_path(dir)?; + } + Ok(MidxStatus::Removed) + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(MidxStatus::Absent), + Err(error) => Err(fsio::io_error(target, error)), + } +} + +#[cfg(test)] +mod tests { + use knot_git::{Layout, RefUpdate}; + use knot_types::{BranchName, ObjectFormat, Oid, RefName, RepoDid}; + + use super::*; + use crate::test_support::{commit_on, empty_tree}; + + fn midx_file(repo: &knot_git::Repo) -> PathBuf { + repo.objects_dir().join("pack").join(FILE_NAME) + } + + fn pack_from(repo: &knot_git::Repo, tip: Oid, kind: gix::hash::Kind) { + let closure = repo + .select_pack_objects(knot_git::Wants::new(&[tip]), knot_git::Haves::new(&[])) + .unwrap(); + let mut bytes = Vec::new(); + knot_pack::write_pack(&repo.objects_dir(), closure, None, &mut bytes, kind).unwrap(); + knot_pack::ingest_pack( + &repo.objects_dir(), + &bytes, + &knot_pack::PackLimits::default(), + kind, + ) + .unwrap(); + } + + fn two_pack_repo(format: ObjectFormat) -> (tempfile::TempDir, knot_git::Repo, Oid, Oid) { + let dir = tempfile::tempdir().unwrap(); + let layout = Layout::new(dir.path()) + .with_object_format(format) + .with_default_branch(BranchName::new("main").unwrap()); + let did = RepoDid::new("did:plc:limpet").unwrap(); + let repo = layout.create(&did).unwrap(); + let kind = format.kind(); + let first = commit_on(&repo, empty_tree(format), Vec::new(), "scallop"); + repo.update_ref(&RefUpdate::Create { + name: RefName::new("refs/heads/main").unwrap(), + new: first, + }) + .unwrap(); + pack_from(&repo, first, kind); + let second = commit_on(&repo, empty_tree(format), Vec::new(), "whelk"); + repo.update_ref(&RefUpdate::Create { + name: RefName::new("refs/heads/side").unwrap(), + new: second, + }) + .unwrap(); + pack_from(&repo, second, kind); + (dir, layout.open(&did).unwrap(), first, second) + } + + fn assert_two_pack_midx(format: ObjectFormat) { + let (_dir, repo, scallop, whelk) = two_pack_repo(format); + assert_eq!(fsio::pack_idx_paths(&repo.objects_dir()).len(), 2); + + let status = write(&repo).unwrap(); + let count = match status { + MidxStatus::Written(count) => count, + other => panic!("expected a written midx, got {other:?}"), + }; + assert_eq!(count, FileCount::new(2)); + assert!(midx_file(&repo).exists()); + + let parsed = + gix_pack::multi_index::File::at(midx_file(&repo), Some(MIDX_ALLOC_LIMIT_BYTES)) + .unwrap(); + assert_eq!(parsed.num_indices() as usize, 2); + assert!(parsed.num_objects() >= 4); + + assert!(repo.contains(scallop)); + assert!(repo.contains(whelk)); + assert_eq!(repo.object_format().kind(), format.kind()); + } + + #[test] + fn writes_a_multi_pack_index_over_two_packs_sha1() { + assert_two_pack_midx(ObjectFormat::SHA1); + } + + #[test] + fn writes_a_multi_pack_index_over_two_packs_sha256() { + assert_two_pack_midx(ObjectFormat::SHA256); + } + + #[test] + fn clear_removes_the_index_and_sidecars_but_keeps_packs() { + let dir = tempfile::tempdir().unwrap(); + let objects_dir = dir.path(); + let pack_dir = objects_dir.join("pack"); + std::fs::create_dir_all(&pack_dir).unwrap(); + let make = |name: &str| std::fs::write(pack_dir.join(name), b"x").unwrap(); + make(FILE_NAME); + make("multi-pack-index-abc.bitmap"); + make("multi-pack-index-abc.rev"); + make("pack-scallop.idx"); + make("pack-scallop.pack"); + + clear(objects_dir).unwrap(); + + assert!(!pack_dir.join(FILE_NAME).exists()); + assert!(!pack_dir.join("multi-pack-index-abc.bitmap").exists()); + assert!(!pack_dir.join("multi-pack-index-abc.rev").exists()); + assert!( + pack_dir.join("pack-scallop.idx").exists(), + "real packs are left in place" + ); + assert!(pack_dir.join("pack-scallop.pack").exists()); + } + + #[test] + fn lookup_resolves_through_the_midx_once_idx_files_are_gone() { + let (_dir, repo, scallop, whelk) = two_pack_repo(ObjectFormat::SHA1); + let objects_dir = repo.objects_dir(); + assert!(matches!(write(&repo).unwrap(), MidxStatus::Written(_))); + + fsio::loose_objects(&objects_dir) + .iter() + .for_each(|(_, path)| std::fs::remove_file(path).unwrap()); + fsio::pack_idx_paths(&objects_dir) + .iter() + .for_each(|idx| std::fs::remove_file(idx).unwrap()); + assert!( + midx_file(&repo).exists(), + "the multi-pack-index is the only index left on disk" + ); + + let via_midx = knot_git::Repo::open(repo.git().git_dir()).unwrap(); + assert!( + via_midx.contains(scallop) && via_midx.contains(whelk), + "objects resolve through the multi-pack-index with no per-pack idx present" + ); + + std::fs::remove_file(midx_file(&repo)).unwrap(); + let bare = knot_git::Repo::open(repo.git().git_dir()).unwrap(); + assert!( + !bare.contains(scallop) && !bare.contains(whelk), + "with the index gone the packs are unreadable, proving the midx served the lookup" + ); + } + + #[test] + fn fewer_than_two_packs_writes_nothing_and_clears_stale() { + let dir = tempfile::tempdir().unwrap(); + let layout = Layout::new(dir.path()).with_default_branch(BranchName::new("main").unwrap()); + let did = RepoDid::new("did:plc:conch").unwrap(); + let repo = layout.create(&did).unwrap(); + + assert_eq!(write(&repo).unwrap(), MidxStatus::Absent); + assert!(!midx_file(&repo).exists()); + + let target = midx_file(&repo); + std::fs::create_dir_all(target.parent().unwrap()).unwrap(); + std::fs::write(&target, b"stale").unwrap(); + assert_eq!(write(&repo).unwrap(), MidxStatus::Removed); + assert!(!midx_file(&repo).exists()); + } +} diff --git a/knot2/crates/knot-maintenance/src/prune.rs b/knot2/crates/knot-maintenance/src/prune.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-maintenance/src/prune.rs @@ -0,0 +1,31 @@ +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use knot_types::Oid; + +use crate::fsio; +use crate::{FileCount, MaintError, ObjectCount, PruneReport}; + +pub fn run( + objects_dir: &Path, + reachable: &HashSet, + loose: &[(Oid, PathBuf)], + grace: Duration, +) -> Result { + let removed = loose + .iter() + .filter(|(oid, _)| !reachable.contains(oid)) + .filter(|(_, path)| fsio::older_than(path, grace)) + .filter(|(_, path)| std::fs::remove_file(path).is_ok()) + .count(); + if removed > 0 { + knot_resource::fsync_path(objects_dir)?; + } + Ok(PruneReport { + removed: FileCount::new(removed), + removed_packs: FileCount::new(0), + crufted: ObjectCount::new(0), + ran: true, + }) +} diff --git a/knot2/crates/knot-maintenance/src/repack.rs b/knot2/crates/knot-maintenance/src/repack.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-maintenance/src/repack.rs @@ -0,0 +1,262 @@ +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use std::sync::atomic::AtomicBool; + +use gix::progress::Discard; +use knot_git::{Haves, Repo, Wants}; +use knot_types::Oid; + +use crate::fsio::{self, PackStem}; +use crate::{FileCount, GeometricFactor, MaintError, ObjectCount, RepackReport, RepackStatus}; + +type RepackOutcome = ( + RepackReport, + Option>, + Option, + Vec, +); + +pub fn run( + repo: &Repo, + objects_dir: &Path, + kind: gix::hash::Kind, + roots: Vec, + max_objects: ObjectCount, + factor: GeometricFactor, + loose: &[(Oid, PathBuf)], +) -> Result { + let closure = match repo.select_pack_objects(Wants::new(&roots), Haves::new(&[])) { + Ok(closure) => closure, + Err(_) => { + return Ok(( + RepackReport::skipped(RepackStatus::ClosureFailed), + None, + None, + Vec::new(), + )); + } + }; + if closure.is_empty() { + return Ok(( + RepackReport::skipped(RepackStatus::NothingReachable), + Some(HashSet::new()), + None, + Vec::new(), + )); + } + if closure.len() > max_objects.get() { + return Ok(( + RepackReport::skipped(RepackStatus::SkippedTooLarge), + None, + None, + Vec::new(), + )); + } + let reachable: HashSet = closure.iter().copied().collect(); + + let kept_large = if factor == GeometricFactor::full_repack() { + Vec::new() + } else { + kept_large_packs(objects_dir, kind, factor) + }; + let kept_large_oids: HashSet = kept_large + .iter() + .filter_map(|idx| fsio::pack_oids(idx, kind)) + .flatten() + .collect(); + let new_objects: Vec = closure + .into_iter() + .filter(|oid| !kept_large_oids.contains(oid)) + .collect(); + + let new_stem = if new_objects.is_empty() { + None + } else { + build_and_install_pack(objects_dir, new_objects, kind)? + }; + + let removed_loose = loose + .iter() + .filter(|(oid, _)| reachable.contains(oid)) + .filter(|(_, path)| std::fs::remove_file(path).is_ok()) + .count(); + + let kept_large_stems: Vec = kept_large + .iter() + .filter_map(|idx| PackStem::of(idx)) + .collect(); + + Ok(( + RepackReport { + status: RepackStatus::Repacked, + packed_objects: ObjectCount::new(reachable.len()), + removed_loose: FileCount::new(removed_loose), + removed_packs: FileCount::new(0), + }, + Some(reachable), + new_stem, + kept_large_stems, + )) +} + +fn kept_large_packs( + objects_dir: &Path, + kind: gix::hash::Kind, + factor: GeometricFactor, +) -> Vec { + let pack_dir = objects_dir.join("pack"); + let mut eligible: Vec<(PathBuf, usize)> = fsio::pack_idx_paths(objects_dir) + .into_iter() + .filter(|idx| !is_excluded(idx, &pack_dir)) + .filter_map(|idx| fsio::pack_oids(&idx, kind).map(|oids| (idx, oids.len()))) + .collect(); + eligible.sort_by_key(|(_, count)| *count); + let weights: Vec = eligible.iter().map(|(_, count)| *count).collect(); + let split = compute_split(&weights, factor); + eligible + .into_iter() + .skip(split) + .map(|(idx, _)| idx) + .collect() +} + +fn is_excluded(idx: &Path, pack_dir: &Path) -> bool { + let Some(stem) = idx.file_stem().and_then(|stem| stem.to_str()) else { + return true; + }; + pack_dir.join(format!("{stem}.mtimes")).exists() +} + +fn compute_split(weights: &[usize], factor: GeometricFactor) -> usize { + let n = weights.len(); + if n == 0 { + return 0; + } + let geometric = + |big: usize, small: usize| (small as u64).saturating_mul(factor.get()) <= big as u64; + let split1 = (1..n) + .rev() + .find(|&i| !geometric(weights[i], weights[i - 1])) + .map(|i| i + 1) + .unwrap_or(0); + let total: u64 = weights[..split1].iter().map(|weight| *weight as u64).sum(); + let extended = (split1..n).try_fold((split1, total), |(split, total), j| { + match total.checked_mul(factor.get()) { + Some(threshold) if (weights[j] as u64) < threshold => std::ops::ControlFlow::Continue( + (split + 1, total.saturating_add(weights[j] as u64)), + ), + _ => std::ops::ControlFlow::Break((split, total)), + } + }); + match extended { + std::ops::ControlFlow::Continue((split, _)) => split, + std::ops::ControlFlow::Break((split, _)) => split, + } +} + +fn build_and_install_pack( + objects_dir: &Path, + closure: Vec, + kind: gix::hash::Kind, +) -> Result, MaintError> { + let pack_dir = objects_dir.join("pack"); + std::fs::create_dir_all(&pack_dir).map_err(|error| fsio::io_error(&pack_dir, error))?; + knot_resource::clear_stale(&pack_dir, ".knot-repack."); + let staging = pack_dir.join(format!( + ".knot-repack.{}.pack", + knot_resource::staging_nonce() + )); + let outcome = write_streaming_pack(objects_dir, closure, kind, &staging) + .and_then(|()| install_streamed_pack(&pack_dir, &staging, kind)); + let _ = std::fs::remove_file(&staging); + outcome +} + +fn write_streaming_pack( + objects_dir: &Path, + closure: Vec, + kind: gix::hash::Kind, + staging: &Path, +) -> Result<(), MaintError> { + let file = std::fs::File::create(staging).map_err(|error| fsio::io_error(staging, error))?; + let mut writer = std::io::BufWriter::new(file); + knot_pack::write_pack(objects_dir, closure, None, &mut writer, kind) + .map_err(|error| MaintError::Pack(error.to_string()))?; + writer + .into_inner() + .map(|_| ()) + .map_err(|error| fsio::io_error(staging, error.into_error())) +} + +fn install_streamed_pack( + pack_dir: &Path, + staging: &Path, + kind: gix::hash::Kind, +) -> Result, MaintError> { + let file = std::fs::File::open(staging).map_err(|error| fsio::io_error(staging, error))?; + let mut reader = std::io::BufReader::new(file); + let outcome = gix_pack::Bundle::write_to_directory( + &mut reader, + Some(pack_dir), + &mut Discard, + &AtomicBool::new(false), + None::, + gix_pack::bundle::write::Options { + thread_limit: Some(1), + iteration_mode: gix_pack::data::input::Mode::Verify, + index_version: gix_pack::index::Version::default(), + object_hash: kind, + }, + ) + .map_err(|error| MaintError::Pack(error.to_string()))?; + + if let Some(keep) = &outcome.keep_path { + let _ = std::fs::remove_file(keep); + } + [&outcome.data_path, &outcome.index_path] + .into_iter() + .flatten() + .try_for_each(|path| knot_resource::fsync_path(path))?; + knot_resource::fsync_path(pack_dir)?; + + Ok(outcome + .data_path + .as_ref() + .and_then(|path| PackStem::of(path))) +} + +#[cfg(test)] +mod tests { + use super::{GeometricFactor, compute_split}; + + #[test] + fn uniform_small_packs_all_roll_up() { + assert_eq!(compute_split(&[1, 1, 1, 1], GeometricFactor::new(2)), 4); + } + + #[test] + fn a_clean_geometric_progression_rolls_up_nothing() { + assert_eq!(compute_split(&[1, 2, 4, 8], GeometricFactor::new(2)), 0); + } + + #[test] + fn one_large_pack_with_tiny_additions_keeps_the_large_one() { + let split = compute_split(&[1, 1, 1, 1000], GeometricFactor::new(2)); + assert_eq!(split, 3); + } + + #[test] + fn a_single_pack_is_never_rolled_up() { + assert_eq!(compute_split(&[42], GeometricFactor::new(2)), 0); + } + + #[test] + fn empty_input_rolls_up_nothing() { + assert_eq!(compute_split(&[], GeometricFactor::new(2)), 0); + } + + #[test] + fn a_huge_factor_saturates_without_overflow() { + assert_eq!(compute_split(&[1, 1, 1], GeometricFactor::full_repack()), 3); + } +} diff --git a/knot2/crates/knot-maintenance/src/scheduler.rs b/knot2/crates/knot-maintenance/src/scheduler.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-maintenance/src/scheduler.rs @@ -0,0 +1,515 @@ +use std::collections::HashSet; +use std::sync::Arc; +use std::time::{Duration, SystemTime}; + +use knot_git::Layout; +use knot_lfs::DiskStore; +use knot_runtime::Clock; +use knot_types::{RepoDid, UnixSeconds}; +use tokio::sync::{mpsc, watch}; + +use crate::{LfsGrace, Options, RepackStatus, Report, SweepInterval, run_repo}; + +pub trait RepoSource: Send + Sync + 'static { + fn repos(&self) -> Vec; + fn ready_repos(&self) -> Option>; +} + +// "Grace" means how old an unreferenced object can get before we reap it, +// and the "interval" is how often we look. +struct LfsGc { + store: Arc, + grace: LfsGrace, + interval: SweepInterval, +} + +const TRIGGER_CAPACITY: usize = 1024; + +knot_types::scalar_newtype! { + pub struct PushBytes(u64) => ordered; +} + +#[derive(Clone)] +pub struct MaintenanceHandle { + trigger: Option>, + large_push: PushBytes, +} + +impl MaintenanceHandle { + pub fn disabled() -> Self { + Self { + trigger: None, + large_push: PushBytes::new(u64::MAX), + } + } + + pub fn note_push(&self, repo: &RepoDid, pack_bytes: PushBytes) { + if pack_bytes >= self.large_push + && let Some(trigger) = &self.trigger + { + let _ = trigger.try_send(repo.clone()); + } + } +} + +pub struct Scheduler { + layout: Layout, + source: Arc, + clock: C, + options: Options, + interval: Duration, + triggers: mpsc::Receiver, + lfs: Option, +} + +impl Scheduler { + pub fn new( + layout: Layout, + source: Arc, + clock: C, + options: Options, + interval: Duration, + large_push: PushBytes, + ) -> (Self, MaintenanceHandle) { + let (trigger, triggers) = mpsc::channel(TRIGGER_CAPACITY); + let handle = MaintenanceHandle { + trigger: Some(trigger), + large_push, + }; + let scheduler = Self { + layout, + source, + clock, + options, + interval, + triggers, + lfs: None, + }; + (scheduler, handle) + } + + pub fn with_lfs_gc( + mut self, + store: Arc, + grace: LfsGrace, + interval: SweepInterval, + ) -> Self { + self.lfs = Some(LfsGc { + store, + grace, + interval, + }); + self + } + + fn now_seconds(&self) -> UnixSeconds { + UnixSeconds::new((self.clock.now_unix_micros().get() / 1_000_000) as i64) + } + + fn now(&self) -> SystemTime { + SystemTime::UNIX_EPOCH + Duration::from_micros(self.clock.now_unix_micros().get()) + } + + async fn gc_repo(&self, repo: &RepoDid) -> knot_lfs::GcReport { + let Some(lfs) = &self.lfs else { + return knot_lfs::GcReport::default(); + }; + let layout = self.layout.clone(); + let store = Arc::clone(&lfs.store); + let grace = lfs.grace.get(); + let now = self.now(); + let target = repo.clone(); + let started = std::time::Instant::now(); + let outcome = tokio::task::spawn_blocking(move || { + layout + .open(&target) + .map_err(knot_lfs::GcError::from) + .and_then(|opened| knot_lfs::collect_repo(&store, &opened, &target, grace, now)) + }) + .await; + match outcome { + Ok(Ok(report)) => { + if report.swept > 0 { + tracing::info!( + repo = %repo, + scanned = report.scanned, + marked = report.marked, + swept = report.swept, + bytes = report.bytes.get(), + duration_ms = started.elapsed().as_millis() as u64, + "lfs gc reclaimed objects" + ); + } + report + } + Ok(Err(error)) => { + tracing::warn!(repo = %repo, %error, "lfs gc skipped, projection uncertain"); + knot_lfs::GcReport::default() + } + Err(join) => { + tracing::error!(repo = %repo, %join, "lfs gc task panicked"); + knot_lfs::GcReport::default() + } + } + } + + async fn sweep_orphans(&self) { + let Some(lfs) = &self.lfs else { + return; + }; + let Some(hosted) = self.source.ready_repos() else { + return; + }; + let store = Arc::clone(&lfs.store); + let grace = lfs.grace.get(); + let now = self.now(); + let hosted: HashSet = hosted.into_iter().collect(); + let outcome = + tokio::task::spawn_blocking(move || store.sweep_orphans(&hosted, grace, now)).await; + match outcome { + Ok(Ok(sweep)) if sweep.prefixes > 0 => tracing::info!( + prefixes = sweep.prefixes, + objects = sweep.objects, + bytes = sweep.bytes.get(), + "lfs gc reclaimed orphan prefixes" + ), + Ok(Ok(_)) => {} + Ok(Err(error)) => tracing::warn!(%error, "lfs orphan sweep failed"), + Err(join) => tracing::error!(%join, "lfs orphan sweep task panicked"), + } + } + + pub async fn run(mut self, mut shutdown: watch::Receiver) { + let mut tick = tokio::time::interval(self.interval); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + tick.tick().await; + let mut lfs_tick = self.lfs.as_ref().map(|lfs| { + let mut lfs_tick = tokio::time::interval(lfs.interval.get()); + lfs_tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + lfs_tick + }); + if let Some(lfs_tick) = &mut lfs_tick { + lfs_tick.tick().await; + } + let probe = shutdown.clone(); + loop { + tokio::select! { + biased; + _ = shutdown.changed() => break, + _ = tick.tick() => self.sweep_all(&probe).await, + _ = tick_lfs(&mut lfs_tick) => self.lfs_sweep_all(&probe).await, + Some(repo) = self.triggers.recv() => self.maintain_batch(repo).await, + } + } + } + + async fn sweep_all(&self, shutdown: &watch::Receiver) { + for repo in self.source.repos() { + if *shutdown.borrow() { + return; + } + self.maintain_one(repo).await; + } + } + + async fn lfs_sweep_all(&self, shutdown: &watch::Receiver) { + if self.lfs.is_none() { + return; + } + let started = std::time::Instant::now(); + let mut totals = knot_lfs::GcReport::default(); + let mut repos = 0usize; + for repo in self.source.repos() { + if *shutdown.borrow() { + return; + } + let report = self.gc_repo(&repo).await; + repos += 1; + totals.scanned += report.scanned; + totals.marked += report.marked; + totals.swept += report.swept; + totals.bytes = totals.bytes.saturating_add(report.bytes); + } + if !*shutdown.borrow() { + self.sweep_orphans().await; + } + tracing::info!( + repos, + scanned = totals.scanned, + marked = totals.marked, + swept = totals.swept, + bytes = totals.bytes.get(), + duration_ms = started.elapsed().as_millis() as u64, + "lfs gc pass finished" + ); + } + + async fn maintain_batch(&mut self, first: RepoDid) { + let pending: HashSet = std::iter::once(first) + .chain(std::iter::from_fn(|| self.triggers.try_recv().ok())) + .collect(); + for repo in pending { + self.maintain_one(repo.clone()).await; + self.gc_repo(&repo).await; + } + } + + async fn maintain_one(&self, repo: RepoDid) { + let layout = self.layout.clone(); + let options = self.options; + let now_seconds = self.now_seconds(); + let target = repo.clone(); + let outcome = tokio::task::spawn_blocking(move || { + layout + .open(&target) + .map_err(crate::MaintError::from) + .and_then(|opened| run_repo(&opened, now_seconds, &options)) + }) + .await; + match outcome { + Ok(Ok(report)) => report_skips(&repo, &report), + Ok(Err(error)) => tracing::error!(repo = %repo, %error, "maintenance run failed"), + Err(join) => tracing::error!(repo = %repo, %join, "maintenance task panicked"), + } + } +} + +async fn tick_lfs(tick: &mut Option) { + match tick { + Some(tick) => { + tick.tick().await; + } + None => std::future::pending::<()>().await, + } +} + +fn report_skips(repo: &RepoDid, report: &Report) { + match report.repack.status { + RepackStatus::SkippedTooLarge => { + tracing::warn!( + repo = %repo, + reason = "reachable set exceeds repack_max_objects", + "skipped repack" + ) + } + RepackStatus::ClosureFailed => { + tracing::warn!( + repo = %repo, + reason = "couldn't compute reachable set", + "skipped repack and prune" + ) + } + _ => {} + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use knot_git::{Layout, RefUpdate}; + use knot_runtime::SystemClock; + use knot_types::{BranchName, RefName, RepoDid}; + + use super::{MaintenanceHandle, PushBytes, RepoSource, Scheduler}; + use crate::test_support::{commit_on, empty_tree, options}; + + struct Fixed(Vec); + impl RepoSource for Fixed { + fn repos(&self) -> Vec { + self.0.clone() + } + + fn ready_repos(&self) -> Option> { + Some(self.0.clone()) + } + } + + fn seed_repo(layout: &Layout, did: &RepoDid) { + let repo = layout.create(did).unwrap(); + let tip = commit_on(&repo, empty_tree(repo.object_format()), Vec::new(), "a"); + repo.update_ref(&RefUpdate::Create { + name: RefName::new("refs/heads/main").unwrap(), + new: tip, + }) + .unwrap(); + } + + #[test] + fn note_push_fires_only_past_the_threshold() { + let (sender, mut receiver) = tokio::sync::mpsc::channel(16); + let handle = MaintenanceHandle { + trigger: Some(sender), + large_push: PushBytes::new(1_000), + }; + let did = RepoDid::new("did:plc:squid").unwrap(); + handle.note_push(&did, PushBytes::new(999)); + assert!(receiver.try_recv().is_err(), "small push is ignored"); + handle.note_push(&did, PushBytes::new(1_000)); + assert_eq!(receiver.try_recv().unwrap(), did, "large push triggers"); + } + + #[test] + fn disabled_handle_never_triggers() { + let did = RepoDid::new("did:plc:squid").unwrap(); + MaintenanceHandle::disabled().note_push(&did, PushBytes::new(u64::MAX)); + } + + #[tokio::test] + async fn the_lfs_interval_collects_without_a_push_trigger() { + use std::time::{Duration, SystemTime, UNIX_EPOCH}; + + use knot_lfs::{DiskStore, LfsOid, LfsStore, LfsStorePath}; + use knot_runtime::{ManualClock, UnixMicros}; + use sha2::{Digest, Sha256}; + + let scan = tempfile::tempdir().unwrap(); + let lfs_dir = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()).with_default_branch(BranchName::new("main").unwrap()); + let did = RepoDid::new("did:plc:limpet").unwrap(); + seed_repo(&layout, &did); + + let store = Arc::new(DiskStore::open(LfsStorePath::new(lfs_dir.path())).unwrap()); + let body: &[u8] = b"unreferenced media reclaimed on the interval alone"; + let oid = LfsOid::from_digest(Sha256::digest(body).into()); + let size = knot_lfs::ClaimedSize::new(body.len() as u64); + store.put(&did, &oid, size, &mut &body[..]).unwrap(); + + let real_micros = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_micros() as u64; + let future = ManualClock::new(UnixMicros::new(real_micros + 5 * 86_400 * 1_000_000)); + + let (scheduler, _handle) = Scheduler::new( + layout.clone(), + Arc::new(Fixed(vec![did.clone()])), + future, + options(), + std::time::Duration::from_secs(3_600), + PushBytes::new(1_000), + ); + let scheduler = scheduler.with_lfs_gc( + Arc::clone(&store), + crate::lfs_grace( + crate::GcGrace::from_secs(86_400), + crate::ReflogRetention::from_secs(90 * 86_400), + ), + crate::SweepInterval::new(Duration::from_millis(40)), + ); + let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); + let task = tokio::spawn(scheduler.run(shutdown_rx)); + + let mut waited = 0; + while store.probe(&did, &oid).unwrap().is_some() && waited < 200 { + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + waited += 1; + } + assert_eq!( + store.probe(&did, &oid).unwrap(), + None, + "the lfs interval alone reclaimed the unreferenced, past-grace object" + ); + + shutdown_tx.send(true).unwrap(); + task.await.unwrap(); + } + + #[tokio::test] + async fn a_triggered_push_collects_an_unreferenced_lfs_object() { + use std::time::{Duration, SystemTime, UNIX_EPOCH}; + + use knot_lfs::{DiskStore, LfsOid, LfsStore, LfsStorePath}; + use knot_runtime::{ManualClock, UnixMicros}; + use sha2::{Digest, Sha256}; + + let scan = tempfile::tempdir().unwrap(); + let lfs_dir = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()).with_default_branch(BranchName::new("main").unwrap()); + let did = RepoDid::new("did:plc:limpet").unwrap(); + seed_repo(&layout, &did); + + let store = Arc::new(DiskStore::open(LfsStorePath::new(lfs_dir.path())).unwrap()); + let body: &[u8] = b"unreferenced media the sweep should reclaim"; + let oid = LfsOid::from_digest(Sha256::digest(body).into()); + let size = knot_lfs::ClaimedSize::new(body.len() as u64); + store.put(&did, &oid, size, &mut &body[..]).unwrap(); + + let real_micros = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_micros() as u64; + let future = ManualClock::new(UnixMicros::new(real_micros + 5 * 86_400 * 1_000_000)); + + let (scheduler, handle) = Scheduler::new( + layout.clone(), + Arc::new(Fixed(vec![did.clone()])), + future, + options(), + std::time::Duration::from_secs(3_600), + PushBytes::new(1_000), + ); + let scheduler = scheduler.with_lfs_gc( + Arc::clone(&store), + crate::lfs_grace( + crate::GcGrace::from_secs(86_400), + crate::ReflogRetention::from_secs(90 * 86_400), + ), + crate::SweepInterval::new(Duration::from_secs(3_600)), + ); + let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); + let task = tokio::spawn(scheduler.run(shutdown_rx)); + + handle.note_push(&did, PushBytes::new(10_000)); + + let mut waited = 0; + while store.probe(&did, &oid).unwrap().is_some() && waited < 200 { + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + waited += 1; + } + assert_eq!( + store.probe(&did, &oid).unwrap(), + None, + "the triggered gc reclaimed the unreferenced, past-grace object" + ); + + shutdown_tx.send(true).unwrap(); + task.await.unwrap(); + } + + #[tokio::test] + async fn a_triggered_repo_is_maintained() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()).with_default_branch(BranchName::new("main").unwrap()); + let did = RepoDid::new("did:plc:limpet").unwrap(); + seed_repo(&layout, &did); + + let (scheduler, handle) = Scheduler::new( + layout.clone(), + Arc::new(Fixed(vec![did.clone()])), + SystemClock, + options(), + std::time::Duration::from_secs(3_600), + PushBytes::new(1_000), + ); + let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); + let task = tokio::spawn(scheduler.run(shutdown_rx)); + + handle.note_push(&did, PushBytes::new(10_000)); + + let graph = layout + .open(&did) + .unwrap() + .objects_dir() + .join("info/commit-graph"); + let mut waited = 0; + while !graph.exists() && waited < 200 { + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + waited += 1; + } + assert!(graph.exists(), "triggered repo got commit-graph"); + + shutdown_tx.send(true).unwrap(); + task.await.unwrap(); + } +} diff --git a/knot2/crates/knot-maintenance/src/test_support.rs b/knot2/crates/knot-maintenance/src/test_support.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-maintenance/src/test_support.rs @@ -0,0 +1,53 @@ +use knot_git::{EntryKind, Identity, NewCommit, Repo, StagedAction, StagedChange}; +use knot_types::{AuthorName, Email, ObjectFormat, Oid, UnixSeconds}; + +use crate::{GeometricFactor, ObjectCount, Options, PruneGrace, ReflogRetention}; + +pub fn identity() -> Identity { + Identity { + name: AuthorName::new("nel"), + email: Email::new("nel@oyster.cafe"), + time: UnixSeconds::new(1_700_000_000), + offset_seconds: 0, + } +} + +pub fn empty_tree(format: ObjectFormat) -> Oid { + Oid::from(gix::ObjectId::empty_tree(format.kind())) +} + +pub fn commit_on(repo: &Repo, empty_tree: Oid, parents: Vec, marker: &str) -> Oid { + let tree = repo + .write_staged_tree( + empty_tree, + &[StagedChange { + path: knot_types::RepoPath::new(format!("{marker}.txt")).unwrap(), + action: StagedAction::Put { + content: marker.as_bytes().to_vec(), + kind: EntryKind::Blob, + }, + }], + ) + .unwrap(); + repo.write_commit(&NewCommit { + tree, + parents, + author: identity(), + committer: identity(), + message: marker.to_string(), + extra_headers: Vec::new(), + }) + .unwrap() +} + +pub fn options() -> Options { + Options { + repack_max_objects: ObjectCount::new(1_000_000), + geometric_factor: GeometricFactor::full_repack(), + prune_grace: PruneGrace::from_secs(0), + reflog_floor: ReflogRetention::from_secs(i64::MAX as u64 / 4), + commit_graph: true, + multi_pack_index: true, + bitmap: true, + } +} diff --git a/knot2/crates/knot-maintenance/tests/chaos.rs b/knot2/crates/knot-maintenance/tests/chaos.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-maintenance/tests/chaos.rs @@ -0,0 +1,235 @@ +use std::path::Path; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +use knot_git::{ + EntryKind, Identity, Layout, NewCommit, RefUpdate, Repo, StagedAction, StagedChange, +}; +use knot_maintenance::{ + GeometricFactor, ObjectCount, Options, PruneGrace, ReflogRetention, run_repo, +}; +use knot_types::{AuthorName, BranchName, Email, Oid, RefName, RepoDid, UnixSeconds}; + +const DID: &str = "did:plc:squid"; +const EMPTY_TREE: &str = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"; +const BLOB_BYTES: usize = 8 * 1024 * 1024; +const HISTORY: u32 = 40; +const NOW_SECONDS: UnixSeconds = UnixSeconds::new(1_700_000_500); + +fn incompressible(len: usize) -> Vec { + let mut state = 0x2545_f491_4f6c_dd1du64; + (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + (state & 0xff) as u8 + }) + .collect() +} + +fn identity() -> Identity { + Identity { + name: AuthorName::new("nel"), + email: Email::new("nel@oyster.cafe"), + time: UnixSeconds::new(1_700_000_000), + offset_seconds: 0, + } +} + +fn options() -> Options { + Options { + repack_max_objects: ObjectCount::new(5_000_000), + geometric_factor: GeometricFactor::full_repack(), + prune_grace: PruneGrace::from_secs(0), + reflog_floor: ReflogRetention::from_secs(i64::MAX as u64 / 4), + commit_graph: true, + multi_pack_index: true, + bitmap: true, + } +} + +fn build_template(scan: &Path) { + let layout = Layout::new(scan).with_default_branch(BranchName::new("main").unwrap()); + let did = RepoDid::new(DID).unwrap(); + let repo = layout.create(&did).unwrap(); + let main = RefName::new("refs/heads/main").unwrap(); + let empty = Oid::from_hex(EMPTY_TREE).unwrap(); + + let big_tree = repo + .write_staged_tree( + empty, + &[StagedChange { + path: knot_types::RepoPath::new("big.bin").unwrap(), + action: StagedAction::Put { + content: incompressible(BLOB_BYTES), + kind: EntryKind::Blob, + }, + }], + ) + .unwrap(); + let mut tip = repo + .write_commit(&NewCommit { + tree: big_tree, + parents: Vec::new(), + author: identity(), + committer: identity(), + message: "big".to_string(), + extra_headers: Vec::new(), + }) + .unwrap(); + (0..HISTORY).for_each(|index| { + let tree = repo + .write_staged_tree( + empty, + &[StagedChange { + path: knot_types::RepoPath::new(format!("file{index}.txt")).unwrap(), + action: StagedAction::Put { + content: format!("contents {index}").into_bytes(), + kind: EntryKind::Blob, + }, + }], + ) + .unwrap(); + let next = repo + .write_commit(&NewCommit { + tree, + parents: vec![tip], + author: identity(), + committer: identity(), + message: format!("commit {index}"), + extra_headers: Vec::new(), + }) + .unwrap(); + let update = match index { + 0 => RefUpdate::Create { + name: main.clone(), + new: next, + }, + _ => RefUpdate::Update { + name: main.clone(), + old: tip, + new: next, + }, + }; + repo.update_ref(&update).unwrap(); + tip = next; + }); +} + +fn copy_tree(src: &Path, dst: &Path) { + walkdir::WalkDir::new(src) + .into_iter() + .filter_map(Result::ok) + .for_each(|entry| { + let relative = entry.path().strip_prefix(src).unwrap(); + let target = dst.join(relative); + if entry.file_type().is_dir() { + std::fs::create_dir_all(&target).unwrap(); + } else { + if let Some(parent) = target.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::copy(entry.path(), &target).unwrap(); + } + }); +} + +fn spawn_worker(scan: &Path) -> std::process::Child { + Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "chaos_maintenance_worker", "--nocapture"]) + .env("KNOT_CHAOS_ROLE", "worker") + .env("KNOT_CHAOS_SCAN", scan) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn chaos worker") +} + +fn fsck_clean(bare: &Path) -> Result<(), String> { + knot_fixtures::fsck(bare) +} + +fn main_tip(scan: &Path) -> Option { + let layout = Layout::new(scan); + let repo = layout + .open(&RepoDid::new(DID).unwrap()) + .expect("repo must reopen cleanly after kill"); + repo.find_ref(&RefName::new("refs/heads/main").unwrap()) + .expect("references must be readable after kill") +} + +#[test] +fn chaos_maintenance_worker() { + if std::env::var("KNOT_CHAOS_ROLE").as_deref() != Ok("worker") { + return; + } + let scan = std::env::var("KNOT_CHAOS_SCAN").unwrap(); + let layout = Layout::new(&scan); + let repo = layout.open(&RepoDid::new(DID).unwrap()).unwrap(); + let _ = run_repo(&repo, NOW_SECONDS, &options()); +} + +#[test] +fn kill9_during_maintenance_leaves_a_consistent_repo() { + let scratch = tempfile::tempdir().unwrap(); + let template = scratch.path().join("template"); + build_template(&template); + let did = RepoDid::new(DID).unwrap(); + let tip = main_tip(&template).expect("template has a main tip"); + + let warm = scratch.path().join("warm"); + copy_tree(&template, &warm); + let started = Instant::now(); + let mut child = spawn_worker(&warm); + child.wait().unwrap(); + let full = started.elapsed(); + let warm_repo = Repo::open(Layout::new(&warm).repo_path(&did).unwrap()).unwrap(); + assert!( + warm_repo + .git() + .git_dir() + .join("objects/info/commit-graph") + .exists(), + "uninterrupted maintenance run writes commit-graph" + ); + + let fractions = [0.20, 0.35, 0.45, 0.55, 0.65, 0.75, 0.85, 0.95, 1.10]; + let delays: Vec = std::iter::once(Duration::from_millis(1)) + .chain(std::iter::once(Duration::from_millis(3))) + .chain(fractions.iter().map(|fraction| full.mul_f64(*fraction))) + .chain(std::iter::once(full.mul_f64(2.0))) + .collect(); + + delays.iter().enumerate().for_each(|(trial, delay)| { + let scan = scratch.path().join(format!("scan-{trial}")); + copy_tree(&template, &scan); + let mut child = spawn_worker(&scan); + std::thread::sleep(*delay); + let _ = child.kill(); + child.wait().unwrap(); + + let bare = Layout::new(&scan).repo_path(&did).unwrap(); + fsck_clean(&bare).unwrap_or_else(|errors| { + panic!("trial {trial}: killed maintenance run left corrupt repo:\n{errors}") + }); + assert_eq!( + main_tip(&scan), + Some(tip), + "trial {trial}: maintenance never changes branch value, so main must still resolve to tip" + ); + + let recovered = Repo::open(&bare).unwrap(); + run_repo(&recovered, NOW_SECONDS, &options()).unwrap_or_else(|error| { + panic!("trial {trial}: maintenance must self-heal after crash, got {error}") + }); + fsck_clean(&bare).unwrap_or_else(|errors| { + panic!("trial {trial}: self-heal pass left corrupt repo:\n{errors}") + }); + assert_eq!( + main_tip(&scan), + Some(tip), + "trial {trial}: recovered repo still resolves main to tip" + ); + }); +} diff --git a/knot2/crates/knot-maintenance/tests/commitgraph.rs b/knot2/crates/knot-maintenance/tests/commitgraph.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-maintenance/tests/commitgraph.rs @@ -0,0 +1,248 @@ +use std::collections::HashMap; +use std::path::Path; + +use knot_git::Repo; +use knot_maintenance::run_repo; + +mod common; +use common::{git_available, now, options}; + +fn skip() -> bool { + if git_available() { + return false; + } + eprintln!("skipping commit-graph differential: git unavailable"); + true +} + +fn git_at(dir: &Path, date: i64, args: &[&str]) { + let stamp = format!("{date} +0000"); + let out = knot_fixtures::command_at(dir, &stamp) + .args(args) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); +} + +fn git(dir: &Path, args: &[&str]) { + git_at(dir, 1_700_000_000, args); +} + +fn write_file(dir: &Path, rel: &str, contents: &str) { + let path = dir.join(rel); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, contents).unwrap(); +} + +fn commit(dir: &Path, rel: &str, contents: &str, message: &str) { + write_file(dir, rel, contents); + git(dir, &["add", "-A"]); + git(dir, &["commit", "-m", message]); +} + +fn seed_history(dir: &Path, format: &str) { + git(dir, &["init", "--object-format", format, "-b", "main"]); + write_file(dir, "dir1/a.txt", "a1"); + write_file(dir, "dir1/sub/b.txt", "b1"); + commit(dir, "c.txt", "c1", "root"); + commit(dir, "dir1/sub/b.txt", "b2", "edit nested"); + git(dir, &["checkout", "-b", "feature"]); + commit(dir, "c.txt", "c2", "feature edit"); + git(dir, &["checkout", "main"]); + commit(dir, "dir2/d.txt", "d1", "add dir2"); + git(dir, &["merge", "--no-ff", "-m", "merge feature", "feature"]); + git(dir, &["tag", "v1"]); +} + +fn read_chunks(bytes: &[u8]) -> HashMap<[u8; 4], Vec> { + let count = bytes[6] as usize; + let table = &bytes[8..8 + (count + 1) * 12]; + let entry = |index: usize| -> ([u8; 4], u64) { + let base = index * 12; + let id = table[base..base + 4].try_into().unwrap(); + ( + id, + u64::from_be_bytes(table[base + 4..base + 12].try_into().unwrap()), + ) + }; + (0..count) + .map(|index| { + let (id, start) = entry(index); + let (_, end) = entry(index + 1); + (id, bytes[start as usize..end as usize].to_vec()) + }) + .collect() +} + +fn graph_verify(root: &Path) -> (bool, String) { + let out = knot_fixtures::command(root) + .args(["commit-graph", "verify"]) + .output() + .expect("git commit-graph verify runs"); + ( + out.status.success(), + String::from_utf8_lossy(&out.stderr).into_owned(), + ) +} + +fn path_log(dir: &Path, path: &str, use_graph: bool) -> String { + let out = knot_fixtures::command(dir) + .args([ + "-c", + &format!("core.commitGraph={use_graph}"), + "-c", + "commitGraph.readChangedPaths=true", + "log", + "--format=%H", + "--", + path, + ]) + .output() + .expect("git log runs"); + assert!(out.status.success()); + String::from_utf8(out.stdout).unwrap() +} + +fn build_and_compare(root: &Path, format: &str, chunks: &[&[u8; 4]]) { + git( + root, + &[ + "-c", + "commitGraph.changedPathsVersion=2", + "commit-graph", + "write", + "--reachable", + "--changed-paths", + ], + ); + let graph_file = root.join(".git/objects/info/commit-graph"); + let canonical = read_chunks(&std::fs::read(&graph_file).unwrap()); + + assert!( + run_repo(&Repo::open(root).unwrap(), now(), &options()) + .unwrap() + .commit_graph, + "knot wrote a graph ({format})" + ); + let ours = read_chunks(&std::fs::read(&graph_file).unwrap()); + chunks.iter().for_each(|id| { + assert_eq!( + ours.get(*id), + canonical.get(*id), + "chunk {} differs ({format})", + String::from_utf8_lossy(*id) + ); + }); + let (ok, stderr) = graph_verify(root); + assert!(ok, "git commit-graph verify failed ({format}): {stderr}"); +} + +fn check_against_git(format: &str) { + if skip() { + return; + } + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + seed_history(root, format); + build_and_compare(root, format, &[b"OIDL", b"CDAT", b"GDA2", b"BIDX", b"BDAT"]); + + ["dir1/sub/b.txt", "dir1/sub", "dir1", "c.txt", "dir2/d.txt"] + .iter() + .for_each(|path| { + assert_eq!( + path_log(root, path, true), + path_log(root, path, false), + "changed-path bloom altered `git log -- {path}` ({format})" + ); + }); +} + +#[test] +fn matches_canonical_git_sha1() { + check_against_git("sha1"); +} + +#[test] +fn matches_canonical_git_sha256() { + check_against_git("sha256"); +} + +#[test] +fn writing_the_graph_clears_a_pre_existing_split_chain() { + if skip() { + return; + } + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + seed_history(root, "sha1"); + + git(root, &["commit-graph", "write", "--reachable", "--split"]); + let chain = root.join(".git/objects/info/commit-graphs"); + assert!(chain.exists(), "git wrote a split commit-graph chain"); + + assert!( + run_repo(&Repo::open(root).unwrap(), now(), &options()) + .unwrap() + .commit_graph, + "knot wrote a monolithic graph" + ); + assert!( + !chain.exists(), + "the stale split chain is removed so it cannot shadow the fresh graph" + ); + assert!(root.join(".git/objects/info/commit-graph").exists()); + let (ok, stderr) = graph_verify(root); + assert!( + ok, + "git verify passes after the chain is replaced: {stderr}" + ); +} + +#[test] +fn large_filter_sentinel_matches_git_when_dirs_overflow_the_limit() { + if skip() { + return; + } + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + git(root, &["init", "--object-format", "sha1", "-b", "main"]); + (0..200).for_each(|i| write_file(root, &format!("a{i}/b{i}/c.txt"), "x")); + git(root, &["add", "-A"]); + git(root, &["commit", "-m", "wide refactor"]); + build_and_compare(root, "sha1", &[b"BIDX", b"BDAT"]); +} + +#[test] +fn corrected_date_overflow_matches_git() { + if skip() { + return; + } + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + git(root, &["init", "--object-format", "sha1", "-b", "main"]); + write_file(root, "a.txt", "1"); + git_at(root, 4_000_000_000, &["add", "-A"]); + git_at(root, 4_000_000_000, &["commit", "-m", "far-future root"]); + write_file(root, "a.txt", "2"); + git_at(root, 1_000_000_000, &["add", "-A"]); + git_at(root, 1_000_000_000, &["commit", "-m", "past child"]); + build_and_compare(root, "sha1", &[b"CDAT", b"GDA2", b"GDO2"]); +} + +#[test] +fn changed_path_filter_matches_git_for_high_byte_names() { + if skip() { + return; + } + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + git(root, &["init", "--object-format", "sha1", "-b", "main"]); + write_file(root, "café/résumé.txt", "1"); + commit(root, "naïve.md", "2", "non-ascii paths"); + commit(root, "café/résumé.txt", "2", "edit non-ascii"); + build_and_compare(root, "sha1", &[b"BIDX", b"BDAT"]); +} diff --git a/knot2/crates/knot-maintenance/tests/cruft.rs b/knot2/crates/knot-maintenance/tests/cruft.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-maintenance/tests/cruft.rs @@ -0,0 +1,143 @@ +use knot_git::Repo; +use knot_maintenance::{Options, PruneGrace, ReflogRetention, run_repo}; +use knot_types::{ObjectFormat, UnixSeconds}; + +mod common; +use common::{ + EMPTY_TREE_SHA1, assert_fsck_clean, commit_on, create_repo, delete_ref, empty_tree, + has_cruft_pack, now, options, set_ref, set_reflog_seconds, +}; + +fn opts(grace: PruneGrace) -> Options { + Options { + prune_grace: grace, + commit_graph: false, + multi_pack_index: false, + bitmap: false, + ..options() + } +} + +fn retain_repeat_then_expire(format: ObjectFormat) { + let scan = tempfile::tempdir().unwrap(); + let empty = empty_tree(format); + let repo = create_repo(scan.path(), format, "did:plc:cuttle"); + + let base = commit_on(&repo, empty, 0, Vec::new()); + let doomed = commit_on(&repo, empty, 1, vec![base]); + set_ref(&repo, "refs/heads/main", base); + set_ref(&repo, "refs/heads/feature", doomed); + run_repo(&repo, now(), &opts(PruneGrace::from_secs(86_400))).unwrap(); + assert!(repo.contains(doomed), "doomed is packed while reachable"); + + delete_ref(&repo, "refs/heads/feature"); + set_ref( + &repo, + "refs/heads/main", + commit_on(&repo, empty, 2, vec![base]), + ); + + let retained = run_repo(&repo, now(), &opts(PruneGrace::from_secs(86_400))).unwrap(); + assert!( + retained.prune.crufted.get() >= 1, + "young unreachable objects are crufted, not dropped" + ); + let reopened = Repo::open(repo.git().git_dir()).unwrap(); + assert!( + has_cruft_pack(&reopened) && reopened.contains(doomed), + "a cruft pack holds the young object" + ); + assert_fsck_clean(&reopened); + + run_repo(&reopened, now(), &opts(PruneGrace::from_secs(86_400))).unwrap(); + let recycled = Repo::open(repo.git().git_dir()).unwrap(); + assert!( + recycled.contains(doomed), + "the young object survives a repeat cruft cycle within grace" + ); + assert_fsck_clean(&recycled); + + assert!( + run_repo(&recycled, now(), &opts(PruneGrace::from_secs(0))) + .unwrap() + .prune + .ran + ); + let settled = Repo::open(repo.git().git_dir()).unwrap(); + assert!( + !settled.contains(doomed), + "the past-grace object is finally dropped" + ); + assert!( + settled.contains(base) && !has_cruft_pack(&settled), + "base survives, no cruft lingers" + ); + assert_fsck_clean(&settled); +} + +#[test] +fn cruft_retains_survives_a_repeat_cycle_then_expires_sha1() { + retain_repeat_then_expire(ObjectFormat::SHA1); +} + +#[test] +fn cruft_retains_survives_a_repeat_cycle_then_expires_sha256() { + retain_repeat_then_expire(ObjectFormat::SHA256); +} + +#[test] +fn a_reflog_floor_above_the_retention_minimum_keeps_referenced_objects() { + let scan = tempfile::tempdir().unwrap(); + let repo = create_repo(scan.path(), ObjectFormat::SHA1, "did:plc:whelk"); + + let base = commit_on(&repo, EMPTY_TREE_SHA1, 0, Vec::new()); + let doomed = commit_on(&repo, EMPTY_TREE_SHA1, 1, vec![base]); + set_ref(&repo, "refs/heads/main", doomed); + set_ref( + &repo, + "refs/heads/main", + commit_on(&repo, EMPTY_TREE_SHA1, 2, vec![base]), + ); + + let opts = Options { + reflog_floor: ReflogRetention::from_secs(3_000_000_000), + ..opts(PruneGrace::from_secs(0)) + }; + run_repo(&repo, UnixSeconds::new(4_000_000_000), &opts).unwrap(); + + let reopened = Repo::open(repo.git().git_dir()).unwrap(); + assert!( + reopened.contains(doomed), + "an object held only by a retained reflog entry is a prune root" + ); +} + +#[test] +fn a_rewound_away_commit_survives_on_its_reflog_old_pointer_alone() { + let scan = tempfile::tempdir().unwrap(); + let repo = create_repo(scan.path(), ObjectFormat::SHA1, "did:plc:periwinkle"); + + let lost = commit_on(&repo, EMPTY_TREE_SHA1, 7, Vec::new()); + set_ref(&repo, "refs/heads/main", lost); + let survivor = commit_on(&repo, EMPTY_TREE_SHA1, 9, Vec::new()); + set_ref(&repo, "refs/heads/main", survivor); + + set_reflog_seconds(&repo, "refs/heads/main", &[1_000_000, 2_000_000_000]); + + let opts = Options { + reflog_floor: ReflogRetention::from_secs(500_000_000), + ..opts(PruneGrace::from_secs(0)) + }; + run_repo(&repo, UnixSeconds::new(2_000_000_000), &opts).unwrap(); + + let reopened = Repo::open(repo.git().git_dir()).unwrap(); + assert!( + reopened.contains(lost), + "a force-rewound commit reachable only through a reflog old-pointer must not be pruned" + ); + assert!( + reopened.contains(survivor), + "the live tip survives the prune" + ); + assert_fsck_clean(&reopened); +} diff --git a/knot2/crates/knot-maintenance/tests/engine.rs b/knot2/crates/knot-maintenance/tests/engine.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-maintenance/tests/engine.rs @@ -0,0 +1,200 @@ +use std::collections::HashSet; + +use knot_git::Repo; +use knot_maintenance::{Options, PruneGrace, RepackStatus, run_repo}; +use knot_types::{ObjectFormat, Oid}; + +mod common; +use common::{ + commit, create_repo, delete_ref, git, has_bitmap, has_midx_bitmap, now, options, set_ref, +}; + +fn create(scan: &std::path::Path, did: &str) -> Repo { + create_repo(scan, ObjectFormat::SHA1, did) +} + +fn loose_count(repo: &Repo) -> usize { + walkdir::WalkDir::new(repo.git().git_dir().join("objects")) + .into_iter() + .filter_map(Result::ok) + .filter(|e| e.file_type().is_file()) + .filter(|e| { + e.path() + .parent() + .and_then(|p| p.file_name()) + .and_then(|n| n.to_str()) + .is_some_and(|n| n.len() == 2) + }) + .count() +} + +#[test] +fn a_single_pack_maintenance_pass_packs_graphs_bitmaps_prunes_then_settles() { + let scan = tempfile::tempdir().unwrap(); + let repo = create(scan.path(), "did:plc:squid"); + + let base = commit(&repo, 0, Vec::new()); + let main_tip = commit(&repo, 1, vec![base]); + set_ref(&repo, "refs/heads/main", main_tip); + set_ref(&repo, "refs/heads/feature", commit(&repo, 2, vec![base])); + let orphan = commit(&repo, 9, vec![main_tip]); + + let info = repo.git().git_dir().join("objects/info"); + std::fs::create_dir_all(&info).unwrap(); + let leaked = info.join("commit-graph.knot-tmp.999999"); + std::fs::write(&leaked, b"a graph write that a kill -9 interrupted").unwrap(); + let long_ago = std::time::SystemTime::now() - std::time::Duration::from_secs(7 * 3600); + std::fs::File::options() + .write(true) + .open(&leaked) + .unwrap() + .set_times(std::fs::FileTimes::new().set_modified(long_ago)) + .unwrap(); + assert!(loose_count(&repo) >= 4 && repo.contains(orphan)); + + let report = run_repo(&repo, now(), &options()).unwrap(); + assert_eq!(report.repack.status, RepackStatus::Repacked); + assert!( + report.repack.removed_loose.get() >= 4, + "loose objects folded into the pack" + ); + assert!(report.commit_graph && info.join("commit-graph").exists()); + assert!( + !leaked.exists(), + "maintenance sweeps a crashed graph write's temp once it is too old to still have a writer" + ); + assert!(report.bitmap && has_bitmap(&repo)); + let (ok, stderr) = git(&repo, &["rev-list", "--test-bitmap", "main"]); + assert!(ok, "canonical git accepts our bitmap: {stderr}"); + assert!(report.packed_refs.packed >= 1); + assert!( + report.prune.ran && report.prune.removed.get() >= 1, + "the orphan is pruned" + ); + + let reopened = Repo::open(repo.git().git_dir()).unwrap(); + assert!(!reopened.contains(orphan) && reopened.contains(main_tip)); + let (clean, stderr) = git(&reopened, &["fsck", "--no-progress"]); + assert!(clean, "fsck-clean after the pass: {stderr}"); + + let settled = run_repo(&reopened, now(), &options()).unwrap(); + assert_eq!(settled.repack.status, RepackStatus::Clean); + assert!( + !settled.prune.ran && !settled.commit_graph && !settled.bitmap, + "a settled repo is a no-op" + ); + assert_eq!(settled.packed_refs.packed, 0); +} + +#[test] +fn the_written_graph_verifies_and_accelerates_selection_through_an_octopus_merge() { + let scan = tempfile::tempdir().unwrap(); + let repo = create(scan.path(), "did:plc:cuttle"); + + let a = commit(&repo, 1, Vec::new()); + let b = commit(&repo, 2, Vec::new()); + let c = commit(&repo, 3, Vec::new()); + let pair = commit(&repo, 4, vec![a, b]); + let octopus = commit(&repo, 5, vec![a, b, c]); + let tip = commit(&repo, 6, vec![pair, octopus]); + set_ref(&repo, "refs/heads/main", tip); + + let graph = repo.git().git_dir().join("objects/info/commit-graph"); + let closure = |r: &Repo| -> HashSet { + r.select_pack_objects(knot_git::Wants::new(&[tip]), knot_git::Haves::new(&[])) + .unwrap() + .into_iter() + .collect() + }; + let decode_closure = closure(&repo); + assert!(!graph.exists()); + + assert!(run_repo(&repo, now(), &options()).unwrap().commit_graph && graph.exists()); + let (ok, stderr) = git(&repo, &["commit-graph", "verify"]); + assert!( + ok, + "git accepts the hand-written graph with an octopus: {stderr}" + ); + + let reopened = Repo::open(repo.git().git_dir()).unwrap(); + assert_eq!( + closure(&reopened), + decode_closure, + "graph-accelerated selection equals the decode walk" + ); +} + +#[test] +fn commit_graph_knob_lifecycle_skips_backfills_and_sweeps() { + let scan = tempfile::tempdir().unwrap(); + let repo = create(scan.path(), "did:plc:limpet"); + + let base = commit(&repo, 0, Vec::new()); + set_ref(&repo, "refs/heads/main", commit(&repo, 1, vec![base])); + + let off = Options { + commit_graph: false, + ..options() + }; + let graph = repo.git().git_dir().join("objects/info/commit-graph"); + + let first = run_repo(&repo, now(), &off).unwrap(); + assert_eq!(first.repack.status, RepackStatus::Repacked); + assert!( + !first.commit_graph && !graph.exists(), + "no graph is written when the knob is off" + ); + + let reopened = Repo::open(repo.git().git_dir()).unwrap(); + let backfill = run_repo(&reopened, now(), &options()).unwrap(); + assert!( + backfill.commit_graph && graph.exists(), + "a packed repo with no graph backfills it" + ); + assert_eq!(backfill.repack.status, RepackStatus::Clean); + + let settled = Repo::open(repo.git().git_dir()).unwrap(); + assert!( + !run_repo(&settled, now(), &options()).unwrap().commit_graph, + "a present graph settles to no-op" + ); + + let swept = Repo::open(repo.git().git_dir()).unwrap(); + assert!(!run_repo(&swept, now(), &off).unwrap().commit_graph); + assert!( + !graph.exists(), + "flipping the knob off sweeps the orphan graph" + ); +} + +#[test] +fn cruft_second_pack_gets_a_midx_bitmap_canonical_git_accepts() { + let scan = tempfile::tempdir().unwrap(); + let repo = create(scan.path(), "did:plc:mussel"); + + let base = commit(&repo, 0, Vec::new()); + let doomed = commit(&repo, 1, vec![base]); + set_ref(&repo, "refs/heads/main", base); + set_ref(&repo, "refs/heads/feature", doomed); + + let opts = Options { + prune_grace: PruneGrace::from_secs(86_400), + ..options() + }; + run_repo(&repo, now(), &opts).unwrap(); + + delete_ref(&repo, "refs/heads/feature"); + set_ref(&repo, "refs/heads/main", commit(&repo, 2, vec![base])); + + let report = run_repo(&repo, now(), &opts).unwrap(); + assert!( + report.prune.crufted.get() >= 1, + "young unreachable object is crufted" + ); + assert!( + report.bitmap && has_midx_bitmap(&repo), + "the multi-pack repo gets a midx bitmap" + ); + let (ok, stderr) = git(&repo, &["rev-list", "--test-bitmap", "main"]); + assert!(ok, "canonical git accepts our midx bitmap: {stderr}"); +} diff --git a/knot2/crates/knot-maintenance/tests/geometric.rs b/knot2/crates/knot-maintenance/tests/geometric.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-maintenance/tests/geometric.rs @@ -0,0 +1,129 @@ +use std::collections::BTreeSet; + +use knot_git::Repo; +use knot_maintenance::{GeometricFactor, Options, PruneGrace, run_repo}; +use knot_types::{ObjectFormat, Oid, RefName}; + +mod common; +use common::{ + EMPTY_TREE_SHA1, chain, create_repo, delete_ref, empty_tree, fsck_clean, git_available, + has_cruft_pack, idx_stems, midx_verifies, now, options, reachable_objects, set_ref, +}; + +fn opts(factor: u64, grace: PruneGrace) -> Options { + Options { + geometric_factor: GeometricFactor::new(factor), + prune_grace: grace, + commit_graph: false, + bitmap: false, + ..options() + } +} + +fn main_ref(repo: &Repo) -> Option { + repo.find_ref(&RefName::new("refs/heads/main").unwrap()) + .unwrap() +} + +fn no_reachable_loss(format: ObjectFormat) { + if !git_available() { + eprintln!("skipping geometric differential: git unavailable"); + return; + } + let scan = tempfile::tempdir().unwrap(); + let empty = empty_tree(format); + let repo = create_repo(scan.path(), format, "did:plc:scallop"); + let opts = opts(2, PruneGrace::from_secs(86_400)); + + (0..6).fold((None, BTreeSet::new()), |(tip, seen), round| { + let next = chain(&repo, empty, round * 2..round * 2 + 2, tip); + set_ref(&repo, "refs/heads/main", next); + run_repo(&repo, now(), &opts).unwrap(); + + let r = Repo::open(repo.git().git_dir()).unwrap(); + assert!(fsck_clean(&r), "{format:?} r{round}: not fsck-clean"); + if let Some((ok, stderr)) = midx_verifies(&r) { + assert!(ok, "{format:?} r{round} midx: {stderr}"); + } + assert_eq!( + main_ref(&r), + Some(next), + "{format:?} r{round}: main lost its tip" + ); + let present = reachable_objects(&r); + assert!( + seen.is_subset(&present), + "{format:?} r{round}: a reachable object went missing" + ); + (Some(next), present) + }); +} + +#[test] +fn geometric_no_reachable_loss_sha1() { + no_reachable_loss(ObjectFormat::SHA1); +} + +#[test] +fn geometric_no_reachable_loss_sha256() { + no_reachable_loss(ObjectFormat::SHA256); +} + +#[test] +fn geometric_keeps_the_large_pack_while_rolling_up_then_crufting_small_packs() { + if !git_available() { + eprintln!("skipping geometric behavior test: git unavailable"); + return; + } + let scan = tempfile::tempdir().unwrap(); + let repo = create_repo(scan.path(), ObjectFormat::SHA1, "did:plc:conch"); + let opts = opts(2, PruneGrace::from_secs(86_400)); + + let big = chain(&repo, EMPTY_TREE_SHA1, 0..10, None); + set_ref(&repo, "refs/heads/main", big); + run_repo(&repo, now(), &opts).unwrap(); + let large = idx_stems(&repo); + assert_eq!( + large.len(), + 1, + "the initial repack settles to one large pack" + ); + + let advanced = chain(&repo, EMPTY_TREE_SHA1, 100..102, Some(big)); + set_ref(&repo, "refs/heads/main", advanced); + run_repo(&repo, now(), &opts).unwrap(); + assert!( + large.is_subset(&idx_stems(&repo)), + "the large pack is kept verbatim" + ); + assert_eq!( + idx_stems(&repo).len(), + 2, + "small additions roll into a second pack" + ); + + let feature = chain(&repo, EMPTY_TREE_SHA1, 200..202, Some(advanced)); + set_ref(&repo, "refs/heads/feature", feature); + run_repo(&repo, now(), &opts).unwrap(); + delete_ref(&repo, "refs/heads/feature"); + + let report = run_repo(&repo, now(), &opts).unwrap(); + assert!( + report.prune.crufted.get() >= 1, + "young rolled-up garbage is crufted" + ); + assert!( + large.is_subset(&idx_stems(&repo)) && has_cruft_pack(&repo), + "large pack untouched, cruft written" + ); + + let reopened = Repo::open(repo.git().git_dir()).unwrap(); + assert!( + fsck_clean(&reopened), + "not fsck-clean after roll-up and cruft" + ); + assert!( + reopened.contains(advanced) && reopened.contains(feature), + "reachable tip and young orphan both survive" + ); +} diff --git a/knot2/crates/knot-messages/src/lib.rs b/knot2/crates/knot-messages/src/lib.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-messages/src/lib.rs @@ -0,0 +1,376 @@ +mod template; + +use confique::Config; + +pub use template::{Key, Line, Lines, NoKeys, Segment, Shape, Template, TemplateError}; + +// Each msg field defines the enum of placeholders it accepts, +// so a typo'ed `{handel}` in a given config gets caught at startup, +// rather than printed at some poor pusher mid-push. +macro_rules! keys { + ( $( $name:ident { $( $variant:ident = $placeholder:literal ),+ $(,)? } )+ ) => { + $( + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum $name { + $( $variant, )+ + } + + impl Key for $name { + const PLACEHOLDERS: &'static [(&'static str, Self)] = + &[ $( ($placeholder, Self::$variant), )+ ]; + } + )+ + }; +} + +macro_rules! config_type { + (Lines) => { Vec }; + (Line) => { String }; +} + +macro_rules! parse_field { + (Lines, $field:expr, $value:expr) => { + Template::parse_lines($field, $value) + }; + (Line, $field:expr, $value:expr) => { + Template::parse($field, $value) + }; +} + +macro_rules! message_group { + ( + $config:ident => $catalog:ident @ $prefix:literal { + $( $field:ident : $shape:ident<$keys:ty> = $default:tt ),+ $(,)? + } + ) => { + #[derive(Debug, ::confique::Config)] + pub struct $config { + $( + #[config(default = $default)] + pub $field: config_type!($shape), + )+ + } + + #[derive(Debug)] + pub struct $catalog { + $( pub $field: Template<$keys, $shape>, )+ + } + + impl $catalog { + pub fn parse(config: &$config) -> Result { + Ok(Self { + $( + $field: parse_field!( + $shape, + concat!($prefix, ".", stringify!($field)), + &config.$field + )?, + )+ + }) + } + } + }; +} + +keys! { + KnotKey { Knot = "knot" } + PushAckKey { Knot = "knot", Refs = "refs" } + UrlKey { Url = "url" } + CiLogsKey { Host = "host", Port = "port", Repo = "repo", Sha = "sha" } + GreetingKey { User = "user", Knot = "knot" } + CountKey { Count = "count" } + RefKey { Ref = "ref" } + ErrorKey { Error = "error" } + CommandKey { Command = "command" } + VersionKey { Version = "version" } + AlgorithmKey { Algorithm = "algorithm" } + ValueKey { Value = "value" } + OidKey { Oid = "oid" } + DetailKey { Detail = "detail" } + DeclaredComputedKey { Declared = "declared", Computed = "computed" } + DeclaredReceivedKey { Declared = "declared", Received = "received" } + DeclaredLimitKey { Declared = "declared", Limit = "limit" } + FreeFloorKey { Free = "free", Floor = "floor" } + WhatLimitKey { What = "what", Limit = "limit" } +} + +message_group! { + PushConfig => PushMessages @ "messages.push" { + ack: Lines = ["{knot} received {refs}."], + pull_request: Lines = [ + "", + "-> Open stinky pull request for this branch:", + " {url}", + "" + ], + pipeline_clean: Lines = ["pipeline compiled with no diagnostics"], + pipeline_none: Lines = ["no pipelines to compile"], + ci_logs: Lines = [ + "-> Browse CI logs in your terminal:", + " ssh -t -p {port} {host} {repo} {sha}" + ], + } +} + +message_group! { + FetchConfig => FetchMessages @ "messages.fetch" { + motd: Lines = ["Thanks for using {knot}!"], + enumerating: Lines = ["Enumerating objects: {count}, done."], + total: Lines = ["Total {count}, done."], + fatal: Line = "knot: {error}", + } +} + +message_group! { + RejectConfig => RejectMessages @ "messages.reject" { + reserved_refs: Line = "refs/cobs/* and refs/hidden/* are reserved and cannot be pushed", + cob_create_only: Line = "existing refs/cobs/* object cannot be modified or deleted over the wire", + cob_delete: Line = "refs/cobs/* stores append-only collaborative objects and cannot be deleted", + hidden_reserved: Line = "refs/hidden/* is reserved for server-side fork staging and cannot be pushed", + cob_verification: Line = "collaborative-object verification failed: {error}", + ref_exists: Line = "reference already exists", + stale_old_value: Line = "stale info: old value doesn't match", + missing_objects: Line = "missing necessary objects", + missing_objects_for: Line = "missing necessary objects for {ref}", + atomic_failed: Line = "atomic transaction failed", + atomic_aborted: Line = "atomic push aborted", + authorization_unavailable: Line = "authorization unavailable", + unpacker_error: Line = "unpacker error", + ref_snapshot_unavailable: Line = "ref snapshot unavailable", + object_migration_failed: Line = "object migration failed", + } +} + +message_group! { + SshConfig => SshMessages @ "messages.ssh" { + greeting: Lines = [ + "Hi {user}! You're authenticated to {knot} knot.", + "This knot serves git over ssh, so there's no shell here. :P", + "Clone repo with: git clone {knot}:" + ], + unsupported_command: Line = "knot: unsupported command", + too_many_operations: Line = "knot: too many concurrent operations from your address, try again shortly", + repo_not_found: Line = "knot: repository not found", + index_warming: Line = "knot: repository index is warming, retry shortly", + lfs_disabled: Line = "knot: LFS isn't enabled on this knot", + key_not_registered: Line = "knot: your ssh key isn't registered to a user authorized to push here. If you offer several keys, make sure the registered one is offered first.", + push_denied: Line = "knot: you aren't authorized to push to this repository.", + shutting_down: Line = "knot: server is shutting down", + archive_malformed: Line = "knot: malformed upload-archive request", + archive_timeout: Line = "knot: upload-archive request timed out", + archive_failed: Line = "knot: upload-archive failed", + advertise_failed: Line = "knot: cannot advertise refs", + push_too_large: Line = "knot: push exceeds configured size limit", + receive_deadline: Line = "knot: receive exceeded its time budget", + malformed_pack: Line = "knot: malformed pack stream", + receive_read_error: Line = "knot: receive read error", + receive_ended_early: Line = "knot: receive stream ended early", + receive_failed: Line = "knot: receive-pack failed", + } +} + +message_group! { + HttpConfig => HttpMessages @ "messages.http" { + push_denied: Line = "you aren't authorized to push to this repository", + repo_not_found: Line = "repository not found", + push_too_large: Line = "push exceeds the configured size limit", + malformed_pack: Line = "malformed pack stream: {error}", + receive_ended_early: Line = "receive stream ended early", + } +} + +message_group! { + LfsConfig => LfsMessages @ "messages.lfs" { + invalid_oid: Line = "invalid LFS oid {value}", + hash_mismatch: Line = "oid mismatch, declared {declared}, computed {computed}", + size_mismatch: Line = "size mismatch, declared {declared}, received {received}", + size_limit_exceeded: Line = "object size {declared} exceeds limit {limit}", + free_space_denied: Line = "free space {free} below floor {floor}", + not_found: Line = "object {oid} not found", + framing: Line = "protocol framing fault: {detail}", + too_many: Line = "too many {what} in one message, limit {limit}", + unknown_command: Line = "unknown command {command}", + unsupported_version: Line = "unsupported version {version}", + unsupported_hash: Line = "unsupported hash algorithm {algorithm}", + put_on_download: Line = "put-object isn't allowed on a download channel", + verify_on_download: Line = "verify-object isn't allowed on a download channel", + get_on_upload: Line = "get-object isn't allowed on an upload channel", + put_no_body: Line = "put-object is missing its object body", + } +} + +#[derive(Debug, Config)] +pub struct MessagesConfig { + #[config(nested)] + pub push: PushConfig, + #[config(nested)] + pub fetch: FetchConfig, + #[config(nested)] + pub reject: RejectConfig, + #[config(nested)] + pub ssh: SshConfig, + #[config(nested)] + pub http: HttpConfig, + #[config(nested)] + pub lfs: LfsConfig, +} + +impl MessagesConfig { + pub fn defaults() -> Self { + Self::builder() + .load() + .expect("message defaults satisfy every field") + } +} + +#[derive(Debug)] +pub struct Catalog { + pub push: PushMessages, + pub fetch: FetchMessages, + pub reject: RejectMessages, + pub ssh: SshMessages, + pub http: HttpMessages, + pub lfs: LfsMessages, +} + +impl Catalog { + pub fn parse(config: &MessagesConfig) -> Result { + Ok(Self { + push: PushMessages::parse(&config.push)?, + fetch: FetchMessages::parse(&config.fetch)?, + reject: RejectMessages::parse(&config.reject)?, + ssh: SshMessages::parse(&config.ssh)?, + http: HttpMessages::parse(&config.http)?, + lfs: LfsMessages::parse(&config.lfs)?, + }) + } + + pub fn defaults() -> Self { + Self::parse(&MessagesConfig::defaults()).expect("built-in message templates parse") + } +} + +pub fn default_catalog() -> &'static Catalog { + static DEFAULTS: std::sync::LazyLock = std::sync::LazyLock::new(Catalog::defaults); + &DEFAULTS +} + +pub fn count_refs(applied: usize) -> String { + match applied { + 1 => "1 ref".to_string(), + n => format!("{n} refs"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_defaults_parse_into_a_full_catalog() { + let catalog = Catalog::defaults(); + assert_eq!(catalog.reject.ref_exists.text(), "reference already exists"); + assert_eq!( + catalog.ssh.repo_not_found.text(), + "knot: repository not found" + ); + } + + #[test] + fn the_pull_request_block_matches_the_shipped_shape() { + let catalog = Catalog::defaults(); + let url = "https://oyster.cafe/nel.pet/anemone/pulls/new"; + let block = catalog + .push + .pull_request + .lines(|UrlKey::Url| url.to_string()); + assert_eq!( + block, + vec![ + "\u{200b}".to_string(), + "-> Open stinky pull request for this branch:".to_string(), + format!(" {url}"), + "\u{200b}".to_string(), + ] + ); + } + + #[test] + fn the_greeting_names_the_user_and_the_knot() { + let catalog = Catalog::defaults(); + let lines = catalog.ssh.greeting.lines(|key| match key { + GreetingKey::User => "@nel.pet".to_string(), + GreetingKey::Knot => "oyster.cafe".to_string(), + }); + assert!(lines[0].contains("@nel.pet")); + assert!(lines.iter().any(|line| line.contains("oyster.cafe"))); + } + + #[test] + fn an_empty_lines_template_mutes_the_message() { + let template: Template = + Template::parse_lines("messages.test", &[]).unwrap(); + assert!(template.text_lines().is_empty()); + } + + #[test] + fn an_unknown_placeholder_is_a_parse_error() { + let error = Template::::parse_lines( + "messages.fetch.motd", + &["hi {handle}".to_string()], + ) + .unwrap_err(); + assert_eq!( + error, + TemplateError::UnknownPlaceholder { + field: "messages.fetch.motd", + name: "handle".to_string(), + } + ); + } + + #[test] + fn doubled_braces_render_as_literal_braces() { + let template: Template = + Template::parse("messages.test", "a {{literal}} brace").unwrap(); + assert_eq!(template.text(), "a {literal} brace"); + } + + #[test] + fn line_templates_reject_empty_and_multiline_text() { + assert_eq!( + Template::::parse("messages.test", "").unwrap_err(), + TemplateError::Empty { + field: "messages.test" + } + ); + assert_eq!( + Template::::parse("messages.test", "a\nb").unwrap_err(), + TemplateError::Multiline { + field: "messages.test" + } + ); + } + + #[test] + fn unbalanced_braces_are_parse_errors() { + assert_eq!( + Template::::parse("messages.test", "open {").unwrap_err(), + TemplateError::UnclosedBrace { + field: "messages.test" + } + ); + assert_eq!( + Template::::parse("messages.test", "close }").unwrap_err(), + TemplateError::StrayBrace { + field: "messages.test" + } + ); + } + + #[test] + fn ref_counts_pluralize() { + assert_eq!(count_refs(1), "1 ref"); + assert_eq!(count_refs(3), "3 refs"); + } +} diff --git a/knot2/crates/knot-messages/src/template.rs b/knot2/crates/knot-messages/src/template.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-messages/src/template.rs @@ -0,0 +1,188 @@ +use std::fmt; + +const SPACER: &str = "\u{200b}"; + +pub trait Key: Copy + 'static { + const PLACEHOLDERS: &'static [(&'static str, Self)]; +} + +#[derive(Debug, Clone, Copy)] +pub enum NoKeys {} + +impl Key for NoKeys { + const PLACEHOLDERS: &'static [(&'static str, Self)] = &[]; +} + +pub trait Shape { + type Repr; +} + +#[derive(Debug)] +pub struct Lines; + +impl Shape for Lines { + type Repr = Vec>>; +} + +#[derive(Debug)] +pub struct Line; + +impl Shape for Line { + type Repr = Vec>; +} + +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum TemplateError { + #[error("{field}: unknown placeholder {{{name}}}")] + UnknownPlaceholder { field: &'static str, name: String }, + #[error("{field}: unclosed {{ in template")] + UnclosedBrace { field: &'static str }, + #[error("{field}: stray }} in template")] + StrayBrace { field: &'static str }, + #[error("{field}: template mustn't be empty")] + Empty { field: &'static str }, + #[error("{field}: template must be a single line")] + Multiline { field: &'static str }, +} + +#[derive(Debug)] +pub enum Segment { + Literal(String), + Placeholder(K), +} + +pub struct Template { + repr: S::Repr, +} + +impl fmt::Debug for Template +where + S::Repr: fmt::Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Template") + .field("repr", &self.repr) + .finish() + } +} + +impl Template { + pub fn parse_lines(field: &'static str, texts: &[String]) -> Result { + Ok(Self { + repr: texts + .iter() + .map(|text| line_segments(field, text)) + .collect::>()?, + }) + } + + pub fn lines(&self, resolve: impl Fn(K) -> String) -> Vec { + self.repr + .iter() + .map(|segments| { + let rendered = render_segments(segments, &resolve); + match rendered.is_empty() { + true => SPACER.to_string(), + false => rendered, + } + }) + .collect() + } +} + +impl Template { + // A reject reason is a Line - `receive-pack` gives it as + // "ng \n" inside one `pkt-line`. + // `line_segments` won't accept newlines + // so the reason stays on that one line. + pub fn parse(field: &'static str, text: &str) -> Result { + match text.is_empty() { + true => Err(TemplateError::Empty { field }), + false => Ok(Self { + repr: line_segments(field, text)?, + }), + } + } + + pub fn line(&self, resolve: impl Fn(K) -> String) -> String { + render_segments(&self.repr, &resolve) + } +} + +impl Template { + pub fn text_lines(&self) -> Vec { + self.lines(|key| match key {}) + } +} + +impl Template { + pub fn text(&self) -> String { + self.line(|key| match key {}) + } +} + +fn line_segments( + field: &'static str, + text: &str, +) -> Result>, TemplateError> { + match text.contains('\n') { + true => Err(TemplateError::Multiline { field }), + false => segments(field, text), + } +} + +fn render_segments(segments: &[Segment], resolve: &dyn Fn(K) -> String) -> String { + segments + .iter() + .map(|segment| match segment { + Segment::Literal(text) => text.clone(), + Segment::Placeholder(key) => resolve(*key), + }) + .collect() +} + +fn segments(field: &'static str, text: &str) -> Result>, TemplateError> { + let Some(at) = text.find(['{', '}']) else { + return Ok(literal(text)); + }; + let (before, rest) = text.split_at(at); + let (parsed, remainder) = brace(field, rest)?; + Ok(literal(before) + .into_iter() + .chain(parsed) + .chain(segments(field, remainder)?) + .collect()) +} + +fn literal(text: &str) -> Vec> { + match text.is_empty() { + true => Vec::new(), + false => vec![Segment::Literal(text.to_string())], + } +} + +type Braced<'a, K> = (Option>, &'a str); + +fn brace<'a, K: Key>(field: &'static str, rest: &'a str) -> Result, TemplateError> { + if let Some(after) = rest.strip_prefix("{{") { + return Ok((Some(Segment::Literal("{".to_string())), after)); + } + if let Some(after) = rest.strip_prefix("}}") { + return Ok((Some(Segment::Literal("}".to_string())), after)); + } + if rest.starts_with('}') { + return Err(TemplateError::StrayBrace { field }); + } + let close = rest + .find('}') + .ok_or(TemplateError::UnclosedBrace { field })?; + let name = &rest[1..close]; + K::PLACEHOLDERS + .iter() + .find(|(candidate, _)| *candidate == name) + .map(|(_, key)| (Some(Segment::Placeholder(*key)), &rest[close + 1..])) + .ok_or_else(|| TemplateError::UnknownPlaceholder { + field, + name: name.to_string(), + }) +} diff --git a/knot2/crates/knot-migrate/src/adopt.rs b/knot2/crates/knot-migrate/src/adopt.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-migrate/src/adopt.rs @@ -0,0 +1,337 @@ +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use knot_git::{GitError, Layout, Repo}; +use knot_types::{ObjectFormat, RepoDid}; + +use crate::mapping::AdoptRepo; +use crate::source::SourceRepoDid; + +#[derive(Debug, thiserror::Error)] +pub enum AdoptError { + #[error("layout path for {repo}: {source}")] + Layout { repo: RepoDid, source: GitError }, + #[error("repo {repo} resolves to the reserved knot meta-repo path")] + ReservesMeta { repo: RepoDid }, + #[error("place {path}: {source}")] + Place { + path: PathBuf, + source: std::io::Error, + }, + #[error("sync {path}: {source}")] + Sync { + path: PathBuf, + source: std::io::Error, + }, + #[error("adopted repo {repo} doesn't open as a git repository: {source}")] + Unopenable { repo: RepoDid, source: GitError }, + #[error("source directory for {repo} vanished between mapping and adoption")] + Vanished { repo: RepoDid }, + #[error("consuming the source needs {scan_path} and {target} on one filesystem")] + CrossDeviceConsume { scan_path: PathBuf, target: PathBuf }, +} + +#[derive(Debug, PartialEq, Eq)] +pub struct AdoptOutcome { + pub transfer: Transfer, + pub adopted: u64, + pub already_present: u64, + pub sha1: u64, + pub sha256: u64, +} + +impl AdoptOutcome { + fn empty(transfer: Transfer) -> Self { + Self { + transfer, + adopted: 0, + already_present: 0, + sha1: 0, + sha256: 0, + } + } + + fn merge(self, other: Self) -> Self { + Self { + transfer: self.transfer, + adopted: self.adopted + other.adopted, + already_present: self.already_present + other.already_present, + sha1: self.sha1 + other.sha1, + sha256: self.sha256 + other.sha256, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourcePolicy { + Preserve, + Consume, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Transfer { + Rename, + Copy, +} + +impl std::fmt::Display for Transfer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Rename => f.write_str("rename"), + Self::Copy => f.write_str("copy"), + } + } +} + +enum Placement { + AlreadyPresent, + Staged(PathBuf), + Moved, +} + +struct Staged<'repo> { + did: &'repo RepoDid, + destination: PathBuf, + placement: Placement, +} + +pub fn source_dir(source_root: &Path, repo_did: &SourceRepoDid) -> PathBuf { + source_root.join(repo_did.as_str()) +} + +pub fn source_is_repo(source_root: &Path, repo_did: &SourceRepoDid) -> bool { + source_dir(source_root, repo_did).join("HEAD").is_file() +} + +pub fn adopt_all( + layout: &Layout, + source_root: &Path, + repos: &[AdoptRepo], + policy: SourcePolicy, +) -> Result { + let root = layout.scratch_dir(); + std::fs::create_dir_all(root).map_err(|source| AdoptError::Place { + path: root.to_path_buf(), + source, + })?; + let transfer = transfer_mode(source_root, root, policy)?; + let staged = in_lanes(repos, |repo| stage_one(layout, source_root, repo, transfer))?; + sync_filesystem(root)?; + staged.iter().try_for_each(commit_one)?; + sync_filesystem(root)?; + in_lanes(&staged, |staged| count_one(staged, transfer)).map(|counted| { + counted + .into_iter() + .fold(AdoptOutcome::empty(transfer), AdoptOutcome::merge) + }) +} + +fn transfer_mode( + source_root: &Path, + target_root: &Path, + policy: SourcePolicy, +) -> Result { + use std::os::unix::fs::MetadataExt; + let device = |path: &Path| { + std::fs::metadata(path) + .map(|meta| meta.dev()) + .map_err(|source| AdoptError::Place { + path: path.to_path_buf(), + source, + }) + }; + let one_filesystem = device(source_root)? == device(target_root)?; + match (policy, one_filesystem) { + (SourcePolicy::Consume, true) => Ok(Transfer::Rename), + (SourcePolicy::Consume, false) => Err(AdoptError::CrossDeviceConsume { + scan_path: source_root.to_path_buf(), + target: target_root.to_path_buf(), + }), + (SourcePolicy::Preserve, _) => Ok(Transfer::Copy), + } +} + +fn in_lanes<'items, T, R, F>(items: &'items [T], work: F) -> Result, AdoptError> +where + T: Sync, + R: Send, + F: Fn(&'items T) -> Result + Sync, +{ + let cursor = AtomicUsize::new(0); + std::thread::scope(|scope| { + (0..lane_count(items.len())) + .map(|_| { + scope.spawn(|| { + std::iter::from_fn(|| items.get(cursor.fetch_add(1, Ordering::Relaxed))) + .map(&work) + .collect::, AdoptError>>() + }) + }) + .collect::>() + .into_iter() + .map(|lane| lane.join().expect("adoption lane doesn't panic")) + .collect::>, AdoptError>>() + .map(|lanes| lanes.into_iter().flatten().collect()) + }) +} + +fn lane_count(items: usize) -> usize { + std::thread::available_parallelism() + .map(std::num::NonZeroUsize::get) + .unwrap_or(1) + .min(items.max(1)) +} + +fn stage_one<'repo>( + layout: &Layout, + source_root: &Path, + repo: &'repo AdoptRepo, + transfer: Transfer, +) -> Result, AdoptError> { + let destination = layout + .guarded_path(&repo.did) + .map_err(|source| match source { + GitError::ReservedDid(_) => AdoptError::ReservesMeta { + repo: repo.did.clone(), + }, + source => AdoptError::Layout { + repo: repo.did.clone(), + source, + }, + })?; + match destination.exists() { + true => Ok(Staged { + did: &repo.did, + destination, + placement: Placement::AlreadyPresent, + }), + false => { + let source = source_dir(source_root, &repo.source_did); + match source.is_dir() { + false => Err(AdoptError::Vanished { + repo: repo.did.clone(), + }), + true => place(&source, &destination, transfer).map(|placement| Staged { + did: &repo.did, + destination, + placement, + }), + } + } + } +} + +fn place(source: &Path, destination: &Path, transfer: Transfer) -> Result { + match transfer { + Transfer::Rename => { + make_parent(destination)?; + std::fs::rename(source, destination) + .map(|()| Placement::Moved) + .map_err(|error| AdoptError::Place { + path: destination.to_path_buf(), + source: error, + }) + } + Transfer::Copy => stage_tree(source, destination).map(Placement::Staged), + } +} + +fn commit_one(staged: &Staged<'_>) -> Result<(), AdoptError> { + match &staged.placement { + Placement::AlreadyPresent | Placement::Moved => Ok(()), + Placement::Staged(staging) => { + std::fs::rename(staging, &staged.destination).map_err(|source| AdoptError::Place { + path: staged.destination.clone(), + source, + }) + } + } +} + +fn count_one(staged: &Staged<'_>, transfer: Transfer) -> Result { + let opened = Repo::open(&staged.destination).map_err(|source| AdoptError::Unopenable { + repo: staged.did.clone(), + source, + })?; + let fresh = u64::from(!matches!(staged.placement, Placement::AlreadyPresent)); + let counted = AdoptOutcome { + adopted: fresh, + already_present: 1 - fresh, + ..AdoptOutcome::empty(transfer) + }; + Ok(match opened.object_format() { + ObjectFormat::SHA1 => AdoptOutcome { sha1: 1, ..counted }, + _ => AdoptOutcome { + sha256: 1, + ..counted + }, + }) +} + +fn make_parent(destination: &Path) -> Result<&Path, AdoptError> { + let parent = destination + .parent() + .expect("layout repo paths always have a parent"); + std::fs::create_dir_all(parent) + .map(|()| parent) + .map_err(|source| AdoptError::Place { + path: parent.to_path_buf(), + source, + }) +} + +fn stage_tree(source: &Path, destination: &Path) -> Result { + let io = |path: &Path| { + let path = path.to_path_buf(); + move |source: std::io::Error| AdoptError::Place { path, source } + }; + let parent = make_parent(destination)?; + let staging = parent.join(format!( + ".migrate-staging.{}", + destination + .file_name() + .expect("layout repo paths always have a file name") + .to_string_lossy() + )); + if staging.exists() { + std::fs::remove_dir_all(&staging).map_err(io(&staging))?; + } + place_tree(source, &staging).map(|()| staging) +} + +fn place_tree(source: &Path, destination: &Path) -> Result<(), AdoptError> { + walkdir::WalkDir::new(source) + .into_iter() + .try_for_each(|entry| { + let entry = entry.map_err(|error| AdoptError::Place { + path: source.to_path_buf(), + source: error.into(), + })?; + let relative = entry + .path() + .strip_prefix(source) + .expect("walkdir yields paths under its root"); + let target = destination.join(relative); + let io = |source: std::io::Error| AdoptError::Place { + path: entry.path().to_path_buf(), + source, + }; + match entry.file_type() { + kind if kind.is_dir() => std::fs::create_dir_all(&target).map_err(io), + #[cfg(unix)] + kind if kind.is_symlink() => std::fs::read_link(entry.path()) + .and_then(|link| std::os::unix::fs::symlink(link, &target)) + .map_err(io), + _ => std::fs::copy(entry.path(), &target).map(|_| ()).map_err(io), + } + }) +} + +fn sync_filesystem(root: &Path) -> Result<(), AdoptError> { + std::fs::File::open(root) + .and_then(|anchor| rustix::fs::syncfs(&anchor).map_err(std::io::Error::from)) + .map_err(|source| AdoptError::Sync { + path: root.to_path_buf(), + source, + }) +} diff --git a/knot2/crates/knot-migrate/src/casbin.rs b/knot2/crates/knot-migrate/src/casbin.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-migrate/src/casbin.rs @@ -0,0 +1,301 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use crate::source::{AclRow, SourceDid, SourceRepoDid, SourceRepoName, SourceRepoObject}; + +const DOMAIN: &str = "thisserver"; +const SKIPPED_ROLES: [&str; 6] = [ + "repo:push", + "repo:settings", + "repo:invite", + "repo:delete", + "repo:create", + "server:invite", +]; + +#[derive(Debug, thiserror::Error)] +pub enum CasbinError { + #[error("unrecognized acl row: {p_type},{v0},{v1},{v2},{v3}")] + UnrecognizedRow { + p_type: String, + v0: String, + v1: String, + v2: String, + v3: String, + }, + #[error("acl names two server owners: {first} and {second}")] + TwoServerOwners { first: SourceDid, second: SourceDid }, +} + +#[derive(Debug, Default, PartialEq, Eq)] +pub struct AclRoster { + pub server_owner: Option, + pub members: BTreeSet, + pub owner_markers: BTreeMap>, + pub collaborators: BTreeMap>, + pub slash_collaborators: BTreeMap>, + pub slash_owner_markers: u64, + pub slash_collab_rows: u64, + pub unresolved_slash_forms: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SlashTarget { + Unique(SourceRepoDid), + Ambiguous, +} + +pub type SlashResolver = BTreeMap<(SourceDid, SourceRepoName), SlashTarget>; + +pub fn resolver( + pairs: impl Iterator, +) -> SlashResolver { + pairs.fold(BTreeMap::new(), |mut map, (owner, name, repo)| { + map.entry((owner, name)) + .and_modify(|target| *target = SlashTarget::Ambiguous) + .or_insert(SlashTarget::Unique(repo)); + map + }) +} + +pub fn decode(rows: &[AclRow], resolve: &SlashResolver) -> Result { + rows.iter().try_fold(AclRoster::default(), |roster, row| { + step(roster, row, resolve) + }) +} + +fn step( + mut roster: AclRoster, + row: &AclRow, + resolve: &SlashResolver, +) -> Result { + match ( + row.p_type.as_str(), + row.v0.as_str(), + row.v1.as_str(), + row.v2.as_str(), + row.v3.as_str(), + ) { + // loadbearing ordering because of how old knots used to work + ("g", "server:owner", "server:member", DOMAIN, _) => Ok(roster), + ("g", did, "server:member", DOMAIN, _) => { + roster.members.insert(SourceDid::from_column(did)); + Ok(roster) + } + ("g", did, "server:owner", DOMAIN, _) => match roster.server_owner.take() { + Some(first) if first.as_str() != did => Err(CasbinError::TwoServerOwners { + first, + second: SourceDid::from_column(did), + }), + _ => { + roster.server_owner = Some(SourceDid::from_column(did)); + Ok(roster) + } + }, + ("p", _, DOMAIN, _, role) if SKIPPED_ROLES.contains(&role) => Ok(roster), + ("p", did, DOMAIN, object, "repo:owner") => Ok(mark( + roster, + Marker::Owner, + &SourceDid::from_column(did), + &SourceRepoObject::from_column(object), + resolve, + )), + ("p", did, DOMAIN, object, "repo:collaborator") => Ok(mark( + roster, + Marker::Collaborator, + &SourceDid::from_column(did), + &SourceRepoObject::from_column(object), + resolve, + )), + _ => Err(CasbinError::UnrecognizedRow { + p_type: row.p_type.clone(), + v0: row.v0.clone(), + v1: row.v1.clone(), + v2: row.v2.clone(), + v3: row.v3.clone(), + }), + } +} + +#[derive(Clone, Copy)] +enum Marker { + Owner, + Collaborator, +} + +fn mark( + mut roster: AclRoster, + marker: Marker, + did: &SourceDid, + object: &SourceRepoObject, + resolve: &SlashResolver, +) -> AclRoster { + match (object.as_str().split_once('/'), marker) { + (None, Marker::Owner) => { + roster + .owner_markers + .entry(SourceRepoDid::from_column(object.as_str())) + .or_default() + .insert(did.clone()); + } + (None, Marker::Collaborator) => { + roster + .collaborators + .entry(SourceRepoDid::from_column(object.as_str())) + .or_default() + .insert(did.clone()); + } + (Some((owner, name)), marker) => { + match marker { + Marker::Owner => roster.slash_owner_markers += 1, + Marker::Collaborator => roster.slash_collab_rows += 1, + } + let resolved = resolve.get(&( + SourceDid::from_column(owner), + SourceRepoName::from_column(name), + )); + match (resolved, marker) { + (Some(SlashTarget::Unique(repo)), Marker::Collaborator) => { + roster + .slash_collaborators + .entry(repo.clone()) + .or_default() + .insert(did.clone()); + } + (Some(SlashTarget::Unique(_)), Marker::Owner) => {} + (Some(SlashTarget::Ambiguous), _) | (None, _) => { + roster.unresolved_slash_forms.push(object.to_string()) + } + } + } + } + roster +} + +#[cfg(test)] +mod tests { + use super::*; + + fn g(did: &str, role: &str) -> AclRow { + AclRow { + p_type: "g".into(), + v0: did.into(), + v1: role.into(), + v2: DOMAIN.into(), + v3: String::new(), + } + } + + fn p(did: &str, object: &str, role: &str) -> AclRow { + AclRow { + p_type: "p".into(), + v0: did.into(), + v1: DOMAIN.into(), + v2: object.into(), + v3: role.into(), + } + } + + #[test] + fn decodes_members_owner_and_repo_markers() { + let rows = [ + g("did:plc:nel", "server:owner"), + g("server:owner", "server:member"), + g("did:plc:olaren", "server:member"), + g("did:plc:teq", "server:member"), + p("did:plc:nel", "did:plc:squid", "repo:owner"), + p("did:plc:nel", "did:plc:squid", "repo:push"), + p("did:plc:nel", "did:plc:squid", "repo:settings"), + p("did:plc:nel", "did:plc:squid", "repo:invite"), + p("did:plc:nel", "did:plc:squid", "repo:delete"), + p("did:plc:teq", "did:plc:squid", "repo:collaborator"), + p("did:plc:nel", "", "repo:create"), + p("did:plc:nel", "", "server:invite"), + ]; + let roster = decode(&rows, &SlashResolver::new()).unwrap(); + assert_eq!( + roster.server_owner.as_ref().map(SourceDid::as_str), + Some("did:plc:nel") + ); + assert_eq!(roster.members.len(), 2); + assert_eq!( + roster.owner_markers["did:plc:squid"], + BTreeSet::from([SourceDid::from_column("did:plc:nel")]) + ); + assert_eq!( + roster.collaborators["did:plc:squid"], + BTreeSet::from([SourceDid::from_column("did:plc:teq")]) + ); + } + + #[test] + fn slash_forms_resolve_through_repo_keys_unless_ambiguous() { + let resolve = resolver( + [ + ("did:plc:nel", "anemone", "did:plc:limpet"), + ("did:plc:isabel", "mussel", "did:plc:whelk"), + ("did:plc:isabel", "mussel", "did:plc:conch"), + ] + .into_iter() + .map(|(owner, name, repo)| { + ( + SourceDid::from_column(owner), + SourceRepoName::from_column(name), + SourceRepoDid::from_column(repo), + ) + }), + ); + assert_eq!( + resolve[&( + SourceDid::from_column("did:plc:isabel"), + SourceRepoName::from_column("mussel") + )], + SlashTarget::Ambiguous + ); + + let roster = decode( + &[ + p("did:plc:nel", "did:plc:nel/anemone", "repo:owner"), + p("did:plc:isabel", "did:plc:nel/anemone", "repo:collaborator"), + p("did:plc:nel", "did:plc:nel/vanished", "repo:owner"), + p("did:plc:teq", "did:plc:isabel/mussel", "repo:collaborator"), + ], + &resolve, + ) + .unwrap(); + assert_eq!(roster.slash_owner_markers, 2); + assert_eq!(roster.slash_collab_rows, 2); + assert!(roster.owner_markers.is_empty()); + assert!(roster.collaborators.is_empty()); + assert!(roster.slash_collaborators["did:plc:limpet"].contains("did:plc:isabel")); + assert_eq!( + roster.slash_collaborators.len(), + 1, + "an ambiguous slash form grants nobody" + ); + assert_eq!( + roster.unresolved_slash_forms, + ["did:plc:nel/vanished", "did:plc:isabel/mussel"] + ); + } + + #[test] + fn decode_rejects_unknown_roles_and_two_server_owners() { + assert!(matches!( + decode( + &[p("did:plc:nel", "did:plc:squid", "repo:mystery")], + &SlashResolver::new() + ), + Err(CasbinError::UnrecognizedRow { .. }) + )); + assert!(matches!( + decode( + &[ + g("did:plc:nel", "server:owner"), + g("did:plc:bailey", "server:owner"), + ], + &SlashResolver::new() + ), + Err(CasbinError::TwoServerOwners { .. }) + )); + } +} diff --git a/knot2/crates/knot-migrate/src/emit.rs b/knot2/crates/knot-migrate/src/emit.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-migrate/src/emit.rs @@ -0,0 +1,533 @@ +use std::path::{Path, PathBuf}; + +use knot_cob::{Checkpoint, CobError, CobHome, CobStore, Evaluate}; +use knot_cobs::{ + CollaboratorsChange, CollaboratorsCob, Grant, MembersChange, MembersCob, Registration, + RegistryChange, RegistryError, RepoRegistryCob, +}; +use knot_git::{GitError, Layout}; +use knot_runtime::Signer; +use knot_types::{AccountDid, ActorId, KnotHostname, KnotId, ObjectFormat, RepoDid, RepoName}; +use serde::Serialize; +use serde::de::DeserializeOwned; +use url::Url; + +use crate::mapping::{AdoptRepo, MappedGrant, Mapping}; + +#[derive(Debug, thiserror::Error)] +pub enum EmitError { + #[error("meta-repo bootstrap: {0}")] + Meta(#[from] GitError), + #[error("open adopted repo {repo}: {source}")] + OpenRepo { repo: RepoDid, source: GitError }, + #[error("registry record for {repo} has name {existing:?} and the mapping has {name:?}")] + RegistryNameChanged { + repo: RepoDid, + existing: RepoName, + name: RepoName, + }, + #[error("{cob} write failed: {source}")] + Cob { cob: &'static str, source: CobError }, + #[error("registry write failed: {0}")] + Registry(#[from] RegistryError), + #[error("{cob} is split across {count} objects")] + SplitObject { cob: &'static str, count: usize }, + #[error("write {path}: {source}")] + Io { + path: PathBuf, + source: std::io::Error, + }, + #[error("host key {path} doesn't parse as an OpenSSH private key: {source}")] + HostKey { + path: PathBuf, + source: ssh_key::Error, + }, + #[error("knot cannot load the passphrase-protected host key {path} unattended")] + EncryptedHostKey { path: PathBuf }, + #[error("config template has no line for {section}.{key}")] + TemplateDrift { + section: &'static str, + key: &'static str, + }, + #[error("{cob} is missing {missing} entries after append")] + Incomplete { cob: &'static str, missing: usize }, +} + +#[derive(Debug, Default, PartialEq, Eq)] +pub struct GrantSetOutcome { + pub appended: u64, + pub already_present: u64, +} + +pub struct CobSummary { + pub members: GrantSetOutcome, + pub registrations: GrantSetOutcome, + pub collaborators: GrantSetOutcome, +} + +pub fn write_cobs( + layout: &Layout, + knot: &KnotId, + mapping: &Mapping, + signer: &dyn Signer, +) -> Result { + let meta = layout.bootstrap_meta(knot)?; + let home = CobHome::from(knot); + let store = CobStore::new(&meta); + + let members = write_grant_set::( + &store, + &home, + "members", + &mapping.members, + signer, + MembersChange::Add, + )?; + let registrations = write_registry(&store, &home, &mapping.repos, signer)?; + let collaborators = + mapping + .repos + .iter() + .try_fold(GrantSetOutcome::default(), |outcome, repo| { + let sum = write_repo_collaborators(layout, repo, signer)?; + Ok::<_, EmitError>(GrantSetOutcome { + appended: outcome.appended + sum.appended, + already_present: outcome.already_present + sum.already_present, + }) + })?; + + Ok(CobSummary { + members, + registrations, + collaborators, + }) +} + +fn write_repo_collaborators( + layout: &Layout, + repo: &AdoptRepo, + signer: &dyn Signer, +) -> Result { + if repo.collaborators.is_empty() { + return Ok(GrantSetOutcome::default()); + } + let git = layout + .open(&repo.did) + .map_err(|source| EmitError::OpenRepo { + repo: repo.did.clone(), + source, + })?; + let home = CobHome::from(&repo.did); + let store = CobStore::new(&git); + write_grant_set::( + &store, + &home, + "collaborators", + &repo.collaborators, + signer, + CollaboratorsChange::Add, + ) +} + +fn write_grant_set( + store: &CobStore, + home: &CobHome, + cob: &'static str, + grants: &[MappedGrant], + signer: &dyn Signer, + make: impl Fn(Grant) -> E::Change, +) -> Result +where + E: Checkpoint + Evaluate, + E::State: Serialize + DeserializeOwned, +{ + write_batch::( + store, + home, + cob, + grants, + signer, + |grant| make(to_grant(grant)), + |grant| grant.created_at, + |roster, grant| roster.contains(&grant.subject), + |_| Ok(()), + ) +} + +fn write_registry( + store: &CobStore, + home: &CobHome, + repos: &[AdoptRepo], + signer: &dyn Signer, +) -> Result { + write_batch::( + store, + home, + "registry", + repos, + signer, + |repo| RegistryChange::Register(registration(repo)), + |repo| repo.created_at, + |registry, repo| { + registry.record_of(&repo.did).is_some_and(|record| { + record.owner == repo.owner && record.rkey == repo.rkey && record.name == repo.name + }) + }, + |registry| { + repos.iter().try_for_each(|repo| { + match ( + registry.record_of(&repo.did), + registry.resolve(&repo.owner, &repo.rkey), + ) { + (Some(record), _) if record.owner != repo.owner || record.rkey != repo.rkey => { + Err(EmitError::Registry(RegistryError::AlreadyRegistered { + repo: repo.did.clone(), + owner: record.owner.clone(), + rkey: record.rkey.clone(), + })) + } + (Some(record), _) if record.name != repo.name => { + Err(EmitError::RegistryNameChanged { + repo: repo.did.clone(), + existing: record.name.clone(), + name: repo.name.clone(), + }) + } + (None, Some(holder)) if holder != &repo.did => { + Err(EmitError::Registry(RegistryError::RkeyTaken { + owner: repo.owner.clone(), + rkey: repo.rkey.clone(), + existing: holder.clone(), + })) + } + _ => Ok(()), + } + }) + }, + ) +} + +#[allow(clippy::too_many_arguments)] +fn write_batch( + store: &CobStore, + home: &CobHome, + cob: &'static str, + items: &[T], + signer: &dyn Signer, + make: impl Fn(&T) -> E::Change, + stamp: impl Fn(&T) -> knot_types::UnixSeconds, + present: impl Fn(&E::State, &T) -> bool, + precheck: impl Fn(&E::State) -> Result<(), EmitError>, +) -> Result +where + E: Checkpoint, + E::State: Serialize + DeserializeOwned, +{ + let fail = |source: CobError| EmitError::Cob { cob, source }; + let objects = store.list::().map_err(fail)?; + let (object, state, created) = match (objects.as_slice(), items) { + (_, []) => return Ok(GrantSetOutcome::default()), + ([], [first, ..]) => { + let change = make(first); + let created = store + .create(home, &change, signer, stamp(first)) + .map_err(fail)?; + let author = ActorId::from_secp256k1(signer.public_key().as_bytes()); + (created.object, E::apply(E::initial(), change, &author), 1) + } + ([object], _) => { + let (state, _) = store.materialize::(*object).map_err(fail)?; + (*object, state, 0) + } + (many, _) => { + return Err(EmitError::SplitObject { + cob, + count: many.len(), + }); + } + }; + precheck(&state)?; + + let missing: Vec<&T> = items.iter().filter(|item| !present(&state, item)).collect(); + missing.split_last().map_or(Ok(()), |(last, head)| { + let changes: Vec = head.iter().map(|item| make(item)).collect(); + store + .extend( + home, + object, + changes.iter().zip(head.iter().map(|item| stamp(item))), + signer, + ) + .map_err(fail)?; + store + .update_with_checkpointed::(home, object, signer, stamp(last), |_| { + Ok(make(last)) + }) + .map(|_| ()) + .map_err(fail) + })?; + + let (folded, _) = store.materialize::(object).map_err(fail)?; + let absent = items.iter().filter(|item| !present(&folded, item)).count(); + match absent { + 0 => Ok(GrantSetOutcome { + appended: created + missing.len() as u64, + already_present: (items.len() as u64) + .saturating_sub(missing.len() as u64) + .saturating_sub(created), + }), + count => Err(EmitError::Incomplete { + cob, + missing: count, + }), + } +} + +fn registration(repo: &AdoptRepo) -> Registration { + Registration { + owner: repo.owner.clone(), + rkey: repo.rkey.clone(), + name: repo.name.clone(), + repo: repo.did.clone(), + created_at: repo.created_at, + } +} + +fn to_grant(grant: &MappedGrant) -> Grant { + Grant { + subject: grant.subject.clone(), + added_by: grant.added_by.clone(), + created_at: grant.created_at, + } +} + +#[derive(Serialize)] +struct ArchivedKey<'a> { + repo_did: &'a RepoDid, + key_type: &'a str, + #[serde(serialize_with = "secret_str")] + secret_key_hex: zeroize::Zeroizing, +} + +fn secret_str( + value: &zeroize::Zeroizing, + serializer: S, +) -> Result { + serializer.serialize_str(value) +} + +// do not be alarmed, for there is a plan for this +pub fn write_key_archive(path: &Path, repos: &[AdoptRepo]) -> Result<(), EmitError> { + let keys: Vec> = repos + .iter() + .map(|repo| ArchivedKey { + repo_did: &repo.did, + key_type: "k256", + secret_key_hex: repo.signing_key.to_hex(), + }) + .collect(); + let body = zeroize::Zeroizing::new( + serde_json::to_string_pretty(&keys).expect("key archive serializes"), + ); + write_private(path, body.as_bytes()) +} + +pub struct HostKey { + bytes: zeroize::Zeroizing>, + pub algorithm: ssh_key::Algorithm, +} + +impl HostKey { + pub fn write_to(&self, destination: &Path) -> Result<(), EmitError> { + write_private(destination, &self.bytes) + } +} + +pub fn load_host_key(source: &Path) -> Result { + let bytes = zeroize::Zeroizing::new(std::fs::read(source).map_err(|error| EmitError::Io { + path: source.to_path_buf(), + source: error, + })?); + let key = ssh_key::PrivateKey::from_openssh(bytes.as_slice()).map_err(|error| { + EmitError::HostKey { + path: source.to_path_buf(), + source: error, + } + })?; + if key.is_encrypted() { + return Err(EmitError::EncryptedHostKey { + path: source.to_path_buf(), + }); + } + Ok(HostKey { + algorithm: key.algorithm(), + bytes, + }) +} + +fn write_private(path: &Path, bytes: &[u8]) -> Result<(), EmitError> { + use std::io::Write; + let io = |error: std::io::Error| EmitError::Io { + path: path.to_path_buf(), + source: error, + }; + let mut options = std::fs::OpenOptions::new(); + options.write(true).create(true).truncate(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options.open(path).map_err(io)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + file.set_permissions(std::fs::Permissions::from_mode(0o600)) + .map_err(io)?; + } + file.write_all(bytes).map_err(io)?; + file.sync_all().map_err(io)?; + Ok(()) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +// This name goes into the generated config as `secrets.master_key_env`, +// where knot-config runs `is_env_var_name`. +// So same rule here such the migration fails *now* instead of at the end. +pub struct MasterKeyEnv(String); + +impl MasterKeyEnv { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + let valid = !value.is_empty() + && !value.starts_with(|c: char| c.is_ascii_digit()) + && value + .chars() + .all(|c| c.is_ascii_uppercase() || c.is_ascii_digit() || c == '_'); + match valid { + true => Ok(Self(value)), + false => Err(value), + } + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl std::fmt::Display for MasterKeyEnv { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +pub struct ConfigValues { + pub hostname: KnotHostname, + pub admins: Vec, + pub scan_path: PathBuf, + pub ssh_host_key_file: PathBuf, + pub sealed_key_file: PathBuf, + pub master_key_env: MasterKeyEnv, + pub object_format: ObjectFormat, + pub plc_directory: Url, +} + +pub fn render_config(values: &ConfigValues) -> Result { + let fills: Vec<((&'static str, &'static str), String)> = [ + (("server", "hostname"), quote(values.hostname.as_str())), + ( + ("server", "admins"), + format!( + "[{}]", + values + .admins + .iter() + .map(|admin| quote(admin.as_str())) + .collect::>() + .join(", ") + ), + ), + ( + ("server", "ssh_host_key_file"), + quote_path(&values.ssh_host_key_file), + ), + (("acl", "admission"), quote("closed")), + (("repo", "scan_path"), quote_path(&values.scan_path)), + ( + ("git", "object_format"), + quote(values.object_format.capability()), + ), + ( + ("secrets", "sealed_key_file"), + quote_path(&values.sealed_key_file), + ), + ( + ("secrets", "master_key_env"), + quote(values.master_key_env.as_str()), + ), + ( + ("atproto", "plc_directory"), + quote(values.plc_directory.as_str()), + ), + ] + .into_iter() + .collect(); + + let template = knot_config::template(); + let (lines, pending, _) = template.lines().fold( + (Vec::new(), fills, ""), + |(mut lines, pending, section), line| { + let section = line + .trim() + .strip_prefix('[') + .and_then(|rest| rest.strip_suffix(']')) + .unwrap_or(section); + let matched = pending.iter().position(|((expected, key), _)| { + *expected == section && line.trim().starts_with(&format!("#{key} =")) + }); + let remaining = match matched { + Some(index) => { + let ((_, key), value) = &pending[index]; + lines.push(format!("{key} = {value}")); + pending + .into_iter() + .enumerate() + .filter(|(position, _)| *position != index) + .map(|(_, fill)| fill) + .collect() + } + None => { + lines.push(line.to_string()); + pending + } + }; + (lines, remaining, section) + }, + ); + pending.first().map_or(Ok(()), |((section, key), _)| { + Err(EmitError::TemplateDrift { section, key }) + })?; + Ok(lines.join("\n") + "\n") +} + +fn quote(value: &str) -> String { + let escaped: String = value + .chars() + .map(|c| match c { + '"' => "\\\"".to_string(), + '\\' => "\\\\".to_string(), + '\u{8}' => "\\b".to_string(), + '\t' => "\\t".to_string(), + '\n' => "\\n".to_string(), + '\u{c}' => "\\f".to_string(), + '\r' => "\\r".to_string(), + c if c.is_control() => format!("\\u{:04X}", u32::from(c)), + c => c.to_string(), + }) + .collect(); + format!("\"{escaped}\"") +} + +fn quote_path(path: &Path) -> String { + quote(&path.to_string_lossy()) +} diff --git a/knot2/crates/knot-migrate/src/envfile.rs b/knot2/crates/knot-migrate/src/envfile.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-migrate/src/envfile.rs @@ -0,0 +1,114 @@ +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +#[derive(Debug, thiserror::Error)] +pub enum EnvFileError { + #[error("read env file {path}: {source}")] + Io { + path: PathBuf, + source: std::io::Error, + }, +} + +#[derive(Debug, Default)] +pub struct EnvFile { + values: BTreeMap, +} + +impl EnvFile { + pub fn read(path: &Path) -> Result { + let body = std::fs::read_to_string(path).map_err(|source| EnvFileError::Io { + path: path.to_path_buf(), + source, + })?; + Ok(Self::parse(&body)) + } + + pub fn parse(body: &str) -> Self { + let values = body + .lines() + .map(str::trim) + .filter(|line| !line.is_empty() && !line.starts_with('#')) + .map(|line| line.strip_prefix("export ").unwrap_or(line)) + .filter_map(|line| line.split_once('=')) + .filter_map(|(key, value)| unquote(value).map(|value| (key.trim().to_string(), value))) + .collect(); + Self { values } + } + + pub fn get(&self, key: &str) -> Option<&str> { + self.values.get(key).map(String::as_str) + } +} + +fn unquote(raw: &str) -> Option { + let trimmed = raw.trim(); + ['"', '\''] + .into_iter() + .find_map(|quote| trimmed.strip_prefix(quote).map(|rest| (quote, rest))) + .map_or_else( + || Some(strip_comment(trimmed).to_string()), + |(quote, rest)| rest.split_once(quote).map(|(inner, _)| inner.to_string()), + ) +} + +fn strip_comment(value: &str) -> &str { + value + .match_indices('#') + .find(|(index, _)| { + value[..*index] + .chars() + .next_back() + .is_some_and(char::is_whitespace) + }) + .map_or(value, |(index, _)| &value[..index]) + .trim_end() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parsing_keeps_assignments_and_drops_every_other_line() { + let file = EnvFile::parse( + r#"# knot1 +KNOT_SERVER_HOSTNAME=oyster.cafe +export KNOT_SERVER_OWNER="did:plc:nel" + +KNOT_SERVER_PLC_URL='https://plc.oyster.cafe' +broken line +KNOT_REPO_SCAN_PATH=/data/repos # tangled default +KNOT_QUOTED_HOSTNAME="nel.pet" # quoted +KNOT_SERVER_SECRET='a # b' +KNOT_DEV_FLAGS=#literal +KNOT_APPVIEW_URL=https://tangled.test/#frag +KNOT_UNTERMINATED="oyster.cafe +KNOT_ALSO_UNTERMINATED='did:plc:nel +KNOT_LAST=kept +"#, + ); + assert_eq!(file.get("KNOT_SERVER_HOSTNAME"), Some("oyster.cafe")); + assert_eq!(file.get("KNOT_SERVER_OWNER"), Some("did:plc:nel")); + assert_eq!( + file.get("KNOT_SERVER_PLC_URL"), + Some("https://plc.oyster.cafe") + ); + assert_eq!(file.get("broken"), None); + assert_eq!(file.get("KNOT_REPO_SCAN_PATH"), Some("/data/repos")); + assert_eq!(file.get("KNOT_QUOTED_HOSTNAME"), Some("nel.pet")); + assert_eq!(file.get("KNOT_SERVER_SECRET"), Some("a # b")); + assert_eq!(file.get("KNOT_DEV_FLAGS"), Some("#literal")); + assert_eq!( + file.get("KNOT_APPVIEW_URL"), + Some("https://tangled.test/#frag") + ); + assert_eq!(file.get("KNOT_UNTERMINATED"), None); + assert_eq!(file.get("KNOT_ALSO_UNTERMINATED"), None); + assert_eq!( + file.get("KNOT_LAST"), + Some("kept"), + "an unterminated quote drops its own line and nothing after it" + ); + } +} diff --git a/knot2/crates/knot-migrate/src/lib.rs b/knot2/crates/knot-migrate/src/lib.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-migrate/src/lib.rs @@ -0,0 +1,7 @@ +pub mod adopt; +pub mod casbin; +pub mod emit; +pub mod envfile; +pub mod mapping; +pub mod report; +pub mod source; diff --git a/knot2/crates/knot-migrate/src/main.rs b/knot2/crates/knot-migrate/src/main.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-migrate/src/main.rs @@ -0,0 +1,492 @@ +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; + +use base64::Engine; +use knot_migrate::adopt::{self, AdoptError, SourcePolicy}; +use knot_migrate::casbin::{self, CasbinError}; +use knot_migrate::emit::{self, ConfigValues, EmitError, MasterKeyEnv}; +use knot_migrate::envfile::{EnvFile, EnvFileError}; +use knot_migrate::mapping::{self, Mapping, MappingError}; +use knot_migrate::report::Report; +use knot_migrate::source::{SourceDb, SourceError, SourceRepoDid, SourceRkey, SourceSchema}; +use knot_runtime::OsEntropy; +use knot_secrets::{MasterKey, SealedStore, SecretsError}; +use knot_types::{AccountDid, KnotHostname, ObjectFormat}; +use url::Url; + +// TODO: I wanted to see how well I could work without clap. I shoulda just used clap. +const USAGE: &str = "\ +knot-migrate: offline conversion of a tangled-knot deployment into a knot deployment + +usage: + knot-migrate --source-db --host-key --target

[options] + +options: + --source-db tangled-knot SQLite database, opened read-only + --source-repos tangled-knot scan path holding directories + defaults to KNOT_REPO_SCAN_PATH from the env file + --env-file tangled-knot environment file + --host-key system sshd host key to import + --target knot data directory to create + --hostname knot hostname, defaults to KNOT_SERVER_HOSTNAME + --plc-url PLC directory, defaults to KNOT_SERVER_PLC_URL + --object-format sha1 or sha256 for repos knot creates, default sha1 + --master-key-env env var holding the base64 master key, default KNOT_MASTER_KEY + --consume-source move the source repos into place instead of copying them, + which empties the source tree and needs one filesystem + --dry-run print the mapping and reconciliation report, write nothing +"; + +#[derive(Debug, thiserror::Error)] +enum MigrateError { + #[error(transparent)] + Source(#[from] SourceError), + #[error(transparent)] + Casbin(#[from] CasbinError), + #[error(transparent)] + Mapping(#[from] MappingError), + #[error(transparent)] + Adopt(#[from] AdoptError), + #[error(transparent)] + Emit(#[from] EmitError), + #[error(transparent)] + EnvFile(#[from] EnvFileError), + #[error(transparent)] + Git(#[from] knot_git::GitError), + #[error(transparent)] + Secrets(#[from] SecretsError), + #[error("{0}")] + Usage(String), + #[error("env file specifies knot owner {env} while the acl specifies {acl}")] + OwnerMismatch { env: String, acl: String }, + #[error("--hostname {flag} doesn't match the env file's KNOT_SERVER_HOSTNAME {env}")] + HostnameMismatch { flag: String, env: String }, + #[error("master key env var {name} isn't set")] + MissingMasterKey { name: MasterKeyEnv }, + #[error("master key env var {name} isn't base64")] + MalformedMasterKey { name: MasterKeyEnv }, + #[error("{context}: {source}")] + Io { + context: String, + source: std::io::Error, + }, +} + +struct Args { + source_db: PathBuf, + source_repos: Option, + env_file: Option, + host_key: Option, + target: PathBuf, + hostname: Option, + plc_url: Option, + object_format: ObjectFormat, + master_key_env: MasterKeyEnv, + source_policy: SourcePolicy, + dry_run: bool, +} + +#[derive(Debug, Default, Clone, Copy)] +struct Switches { + dry_run: bool, + consume_source: bool, +} + +const KNOWN_FLAGS: [&str; 9] = [ + "source-db", + "source-repos", + "env-file", + "host-key", + "target", + "hostname", + "plc-url", + "object-format", + "master-key-env", +]; + +fn parse_args(args: &[String]) -> Result { + let (mut flags, switches, pending) = args.iter().try_fold( + ( + BTreeMap::::new(), + Switches::default(), + None::, + ), + |(mut flags, switches, pending), arg| match (pending, arg.as_str()) { + (Some(key), value) if value.starts_with("--") => { + Err(MigrateError::Usage(format!("--{key} needs a value"))) + } + (Some(key), value) => match flags.insert(key.clone(), value.to_string()) { + None => Ok((flags, switches, None)), + Some(_) => Err(MigrateError::Usage(format!("--{key} given twice"))), + }, + (None, "--dry-run") => Ok(( + flags, + Switches { + dry_run: true, + ..switches + }, + None, + )), + (None, "--consume-source") => Ok(( + flags, + Switches { + consume_source: true, + ..switches + }, + None, + )), + (None, flag) => match flag.strip_prefix("--").map(|rest| { + rest.split_once('=') + .map_or((rest, None), |(key, value)| (key, Some(value))) + }) { + Some((key, None)) if KNOWN_FLAGS.contains(&key) => { + Ok((flags, switches, Some(key.to_string()))) + } + Some((key, Some(value))) if KNOWN_FLAGS.contains(&key) => { + match flags.insert(key.to_string(), value.to_string()) { + None => Ok((flags, switches, None)), + Some(_) => Err(MigrateError::Usage(format!("--{key} given twice"))), + } + } + _ => Err(MigrateError::Usage(format!("unexpected argument {flag}"))), + }, + }, + )?; + pending.map_or(Ok(()), |key| { + Err(MigrateError::Usage(format!("--{key} needs a value"))) + })?; + let mut take = |key: &str| flags.remove(key); + let required = |key: &str, value: Option| { + value.ok_or_else(|| MigrateError::Usage(format!("--{key} is required"))) + }; + let object_format = take("object-format").map_or(Ok(ObjectFormat::SHA1), |value| { + ObjectFormat::from_capability(&value).ok_or_else(|| { + MigrateError::Usage(format!( + "--object-format must be sha1 or sha256, not {value}" + )) + }) + })?; + Ok(Args { + source_db: required("source-db", take("source-db"))?.into(), + source_repos: take("source-repos").map(PathBuf::from), + env_file: take("env-file").map(PathBuf::from), + host_key: take("host-key").map(PathBuf::from), + target: required("target", take("target"))?.into(), + hostname: take("hostname"), + plc_url: take("plc-url"), + object_format, + master_key_env: take("master-key-env") + .map_or_else(|| MasterKeyEnv::new("KNOT_MASTER_KEY"), MasterKeyEnv::new) + .map_err(|value| { + MigrateError::Usage(format!( + "--master-key-env must be an uppercase env var name, not {value}" + )) + })?, + source_policy: match switches.consume_source { + true => SourcePolicy::Consume, + false => SourcePolicy::Preserve, + }, + dry_run: switches.dry_run, + }) +} + +fn main() -> ExitCode { + let args: Vec = std::env::args().skip(1).collect(); + if args.is_empty() || args.iter().any(|arg| arg == "--help" || arg == "-h") { + print!("{USAGE}"); + return ExitCode::SUCCESS; + } + match run(&args) { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + eprintln!("error: {error}"); + ExitCode::FAILURE + } + } +} + +fn run(args: &[String]) -> Result<(), MigrateError> { + let args = parse_args(args)?; + let env = args + .env_file + .as_deref() + .map(EnvFile::read) + .transpose()? + .unwrap_or_default(); + + args.hostname + .as_deref() + .zip(env.get("KNOT_SERVER_HOSTNAME")) + .filter(|(flag, env_value)| flag != env_value) + .map_or(Ok(()), |(flag, env_value)| { + Err(MigrateError::HostnameMismatch { + flag: flag.to_string(), + env: env_value.to_string(), + }) + })?; + let hostname = args + .hostname + .clone() + .or_else(|| env.get("KNOT_SERVER_HOSTNAME").map(str::to_string)) + .ok_or_else(|| { + MigrateError::Usage( + "--hostname is required when the env file specifies none".to_string(), + ) + })?; + let hostname = KnotHostname::new(hostname.as_str()).map_err(|error| { + MigrateError::Usage(format!("hostname {hostname} isn't valid: {error}")) + })?; + let plc_url = args + .plc_url + .clone() + .or_else(|| env.get("KNOT_SERVER_PLC_URL").map(str::to_string)) + .ok_or_else(|| { + MigrateError::Usage( + "--plc-url is required when the env file specifies none".to_string(), + ) + })?; + let plc_directory = Url::parse(&plc_url) + .ok() + .filter(|url| url.scheme() == "https" && url.host().is_some()) + .ok_or_else(|| { + MigrateError::Usage(format!("PLC directory {plc_url} isn't an https URL")) + })?; + let source_repos = args + .source_repos + .clone() + .or_else(|| env.get("KNOT_REPO_SCAN_PATH").map(PathBuf::from)) + .ok_or_else(|| { + MigrateError::Usage( + "--source-repos is required when the env file specifies no scan path".to_string(), + ) + })?; + match source_repos.is_dir() { + true => Ok(()), + false => Err(MigrateError::Usage(format!( + "source repos path {} isn't a directory", + source_repos.display() + ))), + }?; + + let db = SourceDb::open(&args.source_db)?; + let schema = db.schema()?; + let repos = db.repos()?; + let rkeys = repos + .iter() + .filter_map(|repo| { + db.current_rkey(&repo.repo_did) + .map(|rkey| rkey.map(|rkey| (repo.repo_did.clone(), rkey))) + .transpose() + }) + .collect::, SourceError>>()?; + let resolver = casbin::resolver(repos.iter().map(|repo| { + ( + repo.owner_did.clone(), + repo.repo_name.clone(), + repo.repo_did.clone(), + ) + })); + let acl = casbin::decode(&db.acl()?, &resolver)?; + let exists = |repo_did: &SourceRepoDid| adopt::source_is_repo(&source_repos, repo_did); + let mapping = match schema { + SourceSchema::Tables => mapping::map_tables( + &repos, + &rkeys, + &db.members()?, + &db.collaborators()?, + &acl, + exists, + )?, + SourceSchema::PreFlip => { + mapping::map_preflip(&repos, &rkeys, &db.members()?, &acl, exists)? + } + }; + env.get("KNOT_SERVER_OWNER") + .filter(|owner| AccountDid::new(*owner).ok().as_ref() != Some(&mapping.knot_owner)) + .map_or(Ok(()), |owner| { + Err(MigrateError::OwnerMismatch { + env: owner.to_string(), + acl: mapping.knot_owner.to_string(), + }) + })?; + let orphan_alias_count = db.orphan_alias_count()?; + + let written = match args.dry_run { + true => None, + false => Some(materialize( + &args, + &hostname, + &plc_directory, + &source_repos, + &mapping, + )?), + }; + print!( + "{}", + Report { + mapping: &mapping, + orphan_alias_count, + adoption: written.as_ref().map(|written| &written.adoption), + cobs: written.as_ref().map(|written| &written.cobs), + } + ); + written.map_or(Ok(()), |written| { + println!(); + println!("knot key identity: {}", written.knot_did); + println!("host key algorithm: {}", written.host_key_algorithm); + println!("config: {}", written.config_file.display()); + println!("key archive: {}", written.archive_file.display()); + Ok(()) + }) +} + +fn timed(phase: &str, work: impl FnOnce() -> Result) -> Result { + let started = std::time::Instant::now(); + let outcome = work(); + eprintln!("{phase}: {:.1}s", started.elapsed().as_secs_f64()); + outcome +} + +struct Written { + adoption: adopt::AdoptOutcome, + cobs: emit::CobSummary, + knot_did: knot_types::KnotId, + host_key_algorithm: ssh_key::Algorithm, + config_file: PathBuf, + archive_file: PathBuf, +} + +fn materialize( + args: &Args, + hostname: &KnotHostname, + plc_directory: &Url, + source_repos: &Path, + mapping: &Mapping, +) -> Result { + let host_key_source = args + .host_key + .as_deref() + .ok_or_else(|| MigrateError::Usage("--host-key is required for a real run".to_string()))?; + let host_key = emit::load_host_key(host_key_source)?; + std::fs::create_dir_all(&args.target).map_err(|source| MigrateError::Io { + context: format!("create {}", args.target.display()), + source, + })?; + let target = args + .target + .canonicalize() + .map_err(|source| MigrateError::Io { + context: format!("canonicalize {}", args.target.display()), + source, + })?; + let scan_path = target.join("repos"); + let sealed_key_file = target.join("sealed-keys"); + let host_key_file = target.join("ssh_host_key"); + let archive_file = target.join("repo-signing-keys.json"); + let config_file = target.join("config.toml"); + + let knot_did = hostname.knot_did(); + + let master_key_value = + zeroize::Zeroizing::new(std::env::var(args.master_key_env.as_str()).map_err(|_| { + MigrateError::MissingMasterKey { + name: args.master_key_env.clone(), + } + })?); + let master_key = MasterKey::new( + base64::engine::general_purpose::STANDARD + .decode(master_key_value.trim()) + .map_err(|_| MigrateError::MalformedMasterKey { + name: args.master_key_env.clone(), + })?, + )?; + let secrets = SealedStore::open(sealed_key_file.clone(), &master_key, Box::new(OsEntropy))?; + secrets.ensure(&knot_did)?; + let signer = secrets.signer(&knot_did)?; + + let layout = knot_git::Layout::new(&scan_path) + .with_object_format(args.object_format) + .reserving_meta(&knot_did)?; + let adoption = timed("adoption", || { + adopt::adopt_all(&layout, source_repos, &mapping.repos, args.source_policy) + })?; + let cobs = timed("cobs", || { + emit::write_cobs(&layout, &knot_did, mapping, &signer) + })?; + emit::write_key_archive(&archive_file, &mapping.repos)?; + host_key.write_to(&host_key_file)?; + + let config = emit::render_config(&ConfigValues { + hostname: hostname.clone(), + admins: vec![mapping.knot_owner.clone()], + scan_path, + ssh_host_key_file: host_key_file, + sealed_key_file, + master_key_env: args.master_key_env.clone(), + object_format: args.object_format, + plc_directory: plc_directory.clone(), + })?; + std::fs::write(&config_file, config).map_err(|source| MigrateError::Io { + context: format!("write {}", config_file.display()), + source, + })?; + + Ok(Written { + adoption, + cobs, + knot_did, + host_key_algorithm: host_key.algorithm, + config_file, + archive_file, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse(list: &[&str]) -> Result { + let owned: Vec = list.iter().map(|arg| arg.to_string()).collect(); + parse_args(&owned) + } + + #[test] + fn accepts_space_and_equals_forms() { + let args = parse(&[ + "--source-db=/data/knotserver.db", + "--target", + "/data/knot", + "--object-format=sha256", + "--dry-run", + ]) + .unwrap(); + assert_eq!(args.source_db, PathBuf::from("/data/knotserver.db")); + assert_eq!(args.target, PathBuf::from("/data/knot")); + assert_eq!(args.object_format, ObjectFormat::SHA256); + assert!(args.dry_run); + } + + #[test] + fn rejects_duplicates_missing_values_and_unknown_flags() { + [ + &[ + "--source-db=/data/knotserver.db", + "--target=/data/knot", + "--target", + "/data/other", + ][..], + &["--source-db"], + &["--source-db", "--target"], + &["--mystery=1", "--source-db=/data/knotserver.db"], + &["--object-format=blake3", "--source-db=/db", "--target=/t"], + ] + .into_iter() + .for_each(|args| { + assert!( + matches!(parse(args), Err(MigrateError::Usage(_))), + "{args:?} mustn't parse" + ); + }); + } +} diff --git a/knot2/crates/knot-migrate/src/mapping.rs b/knot2/crates/knot-migrate/src/mapping.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-migrate/src/mapping.rs @@ -0,0 +1,671 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt; + +use knot_types::{AccountDid, OwnerDid, ParseError, RepoDid, RepoName, RepoRkey, UnixSeconds}; +use zeroize::{Zeroize, ZeroizeOnDrop}; + +use crate::casbin::AclRoster; +use crate::source::{ + CollabRow, MemberRow, RepoRow, SourceDid, SourceKeyType, SourceRepoDid, SourceRepoName, + SourceRkey, SourceTimestamp, +}; + +#[derive(Debug, thiserror::Error)] +pub enum MappingError { + #[error("repo DID {value} doesn't parse: {source}")] + BadRepoDid { + value: SourceRepoDid, + source: ParseError, + }, + #[error("owner {value} on repo {repo} doesn't parse: {source}")] + BadOwnerDid { + repo: SourceRepoDid, + value: SourceDid, + source: ParseError, + }, + #[error("member DID {value} doesn't parse: {source}")] + BadMemberDid { + value: SourceDid, + source: ParseError, + }, + #[error("collaborator DID {value} on repo {repo} doesn't parse: {source}")] + BadCollaboratorDid { + repo: SourceRepoDid, + value: SourceDid, + source: ParseError, + }, + #[error("{context} timestamp {value} isn't RFC 3339")] + BadTimestamp { + context: &'static str, + value: SourceTimestamp, + }, + #[error("repo {repo} has a {key_type} signing key of {bytes} bytes instead of 32-byte k256")] + BadSigningKey { + repo: SourceRepoDid, + key_type: SourceKeyType, + bytes: usize, + }, + #[error("acl names no server owner")] + MissingServerOwner, + #[error("acl marks {marker} as owner of {repo} while repo_keys names {owner}")] + ConflictingOwnerMarker { + repo: SourceRepoDid, + marker: SourceDid, + owner: SourceDid, + }, + #[error("record key {owner}/{rkey} has no single alias-backed holder")] + AmbiguousRkey { owner: OwnerDid, rkey: RepoRkey }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MappedGrant { + pub subject: AccountDid, + pub added_by: AccountDid, + pub created_at: UnixSeconds, + pub unioned: bool, +} + +#[derive(Clone, PartialEq, Eq, Zeroize, ZeroizeOnDrop)] +pub struct SigningKey([u8; 32]); + +impl fmt::Debug for SigningKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("SigningKey").finish_non_exhaustive() + } +} + +impl SigningKey { + pub fn to_hex(&self) -> zeroize::Zeroizing { + zeroize::Zeroizing::new(knot_types::lowercase_hex(&self.0)) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AdoptRepo { + pub source_did: SourceRepoDid, + pub did: RepoDid, + pub owner: OwnerDid, + pub name: RepoName, + pub rkey: RepoRkey, + pub created_at: UnixSeconds, + pub signing_key: SigningKey, + pub collaborators: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SkipReason { + Name { value: SourceRepoName }, + Rkey { value: SourceRkey }, + RkeyCollision { rkey: RepoRkey, winner: RepoDid }, + NoSourceRepo, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SkippedRepo { + pub repo_did: RepoDid, + pub reason: SkipReason, + pub lost_collaborators: Vec, +} + +#[derive(Debug, Default, PartialEq, Eq)] +pub struct Drift { + pub acl_only_collaborators: Vec<(SourceRepoDid, SourceDid)>, + pub table_only_collaborators: Vec<(SourceRepoDid, SourceDid)>, + pub slash_resolved_collaborators: Vec<(SourceRepoDid, SourceDid)>, + pub orphan_collaborator_pairs: Vec<(SourceRepoDid, SourceDid)>, + pub markerless_owner_repos: Vec, + pub orphan_owner_markers: Vec, + pub extra_owner_markers: Vec<(SourceRepoDid, SourceDid)>, + pub acl_only_members: Vec, + pub table_only_members: Vec, + pub slash_owner_markers: u64, + pub slash_collab_rows: u64, + pub unresolved_slash_forms: Vec, +} + +#[derive(Debug)] +pub struct Mapping { + pub knot_owner: AccountDid, + pub members: Vec, + pub repos: Vec, + pub skipped: Vec, + pub drift: Drift, +} + +pub fn map_tables( + repos: &[RepoRow], + rkeys: &BTreeMap, + members: &[MemberRow], + collabs: &[CollabRow], + acl: &AclRoster, + exists: impl Fn(&SourceRepoDid) -> bool, +) -> Result { + let knot_owner = server_owner(acl)?; + let repo_index: BTreeMap<&SourceRepoDid, &RepoRow> = + repos.iter().map(|repo| (&repo.repo_did, repo)).collect(); + + let table_members: Vec = members + .iter() + .filter(|row| row.subject.as_str() != knot_owner.as_str()) + .map(|row| { + Ok(MappedGrant { + subject: account(&row.subject).map_err(|source| MappingError::BadMemberDid { + value: row.subject.clone(), + source, + })?, + added_by: account(&row.did).map_err(|source| MappingError::BadMemberDid { + value: row.did.clone(), + source, + })?, + created_at: unix("knot_members.created", &row.created)?, + unioned: false, + }) + }) + .collect::>()?; + + let table_member_dids: BTreeSet<&SourceDid> = members.iter().map(|row| &row.subject).collect(); + let acl_only_members: Vec = acl + .members + .iter() + .filter(|did| !table_member_dids.contains(*did) && did.as_str() != knot_owner.as_str()) + .cloned() + .collect(); + let table_only_members: Vec = members + .iter() + .filter(|row| !acl.members.contains(&row.subject)) + .map(|row| row.subject.clone()) + .collect(); + let unioned_members: Vec = acl_only_members + .iter() + .map(|did| { + Ok(MappedGrant { + subject: account(did).map_err(|source| MappingError::BadMemberDid { + value: did.clone(), + source, + })?, + added_by: knot_owner.clone(), + // every casbin-only owner is at epoch + created_at: UnixSeconds::new(0), + unioned: true, + }) + }) + .collect::>()?; + + let (live_collabs, orphan_collabs): (Vec<&CollabRow>, Vec<&CollabRow>) = collabs + .iter() + .partition(|row| repo_index.contains_key(&row.repo_did)); + let table_pairs: BTreeSet<(&SourceRepoDid, &SourceDid)> = collabs + .iter() + .map(|row| (&row.repo_did, &row.subject_did)) + .collect(); + let acl_extra: Vec<(&SourceRepoDid, &SourceDid)> = acl + .collaborators + .iter() + .flat_map(|(repo, dids)| dids.iter().map(move |did| (repo, did))) + .filter(|(repo, did)| !table_pairs.contains(&(*repo, *did))) + .collect(); + let (acl_only, acl_orphans): (Vec<_>, Vec<_>) = acl_extra + .into_iter() + .partition(|(repo, _)| repo_index.contains_key(*repo)); + let orphan_pairs: BTreeSet<(SourceRepoDid, SourceDid)> = acl_orphans + .into_iter() + .map(|(repo, did)| (repo.clone(), did.clone())) + .chain( + orphan_collabs + .iter() + .map(|row| (row.repo_did.clone(), row.subject_did.clone())), + ) + .collect(); + let table_only_collaborators: Vec<(SourceRepoDid, SourceDid)> = live_collabs + .iter() + .filter(|row| { + [&acl.collaborators, &acl.slash_collaborators] + .into_iter() + .all(|grants| { + grants + .get(&row.repo_did) + .is_none_or(|dids| !dids.contains(&row.subject_did)) + }) + }) + .map(|row| (row.repo_did.clone(), row.subject_did.clone())) + .collect(); + + let table_grants = live_collabs.iter().copied().try_fold( + BTreeMap::>::new(), + |mut grants, row| { + let grant = MappedGrant { + subject: account(&row.subject_did).map_err(|source| { + MappingError::BadCollaboratorDid { + repo: row.repo_did.clone(), + value: row.subject_did.clone(), + source, + } + })?, + added_by: account(&row.added_by_did).map_err(|source| { + MappingError::BadCollaboratorDid { + repo: row.repo_did.clone(), + value: row.added_by_did.clone(), + source, + } + })?, + created_at: unix("collaborators.created", &row.created)?, + unioned: false, + }; + let entry = grants.entry(row.repo_did.clone()).or_default(); + if !entry.iter().any(|held| held.subject == grant.subject) { + entry.push(grant); + } + Ok::<_, MappingError>(grants) + }, + )?; + let collab_grants = owner_attributed_grants(&acl_only, &repo_index, true, table_grants)?; + + let owners = owner_drift(repos, &repo_index, acl)?; + let (adopted, skipped) = classify(repos, rkeys, &collab_grants, exists)?; + + Ok(Mapping { + knot_owner, + members: table_members.into_iter().chain(unioned_members).collect(), + repos: adopted, + skipped, + drift: Drift { + acl_only_collaborators: acl_only + .into_iter() + .map(|(repo, did)| (repo.clone(), did.clone())) + .collect(), + table_only_collaborators, + slash_resolved_collaborators: acl + .slash_collaborators + .iter() + .flat_map(|(repo, dids)| dids.iter().map(move |did| (repo, did))) + .filter(|(repo, did)| !table_pairs.contains(&(*repo, *did))) + .map(|(repo, did)| (repo.clone(), did.clone())) + .collect(), + orphan_collaborator_pairs: orphan_pairs.into_iter().collect(), + markerless_owner_repos: owners.markerless, + orphan_owner_markers: owners.orphans, + extra_owner_markers: owners.extras, + acl_only_members, + table_only_members, + slash_owner_markers: acl.slash_owner_markers, + slash_collab_rows: acl.slash_collab_rows, + unresolved_slash_forms: acl.unresolved_slash_forms.clone(), + }, + }) +} + +pub fn map_preflip( + repos: &[RepoRow], + rkeys: &BTreeMap, + members: &[MemberRow], + acl: &AclRoster, + exists: impl Fn(&SourceRepoDid) -> bool, +) -> Result { + let knot_owner = server_owner(acl)?; + let repo_index: BTreeMap<&SourceRepoDid, &RepoRow> = + repos.iter().map(|repo| (&repo.repo_did, repo)).collect(); + let enrich: BTreeMap<&SourceDid, &MemberRow> = + members.iter().map(|row| (&row.subject, row)).collect(); + let table_only_members: Vec = members + .iter() + .filter(|row| !acl.members.contains(&row.subject)) + .map(|row| row.subject.clone()) + .collect(); + + let mapped_members: Vec = acl + .members + .iter() + .filter(|did| did.as_str() != knot_owner.as_str()) + .map(|did| { + let subject = account(did).map_err(|source| MappingError::BadMemberDid { + value: did.clone(), + source, + })?; + match enrich.get(did) { + Some(row) => Ok(MappedGrant { + subject, + added_by: account(&row.did).map_err(|source| MappingError::BadMemberDid { + value: row.did.clone(), + source, + })?, + created_at: unix("knot_members.created", &row.created)?, + unioned: false, + }), + None => Ok(MappedGrant { + subject, + added_by: knot_owner.clone(), + created_at: UnixSeconds::new(0), + unioned: false, + }), + } + }) + .collect::>()?; + + let combined: BTreeMap<&SourceRepoDid, BTreeSet<&SourceDid>> = acl + .collaborators + .iter() + .chain(acl.slash_collaborators.iter()) + .flat_map(|(repo, dids)| dids.iter().map(move |did| (repo, did))) + .fold(BTreeMap::new(), |mut pairs, (repo, did)| { + pairs.entry(repo).or_default().insert(did); + pairs + }); + let live = combined + .iter() + .flat_map(|(repo, dids)| dids.iter().map(move |did| (*repo, *did))); + let (resolvable, orphan_pairs): (Vec<_>, Vec<_>) = + live.partition(|(repo, _)| repo_index.contains_key(*repo)); + let collab_grants = owner_attributed_grants(&resolvable, &repo_index, false, BTreeMap::new())?; + + let owners = owner_drift(repos, &repo_index, acl)?; + let (adopted, skipped) = classify(repos, rkeys, &collab_grants, exists)?; + + Ok(Mapping { + knot_owner, + members: mapped_members, + repos: adopted, + skipped, + drift: Drift { + orphan_collaborator_pairs: orphan_pairs + .into_iter() + .map(|(repo, did)| (repo.clone(), did.clone())) + .collect(), + markerless_owner_repos: owners.markerless, + orphan_owner_markers: owners.orphans, + extra_owner_markers: owners.extras, + table_only_members, + slash_owner_markers: acl.slash_owner_markers, + slash_collab_rows: acl.slash_collab_rows, + unresolved_slash_forms: acl.unresolved_slash_forms.clone(), + ..Drift::default() + }, + }) +} + +fn owner_attributed_grants( + pairs: &[(&SourceRepoDid, &SourceDid)], + repo_index: &BTreeMap<&SourceRepoDid, &RepoRow>, + unioned: bool, + base: BTreeMap>, +) -> Result>, MappingError> { + pairs + .iter() + .copied() + .try_fold(base, |mut grants, (repo, did)| { + let row = repo_index[repo]; + let grant = MappedGrant { + subject: account(did).map_err(|source| MappingError::BadCollaboratorDid { + repo: repo.clone(), + value: did.clone(), + source, + })?, + added_by: account(&row.owner_did).map_err(|source| MappingError::BadOwnerDid { + repo: repo.clone(), + value: row.owner_did.clone(), + source, + })?, + created_at: unix("repo_keys.created_at", &row.created_at)?, + unioned, + }; + grants.entry(repo.clone()).or_default().push(grant); + Ok(grants) + }) +} + +fn server_owner(acl: &AclRoster) -> Result { + let did = acl + .server_owner + .as_ref() + .ok_or(MappingError::MissingServerOwner)?; + account(did).map_err(|source| MappingError::BadMemberDid { + value: did.clone(), + source, + }) +} + +struct OwnerDrift { + markerless: Vec, + orphans: Vec, + extras: Vec<(SourceRepoDid, SourceDid)>, +} + +fn owner_drift( + repos: &[RepoRow], + repo_index: &BTreeMap<&SourceRepoDid, &RepoRow>, + acl: &AclRoster, +) -> Result { + repos.iter().try_for_each(|repo| { + let conflicting = acl.owner_markers.get(&repo.repo_did).and_then(|markers| { + (!markers.contains(&repo.owner_did)) + .then(|| markers.iter().next().cloned()) + .flatten() + }); + match conflicting { + Some(marker) => Err(MappingError::ConflictingOwnerMarker { + repo: repo.repo_did.clone(), + marker, + owner: repo.owner_did.clone(), + }), + None => Ok(()), + } + })?; + let markerless = repos + .iter() + .filter(|repo| !acl.owner_markers.contains_key(&repo.repo_did)) + .map(|repo| repo.repo_did.clone()) + .collect(); + let orphans = acl + .owner_markers + .keys() + .filter(|repo| !repo_index.contains_key(*repo)) + .cloned() + .collect(); + let extras = repos + .iter() + .filter_map(|repo| { + acl.owner_markers + .get(&repo.repo_did) + .map(|markers| (repo, markers)) + }) + .flat_map(|(repo, markers)| { + markers + .iter() + .filter(move |marker| *marker != &repo.owner_did) + .map(move |marker| (repo.repo_did.clone(), marker.clone())) + }) + .collect(); + Ok(OwnerDrift { + markerless, + orphans, + extras, + }) +} + +fn classify( + repos: &[RepoRow], + rkeys: &BTreeMap, + collab_grants: &BTreeMap>, + exists: impl Fn(&SourceRepoDid) -> bool, +) -> Result<(Vec, Vec), MappingError> { + let (adopted, skipped) = repos.iter().try_fold( + (Vec::new(), Vec::new()), + |(mut adopted, mut skipped), row| { + match classify_one(row, rkeys, collab_grants, &exists)? { + Ok(repo) => adopted.push(repo), + Err(skip) => skipped.push(skip), + } + Ok::<_, MappingError>((adopted, skipped)) + }, + )?; + let alias_backed: BTreeSet = rkeys + .keys() + .filter_map(|did| RepoDid::new(did.as_str()).ok()) + .collect(); + resolve_rkey_collisions(adopted, skipped, &alias_backed) +} + +fn resolve_rkey_collisions( + adopted: Vec, + skipped: Vec, + alias_backed: &BTreeSet, +) -> Result<(Vec, Vec), MappingError> { + let groups: BTreeMap<(&OwnerDid, &RepoRkey), Vec> = + adopted + .iter() + .enumerate() + .fold(BTreeMap::new(), |mut groups, (index, repo)| { + groups + .entry((&repo.owner, &repo.rkey)) + .or_default() + .push(index); + groups + }); + let losers: BTreeMap = groups + .into_iter() + .filter(|(_, indices)| indices.len() > 1) + .map(|((owner, rkey), indices)| { + let backed: Vec = indices + .iter() + .copied() + .filter(|index| alias_backed.contains(&adopted[*index].did)) + .collect(); + match backed.as_slice() { + [winner] => Ok(indices + .into_iter() + .filter(|index| index != &*winner) + .map(|index| { + ( + index, + SkippedRepo { + repo_did: adopted[index].did.clone(), + reason: SkipReason::RkeyCollision { + rkey: rkey.clone(), + winner: adopted[*winner].did.clone(), + }, + lost_collaborators: adopted[index] + .collaborators + .iter() + .map(|grant| grant.subject.clone()) + .collect(), + }, + ) + }) + .collect::>()), + _ => Err(MappingError::AmbiguousRkey { + owner: owner.clone(), + rkey: rkey.clone(), + }), + } + }) + .collect::, MappingError>>()? + .into_iter() + .flatten() + .collect(); + let (kept, demoted) = adopted.into_iter().enumerate().fold( + (Vec::new(), losers), + |(mut kept, demoted), (index, repo)| { + if !demoted.contains_key(&index) { + kept.push(repo); + } + (kept, demoted) + }, + ); + Ok(( + kept, + skipped.into_iter().chain(demoted.into_values()).collect(), + )) +} + +fn classify_one( + row: &RepoRow, + rkeys: &BTreeMap, + collab_grants: &BTreeMap>, + exists: &impl Fn(&SourceRepoDid) -> bool, +) -> Result, MappingError> { + let did = RepoDid::new(row.repo_did.as_str()).map_err(|source| MappingError::BadRepoDid { + value: row.repo_did.clone(), + source, + })?; + let owner = + OwnerDid::new(row.owner_did.as_str()).map_err(|source| MappingError::BadOwnerDid { + repo: row.repo_did.clone(), + value: row.owner_did.clone(), + source, + })?; + let created_at = unix("repo_keys.created_at", &row.created_at)?; + let key_bytes: [u8; 32] = + row.signing_key + .as_bytes() + .try_into() + .map_err(|_| MappingError::BadSigningKey { + repo: row.repo_did.clone(), + key_type: row.key_type.clone(), + bytes: row.signing_key.as_bytes().len(), + })?; + if !row.key_type.is_k256() { + return Err(MappingError::BadSigningKey { + repo: row.repo_did.clone(), + key_type: row.key_type.clone(), + bytes: row.signing_key.as_bytes().len(), + }); + } + + let skip = |reason: SkipReason| SkippedRepo { + repo_did: did.clone(), + reason, + lost_collaborators: collab_grants + .get(&row.repo_did) + .map(|grants| grants.iter().map(|grant| grant.subject.clone()).collect()) + .unwrap_or_default(), + }; + let name = match RepoName::new(row.repo_name.as_str()) { + Ok(name) => name, + Err(_) => { + return Ok(Err(skip(SkipReason::Name { + value: row.repo_name.clone(), + }))); + } + }; + let raw_rkey = rkeys + .get(&row.repo_did) + .map(SourceRkey::as_str) + .unwrap_or(row.repo_name.as_str()); + let rkey = match RepoRkey::new(raw_rkey) { + Ok(rkey) => rkey, + Err(_) => { + return Ok(Err(skip(SkipReason::Rkey { + value: SourceRkey::from_column(raw_rkey), + }))); + } + }; + if !exists(&row.repo_did) { + return Ok(Err(skip(SkipReason::NoSourceRepo))); + } + + Ok(Ok(AdoptRepo { + source_did: row.repo_did.clone(), + did, + owner, + name, + rkey, + created_at, + signing_key: SigningKey(key_bytes), + collaborators: collab_grants + .get(&row.repo_did) + .cloned() + .unwrap_or_default(), + })) +} + +fn account(value: &SourceDid) -> Result { + AccountDid::new(value.as_str()) +} + +fn unix(context: &'static str, value: &SourceTimestamp) -> Result { + chrono::DateTime::parse_from_rfc3339(value.as_str()) + .map(|parsed| UnixSeconds::new(parsed.timestamp())) + .map_err(|_| MappingError::BadTimestamp { + context, + value: value.clone(), + }) +} diff --git a/knot2/crates/knot-migrate/src/report.rs b/knot2/crates/knot-migrate/src/report.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-migrate/src/report.rs @@ -0,0 +1,170 @@ +use std::fmt::{self, Display, Formatter}; + +use crate::adopt::AdoptOutcome; +use crate::emit::CobSummary; +use crate::mapping::{Mapping, SkipReason}; + +pub struct Report<'a> { + pub mapping: &'a Mapping, + pub orphan_alias_count: u64, + pub adoption: Option<&'a AdoptOutcome>, + pub cobs: Option<&'a CobSummary>, +} + +impl Display for Report<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let mapping = self.mapping; + let drift = &mapping.drift; + writeln!(f, "knot owner: {}", mapping.knot_owner)?; + writeln!(f, "members to grant: {}", mapping.members.len())?; + writeln!(f, "repos to adopt: {}", mapping.repos.len())?; + writeln!( + f, + "collaborator grants: {}", + mapping + .repos + .iter() + .map(|repo| repo.collaborators.len()) + .sum::() + )?; + writeln!(f)?; + writeln!(f, "casbin cross-check drift:")?; + writeln!( + f, + "acl-only collaborator grants unioned in: {}", + drift.acl_only_collaborators.len() + )?; + drift + .acl_only_collaborators + .iter() + .try_for_each(|(repo, did)| writeln!(f, "{repo} <- {did}"))?; + writeln!( + f, + "table-only collaborator grants missing from acl: {}", + drift.table_only_collaborators.len() + )?; + drift + .table_only_collaborators + .iter() + .try_for_each(|(repo, did)| writeln!(f, "{repo} <- {did}"))?; + writeln!( + f, + "repos with no acl owner marker where the owner regains push: {}", + drift.markerless_owner_repos.len() + )?; + drift + .markerless_owner_repos + .iter() + .try_for_each(|repo| writeln!(f, "{repo}"))?; + writeln!( + f, + "orphan owner markers on unknown repos: {}", + drift.orphan_owner_markers.len() + )?; + writeln!( + f, + "extra acl owner markers dropped: {}", + drift.extra_owner_markers.len() + )?; + drift + .extra_owner_markers + .iter() + .try_for_each(|(repo, did)| writeln!(f, "{repo} <- {did}"))?; + writeln!( + f, + "orphan collaborator pairs on unknown repos: {}", + drift.orphan_collaborator_pairs.len() + )?; + writeln!( + f, + "acl-only members unioned in: {}", + drift.acl_only_members.len() + )?; + drift + .acl_only_members + .iter() + .try_for_each(|did| writeln!(f, "{did}"))?; + writeln!( + f, + "table-only members missing from acl: {}", + drift.table_only_members.len() + )?; + writeln!(f, "slash-form owner markers: {}", drift.slash_owner_markers)?; + writeln!( + f, + "slash-form collaborator rows: {}", + drift.slash_collab_rows + )?; + writeln!( + f, + "slash-resolved collaborator grants left out of the union: {}", + drift.slash_resolved_collaborators.len() + )?; + drift + .slash_resolved_collaborators + .iter() + .try_for_each(|(repo, did)| writeln!(f, "{repo} <- {did}"))?; + writeln!( + f, + "unresolved slash forms: {}", + drift.unresolved_slash_forms.len() + )?; + drift + .unresolved_slash_forms + .iter() + .try_for_each(|form| writeln!(f, "{form}"))?; + writeln!(f, "orphan aliases: {}", self.orphan_alias_count)?; + writeln!(f)?; + writeln!(f, "skipped repos: {}", mapping.skipped.len())?; + mapping.skipped.iter().try_for_each(|skip| { + writeln!(f, "{} {}", skip.repo_did, describe(&skip.reason))?; + skip.lost_collaborators + .iter() + .try_for_each(|did| writeln!(f, "drops collaborator grant for {did}")) + })?; + self.adoption.map_or(Ok(()), |adoption| { + writeln!(f)?; + writeln!( + f, + "adopted by {}: {} new, {} already present, {} sha1, {} sha256", + adoption.transfer, + adoption.adopted, + adoption.already_present, + adoption.sha1, + adoption.sha256 + ) + })?; + self.cobs.map_or(Ok(()), |cobs| { + writeln!(f)?; + writeln!( + f, + "member grants: {} appended, {} already present", + cobs.members.appended, cobs.members.already_present + )?; + writeln!( + f, + "registrations: {} appended, {} already present", + cobs.registrations.appended, cobs.registrations.already_present + )?; + writeln!( + f, + "collaborator grants: {} appended, {} already present", + cobs.collaborators.appended, cobs.collaborators.already_present + ) + }) + } +} + +fn describe(reason: &SkipReason) -> String { + match reason { + SkipReason::Name { value } => format!("unrepresentable name {:?}", value.as_str()), + SkipReason::Rkey { value } => format!("unrepresentable rkey {:?}", value.as_str()), + SkipReason::RkeyCollision { rkey, winner } => { + format!( + "record key {:?} belongs to the alias-backed {winner}", + rkey.as_str() + ) + } + SkipReason::NoSourceRepo => "no git repository at the source path".to_string(), + } +} diff --git a/knot2/crates/knot-migrate/src/source.rs b/knot2/crates/knot-migrate/src/source.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-migrate/src/source.rs @@ -0,0 +1,317 @@ +use std::collections::BTreeSet; +use std::path::Path; + +use rusqlite::{Connection, OpenFlags, OptionalExtension, Row}; + +#[derive(Debug, thiserror::Error)] +pub enum SourceError { + #[error("source database query failed: {0}")] + Sqlite(#[from] rusqlite::Error), + #[error( + "source database predates DID-keyed repos. Upgrade tangled-knot to its latest release, let it finish its own migrations, then run knot-migrate again." + )] + PreDid, + #[error("source database has no acl table for the casbin cross-check")] + MissingAcl, + #[error( + "source database has a collaborators table but no knot_members table. Upgrade tangled-knot to its latest release, let it finish its own migrations, then run knot-migrate again." + )] + CollaboratorsWithoutMembers, + #[error( + "source table `{table}` is missing expected columns {}. This tangled-knot predates the schema knot-migrate reads. Upgrade tangled-knot to its latest release, let it finish its own migrations, then run knot-migrate again.", + .missing.join(", ") + )] + SchemaMismatch { table: String, missing: Vec }, +} + +knot_types::text_newtype! { + pub struct SourceRepoDid(String) => verbatim as from_column; + pub struct SourceDid(String) => verbatim as from_column; + pub struct SourceRkey(String) => verbatim as from_column; + pub struct SourceRepoName(String) => verbatim as from_column; + pub struct SourceRepoObject(String) => verbatim as from_column; + pub struct SourceTimestamp(String) => verbatim as from_column; +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SourceKeyType { + K256, + Other(String), +} + +impl SourceKeyType { + pub fn from_column(value: impl Into) -> Self { + let value = value.into(); + match value.as_str() { + "k256" => Self::K256, + _ => Self::Other(value), + } + } + + pub fn is_k256(&self) -> bool { + matches!(self, Self::K256) + } +} + +impl ::std::fmt::Display for SourceKeyType { + fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { + match self { + Self::K256 => f.write_str("k256"), + Self::Other(value) => f.write_str(value), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceSchema { + Tables, + PreFlip, +} + +#[derive(Clone, PartialEq, Eq, zeroize::Zeroize, zeroize::ZeroizeOnDrop)] +// Every repo's private key, straight outta the old knot's db. +// `Debug` doesn't prints any bytes on purpose, +// as to not compromise a migration. +pub struct SourceSigningKey(Vec); + +impl SourceSigningKey { + pub fn from_column(bytes: Vec) -> Self { + Self(bytes) + } + + pub fn as_bytes(&self) -> &[u8] { + &self.0 + } +} + +impl std::fmt::Debug for SourceSigningKey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_tuple("SourceSigningKey").finish_non_exhaustive() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RepoRow { + pub repo_did: SourceRepoDid, + pub owner_did: SourceDid, + pub repo_name: SourceRepoName, + pub signing_key: SourceSigningKey, + pub key_type: SourceKeyType, + pub created_at: SourceTimestamp, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MemberRow { + pub did: SourceDid, + pub subject: SourceDid, + pub created: SourceTimestamp, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CollabRow { + pub repo_did: SourceRepoDid, + pub subject_did: SourceDid, + pub added_by_did: SourceDid, + pub created: SourceTimestamp, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AclRow { + pub p_type: String, + pub v0: String, + pub v1: String, + pub v2: String, + pub v3: String, +} + +pub struct SourceDb { + conn: Connection, +} + +impl SourceDb { + pub fn open(path: &Path) -> Result { + let conn = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY)?; + Ok(Self { conn }) + } + + pub fn schema(&self) -> Result { + let variant = match ( + self.has_table("repo_keys")?, + self.has_table("repo_aliases")?, + self.has_table("knot_members")?, + self.has_table("collaborators")?, + ) { + (true, true, true, true) => SourceSchema::Tables, + (true, true, false, true) => return Err(SourceError::CollaboratorsWithoutMembers), + (true, true, _, false) => SourceSchema::PreFlip, + _ => return Err(SourceError::PreDid), + }; + let checks: Vec<(&str, &[&str])> = [ + Some(( + "repo_keys", + &[ + "repo_did", + "owner_did", + "repo_name", + "signing_key", + "key_type", + "created_at", + ][..], + )), + Some(("repo_aliases", &["rkey", "repo_did", "rev"][..])), + self.has_table("acl")? + .then_some(("acl", &["p_type", "v0", "v1", "v2", "v3"][..])), + self.has_table("knot_members")? + .then_some(("knot_members", &["id", "did", "subject", "created"][..])), + (variant == SourceSchema::Tables).then_some(( + "collaborators", + &["id", "repo_did", "subject_did", "added_by_did", "created"][..], + )), + ] + .into_iter() + .flatten() + .collect(); + checks + .into_iter() + .try_for_each(|(table, columns)| self.require_columns(table, columns))?; + Ok(variant) + } + + fn has_table(&self, name: &str) -> Result { + let count: i64 = self.conn.query_row( + "select count(*) from sqlite_master where type = 'table' and name = ?1", + [name], + |row| row.get(0), + )?; + Ok(count > 0) + } + + fn require_columns(&self, table: &str, required: &[&str]) -> Result<(), SourceError> { + let present: BTreeSet = self + .conn + .prepare("select name from pragma_table_info(?1)")? + .query_map([table], |row| row.get::<_, String>(0))? + .collect::>()?; + let missing: Vec = required + .iter() + .filter(|column| !present.contains(**column)) + .map(|column| (*column).to_string()) + .collect(); + missing + .is_empty() + .then_some(()) + .ok_or(SourceError::SchemaMismatch { + table: table.to_string(), + missing, + }) + } + + pub fn repos(&self) -> Result, SourceError> { + self.collect( + "select repo_did, owner_did, repo_name, signing_key, key_type, created_at + from repo_keys order by created_at, repo_did", + |row| { + Ok(RepoRow { + repo_did: SourceRepoDid::from_column(row.get::<_, String>(0)?), + owner_did: SourceDid::from_column(row.get::<_, String>(1)?), + repo_name: SourceRepoName::from_column(row.get::<_, String>(2)?), + signing_key: SourceSigningKey::from_column(row.get(3)?), + key_type: SourceKeyType::from_column(row.get::<_, String>(4)?), + created_at: SourceTimestamp::from_column(row.get::<_, String>(5)?), + }) + }, + ) + } + + pub fn members(&self) -> Result, SourceError> { + if !self.has_table("knot_members")? { + return Ok(Vec::new()); + } + self.collect( + "select did, subject, created from knot_members + where id in (select min(id) from knot_members group by subject) + order by id", + |row| { + Ok(MemberRow { + did: SourceDid::from_column(row.get::<_, String>(0)?), + subject: SourceDid::from_column(row.get::<_, String>(1)?), + created: SourceTimestamp::from_column(row.get::<_, String>(2)?), + }) + }, + ) + } + + pub fn collaborators(&self) -> Result, SourceError> { + if !self.has_table("collaborators")? { + return Ok(Vec::new()); + } + self.collect( + "select repo_did, subject_did, added_by_did, created from collaborators order by id", + |row| { + Ok(CollabRow { + repo_did: SourceRepoDid::from_column(row.get::<_, String>(0)?), + subject_did: SourceDid::from_column(row.get::<_, String>(1)?), + added_by_did: SourceDid::from_column(row.get::<_, String>(2)?), + created: SourceTimestamp::from_column(row.get::<_, String>(3)?), + }) + }, + ) + } + + pub fn acl(&self) -> Result, SourceError> { + if !self.has_table("acl")? { + return Err(SourceError::MissingAcl); + } + self.collect( + "select p_type, v0, v1, v2, v3 from acl order by rowid", + |row| { + Ok(AclRow { + p_type: row.get(0)?, + v0: row.get(1)?, + v1: row.get(2)?, + v2: row.get(3)?, + v3: row.get(4)?, + }) + }, + ) + } + + pub fn current_rkey( + &self, + repo_did: &SourceRepoDid, + ) -> Result, SourceError> { + self.conn + .query_row( + "select rkey from repo_aliases + where repo_did = ? + order by rev desc + limit 1", + [repo_did.as_str()], + |row| row.get::<_, String>(0).map(SourceRkey::from_column), + ) + .optional() + .map_err(Into::into) + } + + pub fn orphan_alias_count(&self) -> Result { + let count: i64 = self.conn.query_row( + "select count(*) from repo_aliases ra + where not exists (select 1 from repo_keys rk where rk.repo_did = ra.repo_did)", + [], + |row| row.get(0), + )?; + Ok(count as u64) + } + + fn collect( + &self, + sql: &str, + map: impl Fn(&Row<'_>) -> rusqlite::Result, + ) -> Result, SourceError> { + let mut statement = self.conn.prepare(sql)?; + let rows = statement + .query_map([], map)? + .collect::>>()?; + Ok(rows) + } +} diff --git a/knot2/crates/knot-migrate/tests/migrate.rs b/knot2/crates/knot-migrate/tests/migrate.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-migrate/tests/migrate.rs @@ -0,0 +1,715 @@ +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use knot_index::{Index, Resolved}; +use knot_migrate::adopt; +use knot_migrate::casbin; +use knot_migrate::emit::MasterKeyEnv; +use knot_migrate::emit::{self, ConfigValues}; +use knot_migrate::mapping::{self, SkipReason}; +use knot_migrate::source::{ + SourceDb, SourceDid, SourceError, SourceRepoDid, SourceRkey, SourceSchema, +}; +use knot_runtime::{K256Signer, SeededEntropy}; +use knot_types::{AccountDid, KnotHostname, KnotId, ObjectFormat, RepoDid}; +use url::Url; + +const SCHEMA: &str = " +create table repo_keys ( + repo_did text primary key, + signing_key blob, + created_at text not null, + owner_did text, + repo_name text, + key_type text not null default 'k256' +); +create table repo_aliases ( + owner_did text not null, + rkey text not null, + repo_did text not null, + rev text not null, + primary key (owner_did, rkey) +); +create table knot_members ( + id integer primary key autoincrement, + did text not null, + rkey text, + subject text not null, + created text not null +); +create table collaborators ( + id integer primary key autoincrement, + repo_did text not null, + subject_did text not null, + added_by_did text not null, + created text not null +); +create table acl ( + p_type varchar(32) default '' not null, + v0 varchar(255) default '' not null, + v1 varchar(255) default '' not null, + v2 varchar(255) default '' not null, + v3 varchar(255) default '' not null, + v4 varchar(255) default '' not null, + v5 varchar(255) default '' not null +); +"; + +fn fixture_db(path: &Path, with_collaborators_table: bool) { + let conn = rusqlite::Connection::open(path).unwrap(); + conn.execute_batch(SCHEMA).unwrap(); + conn.execute_batch( + " + insert into repo_keys (repo_did, signing_key, created_at, owner_did, repo_name) values + ('did:plc:squid', x'0101010101010101010101010101010101010101010101010101010101010101', '2026-01-05T10:00:00Z', 'did:plc:nel', 'anemone'), + ('did:plc:limpet', x'0202020202020202020202020202020202020202020202020202020202020202', '2026-02-01T09:30:00Z', 'did:plc:nel', 'barnacle'), + ('did:plc:conch', x'0303030303030303030303030303030303030303030303030303030303030303', '2026-03-10T14:00:00Z', 'did:plc:isabel', 'Test knot'), + ('did:plc:whelk', x'0404040404040404040404040404040404040404040404040404040404040404', '2026-04-20T08:15:00Z', 'did:plc:isabel', 'mussel'), + ('did:plc:nautilus', x'0505050505050505050505050505050505050505050505050505050505050505', '2026-05-01T10:00:00Z', 'did:plc:isabel', 'coralline'), + ('did:plc:scallop', x'0606060606060606060606060606060606060606060606060606060606060606', '2026-05-02T10:00:00Z', 'did:plc:isabel', 'seagrass'), + ('did:plc:clam', x'0707070707070707070707070707070707070707070707070707070707070707', '2026-05-03T10:00:00Z', 'did:plc:isabel', '|'), + ('did:web:nel.pet', x'0808080808080808080808080808080808080808080808080808080808080808', '2026-06-01T10:00:00Z', 'did:plc:nel', 'seashell'); + insert into repo_aliases (owner_did, rkey, repo_did, rev) values + ('did:plc:nel', 'anemone-old', 'did:plc:squid', '1_2026-01-05T10:00:00Z'), + ('did:plc:nel', 'anemone', 'did:plc:squid', '3mq2bmuwq7v2t'), + ('did:plc:isabel', 'Test knot', 'did:plc:conch', '3mniy6vtxn22y'), + ('did:plc:isabel', 'mussel', 'did:plc:whelk', '3moo4vihsva2t'), + ('did:plc:isabel', 'seagrass', 'did:plc:nautilus', '3mpwduty3pw2z'), + ('did:plc:isabel', '|', 'did:plc:clam', '3mq3bmuwq7v2t'), + ('did:plc:nel', 'vanished', 'did:plc:kelp', '3mq4bmuwq7v2t'); + insert into knot_members (did, subject, created) values + ('did:plc:bailey', 'did:plc:nel', '2026-01-02T00:00:00Z'), + ('did:plc:nel', 'did:plc:teq', '2026-01-03T00:00:00Z'), + ('did:plc:bailey', 'did:plc:teq', '2026-01-04T00:00:00Z'), + ('did:plc:nel', 'did:plc:olaren', '2026-01-05T00:00:00Z'), + ('did:plc:teq', 'did:plc:bailey', '2026-01-06T00:00:00Z'); + insert into acl (p_type, v0, v1, v2, v3) values + ('g', 'did:plc:bailey', 'server:owner', 'thisserver', ''), + ('g', 'server:owner', 'server:member', 'thisserver', ''), + ('g', 'did:plc:bailey', 'server:member', 'thisserver', ''), + ('g', 'did:plc:nel', 'server:member', 'thisserver', ''), + ('g', 'did:plc:teq', 'server:member', 'thisserver', ''), + ('g', 'did:plc:uni', 'server:member', 'thisserver', ''), + ('p', 'did:plc:nel', 'thisserver', 'did:plc:squid', 'repo:owner'), + ('p', 'did:plc:nel', 'thisserver', 'did:plc:squid', 'repo:push'), + ('p', 'did:plc:nel', 'thisserver', 'did:plc:squid', 'repo:settings'), + ('p', 'did:plc:nel', 'thisserver', 'did:plc:squid', 'repo:invite'), + ('p', 'did:plc:nel', 'thisserver', 'did:plc:squid', 'repo:delete'), + ('p', 'did:plc:nel', 'thisserver', 'did:plc:limpet', 'repo:owner'), + ('p', 'did:plc:bailey', 'thisserver', 'did:plc:limpet', 'repo:owner'), + ('p', 'did:plc:isabel', 'thisserver', 'did:plc:whelk', 'repo:owner'), + ('p', 'did:plc:isabel', 'thisserver', 'did:plc:nautilus', 'repo:owner'), + ('p', 'did:plc:isabel', 'thisserver', 'did:plc:scallop', 'repo:owner'), + ('p', 'did:plc:isabel', 'thisserver', 'did:plc:clam', 'repo:owner'), + ('p', 'did:plc:nel', 'thisserver', 'did:plc:kelp', 'repo:owner'), + ('p', 'did:plc:isabel', 'thisserver', 'did:plc:squid', 'repo:collaborator'), + ('p', 'did:plc:teq', 'thisserver', 'did:plc:limpet', 'repo:collaborator'), + ('p', 'did:plc:uni', 'thisserver', 'did:plc:whelk', 'repo:collaborator'), + ('p', 'did:plc:cuttle', 'thisserver', 'did:plc:kelp', 'repo:collaborator'), + ('p', 'did:plc:periwinkle', 'thisserver', 'did:plc:nel/anemone', 'repo:collaborator'), + ('p', 'did:plc:teq', 'thisserver', 'did:plc:nel/anemone', 'repo:collaborator'), + ('p', 'did:plc:nel', 'thisserver', 'did:plc:nel/vanished', 'repo:owner'), + ('p', 'did:plc:nel', 'thisserver', 'did:web:nel.pet', 'repo:owner'), + ('p', 'did:plc:nel', 'thisserver', '', 'repo:create'), + ('p', 'did:plc:nel', 'thisserver', '', 'server:invite'); + ", + ) + .unwrap(); + if with_collaborators_table { + conn.execute_batch( + " + insert into collaborators (repo_did, subject_did, added_by_did, created) values + ('did:plc:squid', 'did:plc:isabel', 'did:plc:nel', '2026-01-06T11:00:00Z'), + ('did:plc:squid', 'did:plc:olaren', 'did:plc:nel', '2026-01-07T12:00:00Z'), + ('did:plc:kelp', 'did:plc:teq', 'did:plc:nel', '2026-01-08T13:00:00Z'), + ('did:plc:squid', 'did:plc:isabel', 'did:plc:bailey', '2026-01-09T14:00:00Z'), + ('did:plc:whelk', 'did:plc:uni', 'did:plc:isabel', '2026-01-10T15:00:00Z'), + ('did:plc:squid', 'did:plc:teq', 'did:plc:nel', '2026-01-11T16:00:00Z'); + ", + ) + .unwrap(); + } else { + conn.execute_batch("drop table collaborators;").unwrap(); + } +} + +struct Fixture { + _dir: tempfile::TempDir, + db_path: PathBuf, + source_repos: PathBuf, + target: PathBuf, +} + +fn fixture(with_collaborators_table: bool) -> Fixture { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("knotserver.db"); + fixture_db(&db_path, with_collaborators_table); + let source_repos = dir.path().join("source-repos"); + [ + "did:plc:squid", + "did:plc:limpet", + "did:plc:conch", + "did:plc:nautilus", + "did:plc:scallop", + "did:plc:clam", + "did:web:nel.pet", + ] + .iter() + .for_each(|did| { + knot_git::Repo::create_with_format(source_repos.join(did), ObjectFormat::SHA1).unwrap(); + }); + std::fs::create_dir_all(source_repos.join("did:plc:whelk")).unwrap(); + Fixture { + db_path, + source_repos, + target: dir.path().join("target"), + _dir: dir, + } +} + +fn map(fx: &Fixture) -> mapping::Mapping { + let db = SourceDb::open(&fx.db_path).unwrap(); + let repos = db.repos().unwrap(); + let rkeys: BTreeMap = repos + .iter() + .filter_map(|repo| { + db.current_rkey(&repo.repo_did) + .unwrap() + .map(|rkey| (repo.repo_did.clone(), rkey)) + }) + .collect(); + let resolver = casbin::resolver(repos.iter().map(|repo| { + ( + repo.owner_did.clone(), + repo.repo_name.clone(), + repo.repo_did.clone(), + ) + })); + let acl = casbin::decode(&db.acl().unwrap(), &resolver).unwrap(); + let exists = |did: &SourceRepoDid| adopt::source_is_repo(&fx.source_repos, did); + match db.schema().unwrap() { + SourceSchema::Tables => mapping::map_tables( + &repos, + &rkeys, + &db.members().unwrap(), + &db.collaborators().unwrap(), + &acl, + exists, + ) + .unwrap(), + SourceSchema::PreFlip => { + mapping::map_preflip(&repos, &rkeys, &db.members().unwrap(), &acl, exists).unwrap() + } + } +} + +fn acc(suffix: &str) -> AccountDid { + AccountDid::new(format!("did:plc:{suffix}")).unwrap() +} + +fn srepo(value: &str) -> SourceRepoDid { + SourceRepoDid::from_column(value) +} + +fn sdid(value: &str) -> SourceDid { + SourceDid::from_column(value) +} + +fn subjects(grants: &[mapping::MappedGrant]) -> Vec<&str> { + grants.iter().map(|grant| grant.subject.as_str()).collect() +} + +#[test] +fn table_mapping_reproduces_roster_and_drift() { + let fx = fixture(true); + let db = SourceDb::open(&fx.db_path).unwrap(); + assert_eq!(db.schema().unwrap(), SourceSchema::Tables); + assert_eq!(db.orphan_alias_count().unwrap(), 1); + + let mapping = map(&fx); + assert_eq!(mapping.knot_owner, acc("bailey")); + + assert_eq!( + subjects(&mapping.members), + [ + "did:plc:nel", + "did:plc:teq", + "did:plc:olaren", + "did:plc:uni" + ] + ); + let teq = &mapping.members[1]; + assert_eq!(teq.added_by, acc("nel")); + let uni = &mapping.members[3]; + assert!(uni.unioned); + assert_eq!(uni.added_by, acc("bailey")); + + let dids: Vec<&str> = mapping.repos.iter().map(|repo| repo.did.as_str()).collect(); + assert_eq!( + dids, + [ + "did:plc:squid", + "did:plc:limpet", + "did:plc:nautilus", + "did:web:nel.pet" + ] + ); + let web = &mapping.repos[3]; + assert_eq!(web.rkey.as_str(), "seashell"); + assert_eq!(web.name.as_str(), "seashell"); + assert!(web.collaborators.is_empty()); + let nautilus = &mapping.repos[2]; + assert_eq!(nautilus.rkey.as_str(), "seagrass"); + assert_eq!(nautilus.name.as_str(), "coralline"); + let squid = &mapping.repos[0]; + assert_eq!(squid.rkey.as_str(), "anemone"); + assert_eq!( + subjects(&squid.collaborators), + ["did:plc:isabel", "did:plc:olaren", "did:plc:teq"] + ); + assert_eq!(squid.collaborators[0].added_by, acc("nel")); + let limpet = &mapping.repos[1]; + assert_eq!(limpet.rkey.as_str(), "barnacle"); + assert_eq!(subjects(&limpet.collaborators), ["did:plc:teq"]); + assert!(limpet.collaborators[0].unioned); + assert_eq!(limpet.collaborators[0].added_by, acc("nel")); + + let reasons: Vec<(&str, &SkipReason)> = mapping + .skipped + .iter() + .map(|skip| (skip.repo_did.as_str(), &skip.reason)) + .collect(); + assert_eq!(reasons.len(), 4); + assert!(matches!( + reasons[0], + ("did:plc:conch", SkipReason::Name { .. }) + )); + assert!( + matches!(reasons[1], ("did:plc:whelk", SkipReason::NoSourceRepo)), + "a source directory that exists but holds no git repository skips the repo" + ); + match reasons[2] { + ("did:plc:clam", SkipReason::Rkey { value }) => { + assert_eq!( + value.as_str(), + "|", + "an rkey RepoRkey rejects skips the repo even where RepoName accepts the same text" + ); + } + other => panic!("unexpected third skip {other:?}"), + } + match reasons[3] { + ("did:plc:scallop", SkipReason::RkeyCollision { rkey, winner }) => { + assert_eq!(rkey.as_str(), "seagrass"); + assert_eq!(winner.as_str(), "did:plc:nautilus"); + } + other => panic!("unexpected fourth skip {other:?}"), + } + let lost: Vec> = mapping + .skipped + .iter() + .map(|skip| { + skip.lost_collaborators + .iter() + .map(AccountDid::as_str) + .collect() + }) + .collect(); + assert_eq!(lost, [vec![], vec!["did:plc:uni"], vec![], vec![]]); + + let drift = &mapping.drift; + assert_eq!( + drift.acl_only_collaborators, + [(srepo("did:plc:limpet"), sdid("did:plc:teq"))] + ); + assert_eq!( + drift.table_only_collaborators, + [(srepo("did:plc:squid"), sdid("did:plc:olaren"))] + ); + assert_eq!( + drift.slash_resolved_collaborators, + [(srepo("did:plc:squid"), sdid("did:plc:periwinkle"))] + ); + assert_eq!( + drift.orphan_collaborator_pairs, + [ + (srepo("did:plc:kelp"), sdid("did:plc:cuttle")), + (srepo("did:plc:kelp"), sdid("did:plc:teq")) + ] + ); + assert_eq!(drift.markerless_owner_repos, [srepo("did:plc:conch")]); + assert_eq!(drift.orphan_owner_markers, [srepo("did:plc:kelp")]); + assert_eq!( + drift.extra_owner_markers, + [(srepo("did:plc:limpet"), sdid("did:plc:bailey"))] + ); + assert_eq!(drift.acl_only_members, [sdid("did:plc:uni")]); + assert_eq!(drift.table_only_members, [sdid("did:plc:olaren")]); + assert_eq!(drift.slash_owner_markers, 1); + assert_eq!(drift.slash_collab_rows, 2); + assert_eq!(drift.unresolved_slash_forms, ["did:plc:nel/vanished"]); +} + +#[test] +fn collaborators_without_knot_members_is_rejected() { + let fx = fixture(true); + rusqlite::Connection::open(&fx.db_path) + .unwrap() + .execute_batch("drop table knot_members;") + .unwrap(); + let db = SourceDb::open(&fx.db_path).unwrap(); + assert!(matches!( + db.schema(), + Err(SourceError::CollaboratorsWithoutMembers) + )); +} + +#[test] +fn a_source_table_missing_expected_columns_is_rejected_early() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("knotserver.db"); + rusqlite::Connection::open(&db_path) + .unwrap() + .execute_batch( + " + create table repo_keys ( + repo_did text primary key, + signing_key blob, + created_at text not null + ); + create table repo_aliases ( + owner_did text not null, + rkey text not null, + repo_did text not null, + rev text not null + ); + ", + ) + .unwrap(); + let db = SourceDb::open(&db_path).unwrap(); + match db.schema() { + Err(SourceError::SchemaMismatch { table, missing }) => { + assert_eq!(table, "repo_keys"); + assert_eq!(missing, ["owner_did", "repo_name", "key_type"]); + } + other => panic!("a repo_keys older than knot's schema must be rejected, got {other:?}"), + } +} + +#[test] +fn preflip_mapping_reads_casbin() { + let fx = fixture(false); + let db = SourceDb::open(&fx.db_path).unwrap(); + assert_eq!(db.schema().unwrap(), SourceSchema::PreFlip); + + let mapping = map(&fx); + assert_eq!(mapping.knot_owner, acc("bailey")); + assert_eq!( + subjects(&mapping.members), + ["did:plc:nel", "did:plc:teq", "did:plc:uni"] + ); + let nel = &mapping.members[0]; + assert_eq!(nel.added_by, acc("bailey")); + assert_eq!(nel.created_at, knot_types::UnixSeconds::new(1767312000)); + let uni = &mapping.members[2]; + assert_eq!(uni.added_by, acc("bailey")); + assert_eq!(uni.created_at, knot_types::UnixSeconds::new(0)); + assert_eq!(mapping.drift.table_only_members, [sdid("did:plc:olaren")]); + + let squid = &mapping.repos[0]; + assert_eq!( + subjects(&squid.collaborators), + ["did:plc:isabel", "did:plc:periwinkle", "did:plc:teq"] + ); + let limpet = &mapping.repos[1]; + assert_eq!(subjects(&limpet.collaborators), ["did:plc:teq"]); + assert_eq!( + mapping.drift.orphan_collaborator_pairs, + [(srepo("did:plc:kelp"), sdid("did:plc:cuttle"))] + ); + assert_eq!( + mapping.drift.extra_owner_markers, + [(srepo("did:plc:limpet"), sdid("did:plc:bailey"))] + ); + let whelk = mapping + .skipped + .iter() + .find(|skip| skip.repo_did.as_str() == "did:plc:whelk") + .unwrap(); + assert_eq!(whelk.lost_collaborators, [acc("uni")]); +} + +#[test] +fn adoption_and_cobs_boot_a_working_index() { + let fx = fixture(true); + let mapping = map(&fx); + let knot = KnotId::new("did:web:knot.oyster.cafe").unwrap(); + let scan_path = fx.target.join("repos"); + std::fs::create_dir_all(&scan_path).unwrap(); + let layout = knot_git::Layout::new(&scan_path) + .with_object_format(ObjectFormat::SHA1) + .reserving_meta(&knot) + .unwrap(); + let signer = K256Signer::generate(&SeededEntropy::new(7)); + std::os::unix::fs::symlink( + "config", + fx.source_repos.join("did:plc:squid").join("config-link"), + ) + .unwrap(); + + let adoption = adopt::adopt_all( + &layout, + &fx.source_repos, + &mapping.repos, + adopt::SourcePolicy::Preserve, + ) + .unwrap(); + assert_eq!(adoption.adopted, 4); + assert_eq!(adoption.transfer, adopt::Transfer::Copy); + assert_eq!(adoption.already_present, 0); + assert_eq!(adoption.sha1, 4); + let adopted_link = layout + .repo_path(&RepoDid::new("did:plc:squid").unwrap()) + .unwrap() + .join("config-link"); + assert!( + std::fs::symlink_metadata(&adopted_link) + .unwrap() + .file_type() + .is_symlink() + ); + assert_eq!( + std::fs::read_link(&adopted_link).unwrap(), + Path::new("config") + ); + + let cobs = emit::write_cobs(&layout, &knot, &mapping, &signer).unwrap(); + assert_eq!(cobs.members.appended, 4); + assert_eq!(cobs.registrations.appended, 4); + assert_eq!(cobs.collaborators.appended, 4); + + let index = Index::new(layout.meta_path(&knot).unwrap(), layout.clone()); + index.rebuild().unwrap(); + assert_eq!(index.hosted_repos().len(), 4); + let web = RepoDid::new("did:web:nel.pet").unwrap(); + assert_eq!( + index.owner_of(&web), + Resolved::Ready(Some(knot_types::OwnerDid::new("did:plc:nel").unwrap())) + ); + let squid = RepoDid::new("did:plc:squid").unwrap(); + index.ensure_collaborators(&squid).unwrap(); + assert_eq!( + index.is_collaborator(&squid, &acc("isabel")), + Resolved::Ready(true) + ); + assert_eq!( + index.is_collaborator(&squid, &acc("olaren")), + Resolved::Ready(true) + ); + assert_eq!( + index.is_collaborator(&squid, &acc("teq")), + Resolved::Ready(true) + ); + assert_eq!( + index.is_collaborator(&squid, &acc("periwinkle")), + Resolved::Ready(false) + ); + assert_eq!( + index.owner_of(&squid), + Resolved::Ready(Some(knot_types::OwnerDid::new("did:plc:nel").unwrap())) + ); + + let again = adopt::adopt_all( + &layout, + &fx.source_repos, + &mapping.repos, + adopt::SourcePolicy::Preserve, + ) + .unwrap(); + assert_eq!(again.adopted, 0); + assert_eq!(again.already_present, 4); + let recobs = emit::write_cobs(&layout, &knot, &mapping, &signer).unwrap(); + assert_eq!(recobs.members.appended, 0); + assert_eq!(recobs.members.already_present, 4); + assert_eq!(recobs.registrations.appended, 0); + assert_eq!(recobs.registrations.already_present, 4); + assert_eq!(recobs.collaborators.appended, 0); + assert_eq!(recobs.collaborators.already_present, 4); +} + +#[test] +fn consuming_the_source_moves_each_adopted_repo_out_of_the_scan_path() { + let fx = fixture(true); + let mapping = map(&fx); + let knot = KnotId::new("did:web:knot.oyster.cafe").unwrap(); + let scan_path = fx.target.join("repos"); + std::fs::create_dir_all(&scan_path).unwrap(); + let layout = knot_git::Layout::new(&scan_path) + .with_object_format(ObjectFormat::SHA1) + .reserving_meta(&knot) + .unwrap(); + + let adoption = adopt::adopt_all( + &layout, + &fx.source_repos, + &mapping.repos, + adopt::SourcePolicy::Consume, + ) + .unwrap(); + assert_eq!(adoption.transfer, adopt::Transfer::Rename); + assert_eq!(adoption.adopted, 4); + assert_eq!(adoption.already_present, 0); + assert!( + mapping + .repos + .iter() + .all(|repo| !adopt::source_dir(&fx.source_repos, &repo.source_did).exists()), + "every adopted repo leaves the source tree" + ); + assert!( + fx.source_repos.join("did:plc:conch").is_dir(), + "a skipped repo stays where it was" + ); + + let again = adopt::adopt_all( + &layout, + &fx.source_repos, + &mapping.repos, + adopt::SourcePolicy::Consume, + ) + .unwrap(); + assert_eq!(again.adopted, 0); + assert_eq!(again.already_present, 4); +} + +#[test] +fn adopting_a_repo_that_resolves_to_the_knot_meta_path_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("knotserver.db"); + let conn = rusqlite::Connection::open(&db_path).unwrap(); + conn.execute_batch(SCHEMA).unwrap(); + conn.execute_batch( + " + insert into repo_keys (repo_did, signing_key, created_at, owner_did, repo_name) values + ('did:web:knot.oyster.cafe', x'0909090909090909090909090909090909090909090909090909090909090909', '2026-06-01T10:00:00Z', 'did:plc:nel', 'seashell'); + insert into acl (p_type, v0, v1, v2, v3) values + ('g', 'did:plc:bailey', 'server:owner', 'thisserver', ''); + ", + ) + .unwrap(); + + let source_repos = dir.path().join("source-repos"); + knot_git::Repo::create_with_format( + source_repos.join("did:web:knot.oyster.cafe"), + ObjectFormat::SHA1, + ) + .unwrap(); + + let fx = Fixture { + db_path, + source_repos, + target: dir.path().join("target"), + _dir: dir, + }; + let mapping = map(&fx); + assert_eq!(mapping.repos.len(), 1); + + let knot = KnotId::new("did:web:knot.oyster.cafe").unwrap(); + let scan_path = fx.target.join("repos"); + std::fs::create_dir_all(&scan_path).unwrap(); + let layout = knot_git::Layout::new(&scan_path) + .with_object_format(ObjectFormat::SHA1) + .reserving_meta(&knot) + .unwrap(); + + let error = adopt::adopt_all( + &layout, + &fx.source_repos, + &mapping.repos, + adopt::SourcePolicy::Preserve, + ) + .unwrap_err(); + assert!(matches!( + error, + adopt::AdoptError::ReservesMeta { repo } if repo.as_str() == "did:web:knot.oyster.cafe" + )); + assert!(!layout.meta_path(&knot).unwrap().exists()); +} + +#[test] +fn rendered_config_loads() { + let dir = tempfile::tempdir().unwrap(); + let scan_path = dir.path().join("re\"pos\u{7f}"); + std::fs::create_dir_all(&scan_path).unwrap(); + let config = emit::render_config(&ConfigValues { + hostname: KnotHostname::new("knot.oyster.cafe").unwrap(), + admins: vec![acc("bailey")], + scan_path: scan_path.clone(), + ssh_host_key_file: dir.path().join("ssh_host_key"), + sealed_key_file: dir.path().join("sealed-keys"), + master_key_env: MasterKeyEnv::new("KNOT_MASTER_KEY").unwrap(), + object_format: ObjectFormat::SHA1, + plc_directory: Url::parse("https://plc.directory").unwrap(), + }) + .unwrap(); + assert!(config.contains("hostname = \"knot.oyster.cafe\"")); + assert!(config.contains("admins = [\"did:plc:bailey\"]")); + assert!(config.contains("admission = \"closed\"")); + assert!(config.contains("object_format = \"sha1\"")); + let path = dir.path().join("config.toml"); + std::fs::write(&path, &config).unwrap(); + knot_config::load(Some(&path)).unwrap(); +} + +const HOST_KEY: &str = "-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACAuLv0N4MHTuclN/afhoL60chkky1gCLCFCA2T1qOKGhwAAAJgv0qFlL9Kh +ZQAAAAtzc2gtZWQyNTUxOQAAACAuLv0N4MHTuclN/afhoL60chkky1gCLCFCA2T1qOKGhw +AAAEAnnapXprdwlEwD6xIxSqm3szQrvfQdhRp6UfONp85Uky4u/Q3gwdO5yU39p+GgvrRy +GSTLWAIsIUIDZPWo4oaHAAAAEWtub3QtbWlncmF0ZS10ZXN0AQIDBA== +-----END OPENSSH PRIVATE KEY----- +"; + +const ECDSA_HOST_KEY: &str = "-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAaAAAABNlY2RzYS +1zaGEyLW5pc3RwMjU2AAAACG5pc3RwMjU2AAAAQQS6SLu5jEz+0ScKcByJBs53LlSkz8dT +ELlhV5QrNPQvk+h5UduwxR7ShN3IL9AhjiVugVN3I9vHHB1BwcNXE6exAAAAsD5jy3k+Y8 +t5AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLpIu7mMTP7RJwpw +HIkGzncuVKTPx1MQuWFXlCs09C+T6HlR27DFHtKE3cgv0CGOJW6BU3cj28ccHUHBw1cTp7 +EAAAAhAI2ARTG/6mM9qJfmdg8rASQudcrZ5KFLkH6FjB0V6EUgAAAAEWtub3QtbWlncmF0 +ZS10ZXN0AQIDBAUG +-----END OPENSSH PRIVATE KEY----- +"; + +#[test] +fn host_key_import_preserves_every_algorithm() { + let dir = tempfile::tempdir().unwrap(); + [ + ( + "ssh_host_ed25519_key", + HOST_KEY, + ssh_key::Algorithm::Ed25519, + ), + ( + "ssh_host_ecdsa_key", + ECDSA_HOST_KEY, + ssh_key::Algorithm::Ecdsa { + curve: ssh_key::EcdsaCurve::NistP256, + }, + ), + ] + .into_iter() + .for_each(|(name, pem, algorithm)| { + let source = dir.path().join(name); + std::fs::write(&source, pem).unwrap(); + let destination = dir.path().join(format!("{name}.imported")); + let host_key = emit::load_host_key(&source).unwrap(); + assert_eq!(host_key.algorithm, algorithm); + host_key.write_to(&destination).unwrap(); + assert_eq!( + std::fs::read(&source).unwrap(), + std::fs::read(&destination).unwrap(), + "{name} must round-trip byte-for-byte so the pinned fingerprint survives" + ); + }); +} diff --git a/knot2/crates/knot-pack/examples/ingest_mem.rs b/knot2/crates/knot-pack/examples/ingest_mem.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/examples/ingest_mem.rs @@ -0,0 +1,86 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::time::Instant; + +#[global_allocator] +static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +const PAGE: u64 = 4096; + +fn rss_bytes() -> u64 { + let statm = std::fs::read_to_string("/proc/self/statm").unwrap(); + statm + .split_whitespace() + .nth(1) + .and_then(|pages| pages.parse::().ok()) + .map(|pages| pages * PAGE) + .unwrap() +} + +fn vm_hwm_bytes() -> u64 { + let status = std::fs::read_to_string("/proc/self/status").unwrap(); + status + .lines() + .find_map(|line| line.strip_prefix("VmHWM:")) + .and_then(|rest| rest.split_whitespace().next()) + .and_then(|kb| kb.parse::().ok()) + .map(|kb| kb * 1024) + .unwrap() +} + +fn mib(bytes: u64) -> u64 { + bytes / (1024 * 1024) +} + +fn main() { + let pack_path = std::path::PathBuf::from( + std::env::args() + .nth(1) + .expect("usage: ingest_mem "), + ); + + let pack_size = std::fs::metadata(&pack_path).map(|m| m.len()).unwrap_or(0); + println!( + "pack: {} MiB on disk, streamed not resident", + mib(pack_size) + ); + println!( + "governor: {} threads, {}", + knot_resource::threads().get(), + knot_resource::available_bytes() + .map(|bytes| format!("{} MiB available in cgroup", mib(bytes.get()))) + .unwrap_or_else(|| "unconstrained".to_string()) + ); + + let dir = tempfile::tempdir().expect("tempdir"); + let objects_dir = dir.path().to_path_buf(); + + let peak = Arc::new(AtomicU64::new(0)); + let stop = Arc::new(AtomicBool::new(false)); + let sampler = { + let peak = Arc::clone(&peak); + let stop = Arc::clone(&stop); + std::thread::spawn(move || { + while !stop.load(Ordering::Relaxed) { + let now = rss_bytes(); + peak.fetch_max(now, Ordering::Relaxed); + std::thread::sleep(std::time::Duration::from_millis(5)); + } + }) + }; + + let before = rss_bytes(); + let start = Instant::now(); + let folded = knot_pack::bench_ingest_fresh(&objects_dir, &pack_path, gix::hash::Kind::Sha1) + .expect("ingest"); + let elapsed = start.elapsed(); + stop.store(true, Ordering::Relaxed); + sampler.join().ok(); + + let sampled_peak = peak.load(Ordering::Relaxed); + println!("fold engaged: {folded}"); + println!("took: {:.1}s", elapsed.as_secs_f64()); + println!("rss before ingest: {} MiB", mib(before)); + println!("rss sampled peak: {} MiB", mib(sampled_peak)); + println!("VmHWM, kernel peak: {} MiB", mib(vm_hwm_bytes())); +} diff --git a/knot2/crates/knot-pack/examples/knot_http_backend.rs b/knot2/crates/knot-pack/examples/knot_http_backend.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/examples/knot_http_backend.rs @@ -0,0 +1,205 @@ +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; + +use knot_git::Repo; + +#[global_allocator] +static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +const SUFFIXES: [&str; 3] = ["/info/refs", "/git-upload-pack", "/git-receive-pack"]; + +fn env(key: &str) -> Option { + std::env::var(key).ok() +} + +fn resolve() -> Option<(PathBuf, &'static str)> { + let path_info = env("PATH_INFO")?; + let (repo_sub, suffix) = SUFFIXES + .iter() + .find_map(|suffix| path_info.strip_suffix(suffix).map(|rest| (rest, *suffix)))?; + let root = env("GIT_PROJECT_ROOT") + .filter(|root| !root.is_empty()) + .or_else(|| { + let translated = env("PATH_TRANSLATED")?; + translated + .strip_suffix(path_info.as_str()) + .map(str::to_string) + })?; + let repo_sub = repo_sub.trim_start_matches('/'); + Some((Path::new(&root).join(repo_sub), suffix)) +} + +fn wants_v2() -> bool { + env("HTTP_GIT_PROTOCOL") + .or_else(|| env("GIT_PROTOCOL")) + .is_some_and(|value| value.split(':').any(|token| token.trim() == "version=2")) +} + +fn dev_push_allowed() -> bool { + env("KNOT_DEV_ALLOW_HTTP_PUSH").is_some_and(|value| { + let value = value.trim(); + value == "1" || value.eq_ignore_ascii_case("true") + }) +} + +fn dev_pack_limits() -> knot_pack::PackLimits { + let mut limits = knot_pack::PackLimits::default(); + if let Some(value) = env("KNOT_DEV_MAX_OBJECTS").and_then(|value| value.trim().parse().ok()) { + limits.max_objects = knot_types::ObjectCount::new(value); + } + if let Some(value) = env("KNOT_DEV_MAX_TOTAL_BYTES").and_then(|value| value.trim().parse().ok()) + { + limits.max_total_bytes = knot_pack::MaxTotalBytes::new(value); + } + limits +} + +fn apply_dev_selection_limits() { + let max_objects = env("KNOT_DEV_SELECTION_MAX_OBJECTS") + .and_then(|value| value.trim().parse().ok()) + .map(knot_types::ObjectCount::new); + let secs = + env("KNOT_DEV_SELECTION_TIME_BUDGET_SECS").and_then(|value| value.trim().parse().ok()); + if max_objects.is_none() && secs.is_none() { + return; + } + let base = knot_pack::SelectionLimits::default(); + knot_pack::init_selection_limits(knot_pack::SelectionLimits { + max_objects: max_objects.unwrap_or(base.max_objects), + time_budget: secs + .map(std::time::Duration::from_secs) + .unwrap_or(base.time_budget), + }); +} + +fn apply_dev_resources() { + let max_threads = env("KNOT_DEV_MAX_THREADS") + .and_then(|value| value.trim().parse::().ok()) + .filter(|threads| *threads != 0) + .map(knot_resource::ThreadCount::new); + knot_resource::init(knot_resource::Ceilings { + max_threads, + ..knot_resource::Ceilings::default() + }); +} + +fn read_body() -> Vec { + let mut raw = Vec::new(); + std::io::stdin().read_to_end(&mut raw).ok(); + let gzipped = env("HTTP_CONTENT_ENCODING").is_some_and(|value| { + value + .split(',') + .any(|token| token.trim().eq_ignore_ascii_case("gzip")) + }); + if !gzipped { + return raw; + } + let mut decoded = Vec::new(); + flate2::read::GzDecoder::new(raw.as_slice()) + .read_to_end(&mut decoded) + .ok(); + decoded +} + +fn emit(out: &mut dyn Write, content_type: &str, body: &[u8]) { + let _ = write!( + out, + "Expires: Fri, 01 Jan 1980 00:00:00 GMT\r\nPragma: no-cache\r\nCache-Control: no-cache, max-age=0, must-revalidate\r\nContent-Type: {content_type}\r\n\r\n" + ); + let _ = out.write_all(body); +} + +fn fail(out: &mut dyn Write, status: &str, message: &str) { + let _ = write!( + out, + "Status: {status}\r\nContent-Type: text/plain\r\n\r\n{message}\n" + ); +} + +const PUSH_REFUSED: &str = "knot accepts pushes over SSH, not HTTP"; + +fn serve_receive_advert(out: &mut dyn Write, repo: &Repo) { + if !dev_push_allowed() { + return fail(out, "403 Forbidden", PUSH_REFUSED); + } + match knot_pack::advertise_receive(repo) { + Ok(body) => emit(out, "application/x-git-receive-pack-advertisement", &body), + Err(error) => fail(out, "500 Internal Server Error", &error.to_string()), + } +} + +fn serve_receive_pack(out: &mut dyn Write, repo: &Repo) { + if !dev_push_allowed() { + return fail(out, "403 Forbidden", PUSH_REFUSED); + } + match knot_pack::receive_pack_with_limits(repo, &read_body(), &dev_pack_limits()) { + Ok(result) => emit(out, "application/x-git-receive-pack-result", &result), + Err(error) => fail(out, "500 Internal Server Error", &error.to_string()), + } +} + +fn main() { + apply_dev_selection_limits(); + apply_dev_resources(); + let stdout = std::io::stdout(); + let mut out = stdout.lock(); + + let (repo_dir, suffix) = match resolve() { + Some(parts) => parts, + None => { + return fail( + &mut out, + "400 Bad Request", + "unrecognized git smart-http path", + ); + } + }; + let repo = match Repo::open(&repo_dir) { + Ok(repo) => repo, + Err(_) => return fail(&mut out, "404 Not Found", "no such repository"), + }; + let method = env("REQUEST_METHOD").unwrap_or_default(); + + match (method.as_str(), suffix) { + ("GET", "/info/refs") => { + let service = env("QUERY_STRING") + .and_then(|query| { + query + .split('&') + .find_map(|pair| pair.strip_prefix("service=").map(str::to_string)) + }) + .unwrap_or_default(); + match service.as_str() { + "git-upload-pack" => { + let body = if wants_v2() { + knot_pack::advertise_upload(&repo) + } else { + knot_pack::advertise_upload_v0(&repo) + }; + match body { + Ok(body) => emit( + &mut out, + "application/x-git-upload-pack-advertisement", + &body, + ), + Err(error) => { + fail(&mut out, "500 Internal Server Error", &error.to_string()) + } + } + } + "git-receive-pack" => serve_receive_advert(&mut out, &repo), + _ => fail(&mut out, "403 Forbidden", "unsupported service"), + } + } + ("POST", "/git-upload-pack") => match knot_pack::upload_pack(&repo, &read_body()) { + Ok(result) => emit(&mut out, "application/x-git-upload-pack-result", &result), + Err(error) => fail(&mut out, "500 Internal Server Error", &error.to_string()), + }, + ("POST", "/git-receive-pack") => serve_receive_pack(&mut out, &repo), + _ => fail( + &mut out, + "400 Bad Request", + "unsupported git smart-http request", + ), + } +} diff --git a/knot2/crates/knot-pack/examples/receive_mem.rs b/knot2/crates/knot-pack/examples/receive_mem.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/examples/receive_mem.rs @@ -0,0 +1,129 @@ +use std::io::Read; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::time::Instant; + +#[global_allocator] +static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +const PAGE: u64 = 4096; + +fn rss_bytes() -> u64 { + let statm = std::fs::read_to_string("/proc/self/statm").unwrap(); + statm + .split_whitespace() + .nth(1) + .and_then(|pages| pages.parse::().ok()) + .map(|pages| pages * PAGE) + .unwrap() +} + +fn vm_hwm_bytes() -> u64 { + let status = std::fs::read_to_string("/proc/self/status").unwrap(); + status + .lines() + .find_map(|line| line.strip_prefix("VmHWM:")) + .and_then(|rest| rest.split_whitespace().next()) + .and_then(|kb| kb.parse::().ok()) + .map(|kb| kb * 1024) + .unwrap() +} + +fn mib(bytes: u64) -> u64 { + bytes / (1024 * 1024) +} + +fn pktline(data: &[u8]) -> Vec { + let mut out = format!("{:04x}", 4 + data.len()).into_bytes(); + out.extend_from_slice(data); + out +} + +fn main() { + let max_threads = std::env::var("KNOT_MAX_THREADS") + .ok() + .and_then(|value| value.parse::().ok()); + knot_resource::init(knot_resource::Ceilings { + max_threads: max_threads.map(knot_resource::ThreadCount::new), + max_memory: None, + }); + + let pack_path = std::path::PathBuf::from( + std::env::args() + .nth(1) + .expect("usage: receive_mem "), + ); + let pack_size = std::fs::metadata(&pack_path).map(|m| m.len()).unwrap_or(0); + println!("pack: {} MiB on disk", mib(pack_size)); + + let mut preamble = pktline( + b"0000000000000000000000000000000000000000 \ + 1111111111111111111111111111111111111111 refs/heads/main\0report-status\n", + ); + preamble.extend_from_slice(b"0000"); + + let dir = tempfile::tempdir().expect("tempdir"); + let limit = knot_pack::MaxWireBytes::new(16 * 1024 * 1024 * 1024); + let mut receiver = knot_pack::PackReceiver::new( + dir.path(), + limit, + knot_pack::PackLimits::default(), + gix::hash::Kind::Sha1, + ) + .expect("receiver"); + + let peak = Arc::new(AtomicU64::new(0)); + let stop = Arc::new(AtomicBool::new(false)); + let sampler = { + let peak = Arc::clone(&peak); + let stop = Arc::clone(&stop); + std::thread::spawn(move || { + while !stop.load(Ordering::Relaxed) { + peak.fetch_max(rss_bytes(), Ordering::Relaxed); + std::thread::sleep(std::time::Duration::from_millis(5)); + } + }) + }; + + let start = Instant::now(); + receiver.write(&preamble).expect("preamble"); + let mut file = std::fs::File::open(&pack_path).expect("open pack"); + let mut buf = vec![0u8; 1 << 20]; + loop { + let read = file.read(&mut buf).expect("read pack"); + if read == 0 { + break; + } + receiver.write(&buf[..read]).expect("receive"); + } + let received = receiver.finish().expect("finish"); + let receipt_elapsed = start.elapsed(); + let receipt_peak = peak.load(Ordering::Relaxed); + println!( + "receipt: {:.1}s, peak {} MiB (pack streamed to a temp file, never resident)", + receipt_elapsed.as_secs_f64(), + mib(receipt_peak) + ); + + let staged = received.open_pack().expect("open pack").expect("has pack"); + let objects = tempfile::tempdir().expect("objects tempdir"); + let result = + knot_pack::bench_admit_and_ingest(objects.path(), staged.path(), gix::hash::Kind::Sha1); + let total_elapsed = start.elapsed(); + let total_peak = peak.load(Ordering::Relaxed); + stop.store(true, Ordering::Relaxed); + sampler.join().ok(); + + match result { + Ok(folded) => { + println!("ADMITTED, fold engaged: {folded}"); + println!("receive + ingest: {:.1}s", total_elapsed.as_secs_f64()); + println!("receive + ingest peak: {} MiB", mib(total_peak)); + println!("VmHWM,kernel peak: {} MiB", mib(vm_hwm_bytes())); + } + Err(error) => { + println!("REFUSED cleanly before the fold: {error:?}"); + println!("peak at refusal: {} MiB", mib(total_peak)); + } + } +} diff --git a/knot2/crates/knot-pack/fuzz/.gitignore b/knot2/crates/knot-pack/fuzz/.gitignore new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/fuzz/.gitignore @@ -0,0 +1,4 @@ +target +corpus +artifacts +coverage diff --git a/knot2/crates/knot-pack/fuzz/Cargo.lock b/knot2/crates/knot-pack/fuzz/Cargo.lock new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/fuzz/Cargo.lock @@ -0,0 +1,6275 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + +[[package]] +name = "arc-swap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +dependencies = [ + "rustversion", +] + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "asn1-rs" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5493c3bedbacf7fd7382c6346bbd66687d12bbaad3a89a2d2c303ee6cf20b048" +dependencies = [ + "asn1-rs-derive 0.5.1", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive 0.6.0", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-compression" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-http-codec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "096146020b08dbc4587685b0730a7ba905625af13c65f8028035cdfd69573c91" +dependencies = [ + "anyhow", + "futures", + "http", + "httparse", + "log", +] + +[[package]] +name = "async-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" +dependencies = [ + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-net" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b948000fad4873c1c9339d60f2623323a0cfd3816e5181033c6a5cb68b2accf7" +dependencies = [ + "async-io", + "blocking", + "futures-lite", +] + +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-web-client" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8caf502b44d6d4be6154ac33af012cbb5fef11e6066edcfb42834217fbaf501b" +dependencies = [ + "async-http-codec", + "async-net", + "futures", + "futures-rustls", + "http", + "lazy_static", + "log", + "rustls-pki-types", + "serde", + "thiserror 1.0.69", + "webpki-roots 0.26.11", +] + +[[package]] +name = "atomic-polyfill" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" +dependencies = [ + "critical-section", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "aws-lc-rs" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "base-x" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" + +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + +[[package]] +name = "base256emoji" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e9430d9a245a77c92176e649af6e275f20839a48389859d1661e9a128d077c" +dependencies = [ + "const-str", + "match-lookup", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "blocking" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21" +dependencies = [ + "async-channel", + "async-task", + "futures-io", + "futures-lite", + "piper", +] + +[[package]] +name = "bon" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f47dbe92550676ee653353c310dfb9cf6ba17ee70396e1f7cf0a2020ad49b2fe" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + +[[package]] +name = "borrow-or-share" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" + +[[package]] +name = "borsh" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfd1e3f8955a5d7de9fab72fc8373fade9fb8a703968cb200ae3dc6cf08e185a" +dependencies = [ + "bytes", + "cfg_aliases", +] + +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "bstr" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +dependencies = [ + "serde", +] + +[[package]] +name = "bytesize" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd91ee7b2422bcb158d90ef4d14f75ef67f340943fc4149891dcce8f8b972a3" + +[[package]] +name = "cbor4ii" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544cf8c89359205f4f990d0e6f3828db42df85b5dac95d09157a250eb0749c4" +dependencies = [ + "serde", +] + +[[package]] +name = "cc" +version = "1.2.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "cid" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a304f95f84d169a6f31c4d0a30d784643aaa0bbc9c1e449a2c23e963ec4971" +dependencies = [ + "multibase", + "multihash", + "serde", + "serde_bytes", + "unsigned-varint 0.8.0", +] + +[[package]] +name = "clru" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "197fd99cb113a8d5d9b6376f3aa817f32c1078f2343b714fff7d2ca44fdf67d5" +dependencies = [ + "hashbrown 0.16.1", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.18", +] + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "brotli", + "compression-core", + "flate2", + "memchr", + "zstd", + "zstd-safe", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "const-str" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f421161cb492475f1661ddc9815a745a1c894592070661180fdec3d4872e9c3" + +[[package]] +name = "cordyceps" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "688d7fbb8092b8de775ef2536f36c8c31f2bc4006ece2e8d8ad2d17d00ce0a2a" +dependencies = [ + "loom", + "tracing", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "data-encoding-macro" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3259c913752a86488b501ed8680446a5ed2d5aeac6e596cb23ba3800768ea32c" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccc2776f0c61eca1ca32528f85548abd1a4be8fb53d1b21c013e4f18da1e7090" +dependencies = [ + "data-encoding", + "syn", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "der-parser" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" +dependencies = [ + "asn1-rs 0.6.2", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs 0.7.2", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "derive_more" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "diatomic-waker" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab03c107fafeb3ee9f5925686dbb7a73bc76e3932abb0d2b365cb64b169cf04c" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.6", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest 0.10.7", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "rand_core 0.6.4", + "serde", + "sha2 0.10.9", + "subtle", + "zeroize", +] + +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "pem-rfc7468", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "faster-hex" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7223ae2d2f179b803433d9c830478527e92b8117eab39460edae7f1614d9fb73" +dependencies = [ + "heapless 0.8.0", + "serde", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "fluent-uri" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "forwarded-header-value" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8835f84f38484cc86f110a805655697908257fb9a7af005234060891557198e9" +dependencies = [ + "nonempty 0.7.0", + "thiserror 1.0.69", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-buffered" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4421cb78ee172b6b06080093479d3c50f058e7c81b7d577bbb8d118d551d4cd5" +dependencies = [ + "cordyceps", + "diatomic-waker", + "futures-core", + "pin-project-lite", + "spin 0.10.0", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f2f12607f92c69b12ed746fabf9ca4f5c482cba46679c1a75b874ed7c26adb" +dependencies = [ + "futures-io", + "rustls", + "rustls-pki-types", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-timer" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generator" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + +[[package]] +name = "generic-array" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +dependencies = [ + "typenum", + "version_check", + "zeroize", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + +[[package]] +name = "gix" +version = "0.84.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae54ae0ebd1a5a3c3f8d95dd3b5ca6e63f4fed9bfd585e13801a97d7bde8f9ce" +dependencies = [ + "gix-actor", + "gix-archive", + "gix-attributes", + "gix-blame", + "gix-command", + "gix-commitgraph", + "gix-config", + "gix-credentials", + "gix-date", + "gix-diff", + "gix-dir", + "gix-discover", + "gix-error", + "gix-features", + "gix-filter", + "gix-fs", + "gix-glob", + "gix-hash", + "gix-hashtable", + "gix-ignore", + "gix-index", + "gix-lock", + "gix-mailmap", + "gix-negotiate", + "gix-object", + "gix-odb", + "gix-pack", + "gix-path", + "gix-pathspec", + "gix-prompt", + "gix-protocol", + "gix-ref", + "gix-refspec", + "gix-revision", + "gix-revwalk", + "gix-sec", + "gix-shallow", + "gix-status", + "gix-submodule", + "gix-tempfile", + "gix-trace", + "gix-traverse", + "gix-url", + "gix-utils", + "gix-validate", + "gix-worktree", + "gix-worktree-state", + "gix-worktree-stream", + "nonempty 0.12.0", + "parking_lot", + "regex", + "signal-hook", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-actor" +version = "0.41.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bc998b8f746dda8565450d08a63b792ced9165d8c27a1ed3f02799ec6a7820f" +dependencies = [ + "bstr", + "gix-date", + "gix-error", +] + +[[package]] +name = "gix-archive" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16909cacc78936ab96f6c3be08379d0a2e88bfa3a7527972d2ed75c7517ef31e" +dependencies = [ + "bstr", + "flate2", + "gix-date", + "gix-error", + "gix-object", + "gix-path", + "gix-worktree-stream", + "rawzip", + "tar", +] + +[[package]] +name = "gix-attributes" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d43f12e246d3bf7ec624c8fc15ac4a4b62b7c4c6f586cb82be6c90bf84c9d02" +dependencies = [ + "bstr", + "gix-glob", + "gix-path", + "gix-quote", + "gix-trace", + "kstring", + "smallvec", + "thiserror 2.0.18", + "unicode-bom", +] + +[[package]] +name = "gix-bitmap" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ebef0c26ad305747649e727bbcd56a7b7910754eb7cea88f6dff6f93c51283" +dependencies = [ + "gix-error", +] + +[[package]] +name = "gix-blame" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d39a0c14af94c2edaa5eefe06d5ef2cdea55316ae9a9321314288e3f55fa4c0" +dependencies = [ + "gix-commitgraph", + "gix-date", + "gix-diff", + "gix-error", + "gix-hash", + "gix-object", + "gix-revwalk", + "gix-trace", + "gix-traverse", + "gix-worktree", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-chunk" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9faee47943b638e58ddd5e275a4906ad3e4b6c8584f1d41bd18ab9032ec52afb" +dependencies = [ + "gix-error", +] + +[[package]] +name = "gix-command" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00706d4fef135ef4b01680d5218c6ee40cda8baf697b864296cbc887d19118f6" +dependencies = [ + "bstr", + "gix-path", + "gix-quote", + "gix-trace", + "shell-words", +] + +[[package]] +name = "gix-commitgraph" +version = "0.37.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f675d0df484a7f6a47e64bd6f311af489d947c0323b0564f36d14f3d7762abb" +dependencies = [ + "bstr", + "gix-chunk", + "gix-error", + "gix-hash", + "memmap2", + "nonempty 0.12.0", +] + +[[package]] +name = "gix-config" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2372d4b49ca28431e7d150cab9d25edc1890f0184bd57eb0e917c7799e63de" +dependencies = [ + "bstr", + "gix-config-value", + "gix-features", + "gix-glob", + "gix-path", + "gix-ref", + "gix-sec", + "smallvec", + "thiserror 2.0.18", + "unicode-bom", +] + +[[package]] +name = "gix-config-value" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed42168329552f6c2e5df09665c104199d45d84bedb53683738a49b57fe1baab" +dependencies = [ + "bitflags", + "bstr", + "gix-path", + "libc", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-credentials" +version = "0.38.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40cd22f0dd71988be12d6e78b1709de2370e1957c5f107ff31e56caeba3745d" +dependencies = [ + "bstr", + "gix-command", + "gix-config-value", + "gix-date", + "gix-path", + "gix-prompt", + "gix-sec", + "gix-trace", + "gix-url", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-date" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3ecab64a98bbac9f8e02990a9ea5e3c974a7d49b95f2bd70ad94ad22fa6b48c" +dependencies = [ + "bstr", + "gix-error", + "itoa", + "jiff", +] + +[[package]] +name = "gix-diff" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b6d9528f32d94cef2edf39a1ac01fe5a0fc44ddbb18d9e44099936047c3302b" +dependencies = [ + "bstr", + "gix-attributes", + "gix-command", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-imara-diff", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-tempfile", + "gix-trace", + "gix-traverse", + "gix-worktree", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-dir" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21bb2a53a6fd917ec499ed0bfb5b6887de7a15bd79197dcea7c987938749a9f1" +dependencies = [ + "bstr", + "gix-discover", + "gix-fs", + "gix-ignore", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-trace", + "gix-utils", + "gix-worktree", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-discover" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77bacdd12b7879d2178a80c58c2f319995e4654e1a7a23e3181e5c8a12b824f7" +dependencies = [ + "bstr", + "dunce", + "gix-fs", + "gix-path", + "gix-ref", + "gix-sec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-error" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57831e199be480af90dcd7e459abed8a174c09ec9a6e2cc8f7ca6c54598b06b" +dependencies = [ + "bstr", +] + +[[package]] +name = "gix-features" +version = "0.48.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1849ae154d38bc403185be14fa871e38e3c93ee606875d94e207fdb9fba52dbc" +dependencies = [ + "bytes", + "bytesize", + "crc32fast", + "crossbeam-channel", + "gix-path", + "gix-trace", + "gix-utils", + "libc", + "once_cell", + "parking_lot", + "prodash", + "thiserror 2.0.18", + "walkdir", + "zlib-rs", +] + +[[package]] +name = "gix-filter" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecf74b7d16f6694ce4a3049074c41be0c7987105743674f1671807bd6dce09fa" +dependencies = [ + "bstr", + "encoding_rs", + "gix-attributes", + "gix-command", + "gix-hash", + "gix-object", + "gix-packetline", + "gix-path", + "gix-quote", + "gix-trace", + "gix-utils", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-fs" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cdff46db8798e47e2f727d84b9379aac5add3dd3d9d0b07bb4d7d5d640771fe" +dependencies = [ + "bstr", + "fastrand", + "gix-features", + "gix-path", + "gix-utils", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-glob" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1fcb8ef5b16bcf874abe9b68d8abb3c0493c876d367ab824151f30a0f3f3756" +dependencies = [ + "bitflags", + "bstr", + "gix-features", + "gix-path", +] + +[[package]] +name = "gix-hash" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0926d3819c837750b4e03c7754901e73f68b8c9b690753a6372a1bed4eedce" +dependencies = [ + "faster-hex", + "gix-features", + "sha1-checked", + "sha2 0.11.0", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-hashtable" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0e30b93eea8718baf7d8153fcb938e2926175bbf18097c09f1c01b6f0be0563" +dependencies = [ + "gix-hash", + "hashbrown 0.17.1", + "parking_lot", +] + +[[package]] +name = "gix-ignore" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d491bab9bf2c9f341dc754f425c31d5d3f63aca615312167b82e1deeaca97d8d" +dependencies = [ + "bstr", + "gix-glob", + "gix-path", + "gix-trace", + "unicode-bom", +] + +[[package]] +name = "gix-imara-diff" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19753d40da53d0ec41604750eeb969097a90fb2d7f7992730d904541c04e2c19" +dependencies = [ + "bstr", + "hashbrown 0.17.1", +] + +[[package]] +name = "gix-index" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e6b28cc592dc753adb58302bb14a64e412ee591a3bec77aa4df87bff74fa80d" +dependencies = [ + "bitflags", + "bstr", + "filetime", + "fnv", + "gix-bitmap", + "gix-features", + "gix-fs", + "gix-hash", + "gix-lock", + "gix-object", + "gix-traverse", + "gix-utils", + "gix-validate", + "hashbrown 0.17.1", + "itoa", + "libc", + "memmap2", + "rustix", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-lock" +version = "23.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9dedd9e90b0d47624d2ed241d394e09294118364e87b9b7e5f1fe755f3c2c" +dependencies = [ + "gix-tempfile", + "gix-utils", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-mailmap" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "195fd20808055824531be2fd0d34136d900e5fbca3ffb0a3c07e8beeefb9c828" +dependencies = [ + "bstr", + "gix-actor", + "gix-date", + "gix-error", +] + +[[package]] +name = "gix-negotiate" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "890c936a215bae25818c076cb881cb2e54d2c66ba947ba58b8dd47cff921bf55" +dependencies = [ + "bitflags", + "gix-commitgraph", + "gix-date", + "gix-hash", + "gix-object", + "gix-revwalk", +] + +[[package]] +name = "gix-object" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5cd857e29429c7213bdef3f5aef83f8cc124774fe8ae0d27b1607d218d6d525" +dependencies = [ + "bstr", + "gix-actor", + "gix-date", + "gix-features", + "gix-hash", + "gix-hashtable", + "gix-utils", + "gix-validate", + "itoa", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-odb" +version = "0.81.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d004c32858b1556f2d7874405edb3c97dc78fc09beaa87d57bb077ee2858a7d" +dependencies = [ + "arc-swap", + "gix-features", + "gix-fs", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-pack", + "gix-path", + "gix-quote", + "memmap2", + "parking_lot", + "tempfile", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-pack" +version = "0.71.0" +dependencies = [ + "clru", + "gix-chunk", + "gix-diff", + "gix-error", + "gix-features", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-path", + "gix-tempfile", + "gix-traverse", + "parking_lot", + "smallvec", + "thiserror 2.0.18", + "uluru", +] + +[[package]] +name = "gix-packetline" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb18337ba2830bb43367d1af43819c8c78f31337f079fc76d0f1f1750a173126" +dependencies = [ + "bstr", + "faster-hex", + "gix-trace", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-path" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afa6ac14cd14939ea94a496ce7460daa6511c09f5b84757e9cfc6f9c8d0f93a6" +dependencies = [ + "bstr", + "gix-trace", + "gix-validate", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-pathspec" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3050783b41ee11511e1e8fb35623df81806194f4030395f14f48ea37c2798c9f" +dependencies = [ + "bitflags", + "bstr", + "gix-attributes", + "gix-config-value", + "gix-glob", + "gix-path", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-prompt" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee604d7746080ae7e1023bf47204bcc2c5f307bfbe2306a3c90b1bfd1a2c6d8" +dependencies = [ + "gix-command", + "gix-config-value", + "parking_lot", + "rustix", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-protocol" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51dea3acb390707ab868f1f9584f18449eb95d869deffae96768e47d303595ee" +dependencies = [ + "bstr", + "gix-date", + "gix-features", + "gix-hash", + "gix-ref", + "gix-shallow", + "gix-transport", + "gix-utils", + "maybe-async", + "nonempty 0.12.0", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-quote" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6e541fc33cc2b783b7979040d445a0c86a2eca747c8faea4ca84230d06ae6ef" +dependencies = [ + "bstr", + "gix-error", + "gix-utils", +] + +[[package]] +name = "gix-ref" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c04f64c37eb7e6feb73c7060f8dc6f381cc5de5d53249bfd450bc48a86b2e8b" +dependencies = [ + "gix-actor", + "gix-features", + "gix-fs", + "gix-hash", + "gix-lock", + "gix-object", + "gix-path", + "gix-tempfile", + "gix-utils", + "gix-validate", + "memmap2", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-refspec" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b216ae06ec74b5f24ad0142026a997fb0a935b7410eaf9c1616fc3f0e6c5a6d3" +dependencies = [ + "bstr", + "gix-error", + "gix-glob", + "gix-hash", + "gix-revision", + "gix-validate", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-revision" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b47c88884dd3c1a19a39da19d10211fcdea2809aadc86869b6e824a1774340f" +dependencies = [ + "bitflags", + "bstr", + "gix-commitgraph", + "gix-date", + "gix-error", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-revwalk", + "gix-trace", + "nonempty 0.12.0", +] + +[[package]] +name = "gix-revwalk" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85f5756abffe0917827aac683b13684ed99875bc398fa1f9b8f479b0681ef9e6" +dependencies = [ + "gix-commitgraph", + "gix-date", + "gix-error", + "gix-hash", + "gix-hashtable", + "gix-object", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-sec" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab8519976e4c7e486270740a5400369f37940779b80bd1377d94cfa1125d01b3" +dependencies = [ + "bitflags", + "gix-path", + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "gix-shallow" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a292fc2fe548c5dfa575479d16b445b0ddf1dd2f56f1fec6aed386f82553cd97" +dependencies = [ + "bstr", + "gix-hash", + "gix-lock", + "nonempty 0.12.0", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-status" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22042e385d28a34275e029d98f4970285045be14b9073658ca897923f2ed8700" +dependencies = [ + "bstr", + "filetime", + "gix-diff", + "gix-dir", + "gix-features", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-index", + "gix-object", + "gix-path", + "gix-pathspec", + "gix-worktree", + "portable-atomic", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-submodule" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3059890ef054066c22a94bfc6a3eaba0d806aedcd630a0bc9e5783fd88884781" +dependencies = [ + "bstr", + "gix-config", + "gix-path", + "gix-pathspec", + "gix-refspec", + "gix-url", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-tempfile" +version = "23.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27850097e1ff9515f46a0dad0f5f9c9d020e972727772dabab9450690c4adb22" +dependencies = [ + "dashmap", + "gix-fs", + "libc", + "parking_lot", + "signal-hook", + "signal-hook-registry", + "tempfile", +] + +[[package]] +name = "gix-trace" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44dc45eae785c0eb14173e0f152e6e224dcf4d45b6a6999a3aed22af541ad678" + +[[package]] +name = "gix-transport" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd0e34995b1aab0fa8dff2af8db726a0bfad3e119c89302604463264046e7ff" +dependencies = [ + "bstr", + "gix-command", + "gix-features", + "gix-packetline", + "gix-quote", + "gix-sec", + "gix-url", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-traverse" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8de590ecc86a3b2870665f2288324fa9f7f8672c7fc2d4e020fdd81cd1f7aed" +dependencies = [ + "bitflags", + "gix-commitgraph", + "gix-date", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-revwalk", + "smallvec", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-url" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bb01ec69d55e82ccb7a19e264501ead4e6aac38463a8cebfdd81e22bb67ab2" +dependencies = [ + "bstr", + "gix-path", + "percent-encoding", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-utils" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66c50966184123caf580ffa64e28031a878597f1c7fceb8fe19566c38eb1b771" +dependencies = [ + "bstr", + "fastrand", + "unicode-normalization", +] + +[[package]] +name = "gix-validate" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bc6fc771c4063ba7cd2f47b91fb6076251c6a823b64b7fe7b8874b0fe4afae3" +dependencies = [ + "bstr", +] + +[[package]] +name = "gix-worktree" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef414ed275e8407cd5d53d301e83be19700b0dd3f859d2434417b58f454a2d1" +dependencies = [ + "bstr", + "gix-attributes", + "gix-fs", + "gix-glob", + "gix-hash", + "gix-ignore", + "gix-index", + "gix-object", + "gix-path", + "gix-validate", +] + +[[package]] +name = "gix-worktree-state" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bffae8b3ca258fdd50370cd51f06deb4c76a3b43db3868bc28dde45ffa77d69" +dependencies = [ + "bstr", + "gix-features", + "gix-filter", + "gix-fs", + "gix-index", + "gix-object", + "gix-path", + "gix-worktree", + "io-close", + "thiserror 2.0.18", +] + +[[package]] +name = "gix-worktree-stream" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25e9ed30100c63f7590bc581c225e53f731a53e06aa79a245739c07f7dcc557" +dependencies = [ + "gix-attributes", + "gix-error", + "gix-features", + "gix-filter", + "gix-fs", + "gix-hash", + "gix-object", + "gix-path", + "gix-traverse", + "parking_lot", +] + +[[package]] +name = "governor" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9efcab3c1958580ff1f25a2a41be1668f7603d849bb63af523b208a3cc1223b8" +dependencies = [ + "cfg-if", + "dashmap", + "futures-sink", + "futures-timer", + "futures-util", + "getrandom 0.3.4", + "hashbrown 0.16.1", + "nonzero_ext", + "parking_lot", + "portable-atomic", + "quanta", + "rand 0.9.4", + "smallvec", + "spinning_top", + "web-time", +] + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "h2" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h3" +version = "0.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10872b55cfb02a821b69dc7cf8dc6a71d6af25eb9a79662bec4a9d016056b3be" +dependencies = [ + "bytes", + "fastrand", + "futures-util", + "http", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "h3-quinn" +version = "0.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2e732c8d91a74731663ac8479ab505042fbf547b9a207213ab7fbcbfc4f8b4" +dependencies = [ + "bytes", + "futures", + "h3", + "quinn", + "tokio", + "tokio-util", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "heapless" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" +dependencies = [ + "atomic-polyfill", + "hash32 0.2.1", + "rustc_version", + "serde", + "spin 0.9.8", + "stable_deref_trait", +] + +[[package]] +name = "heapless" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32 0.3.1", + "stable_deref_trait", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "http" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "human_format" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaec953f16e5bcf6b8a3cb3aa959b17e5577dbd2693e94554c462c08be22624b" + +[[package]] +name = "hybrid-array" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots 1.0.7", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "inventory" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +dependencies = [ + "rustversion", +] + +[[package]] +name = "io-close" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cadcf447f06744f8ce713d2d6239bb5bde2c357a452397a9ed90c625da390bc" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "ipld-core" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090f624976d72f0b0bb71b86d58dc16c15e069193067cb3a3a09d655246cbbda" +dependencies = [ + "cid", + "serde", + "serde_bytes", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "iroh-car" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f8cd4cb9aa083fba8b52e921764252d0b4dcb1cd6d120b809dbfe1106e81a" +dependencies = [ + "anyhow", + "cid", + "futures", + "serde", + "serde_ipld_dagcbor", + "thiserror 1.0.69", + "tokio", + "unsigned-varint 0.7.2", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jacquard-api" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5c803a3c097e3ef8aea63747b4fe3fc9e339cd18272dd0366b1d10dd90d5c3f" +dependencies = [ + "jacquard-common", + "jacquard-derive", + "jacquard-lexicon", + "miette", + "serde", + "thiserror 2.0.18", +] + +[[package]] +name = "jacquard-common" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fec8b6661e6fcfb4d8b9eb1132503be71bc3e3c6f6ff15fa5500ec62655f6da7" +dependencies = [ + "base64", + "bon", + "bytes", + "chrono", + "ciborium", + "ciborium-io", + "cid", + "ed25519-dalek", + "fluent-uri", + "futures", + "getrandom 0.2.17", + "getrandom 0.3.4", + "hashbrown 0.15.5", + "http", + "ipld-core", + "k256", + "maitake-sync", + "miette", + "multibase", + "multihash", + "n0-future", + "oxilangtag", + "p256", + "phf", + "postcard", + "rand 0.9.4", + "regex", + "regex-automata", + "regex-lite", + "reqwest 0.12.28", + "rustversion", + "serde", + "serde_bytes", + "serde_html_form", + "serde_ipld_dagcbor", + "serde_json", + "signature", + "smol_str", + "spin 0.10.0", + "thiserror 2.0.18", + "tokio", + "tokio-tungstenite-wasm", + "tokio-util", + "trait-variant", + "unicode-segmentation", +] + +[[package]] +name = "jacquard-derive" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e41b82a9a72c0d2b907a5e0734e0692615e717cda903502c6c034ae6f3ddddf7" +dependencies = [ + "heck", + "jacquard-lexicon", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jacquard-lexicon" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "160049c269e3d7ec56f130736d25d61953fc6216da660c8b5e102f6b052dd3a1" +dependencies = [ + "cid", + "dashmap", + "heck", + "inventory", + "jacquard-common", + "miette", + "multihash", + "prettyplease", + "proc-macro2", + "quote", + "serde", + "serde_ipld_dagcbor", + "serde_json", + "serde_path_to_error", + "serde_repr", + "serde_with", + "sha2 0.10.9", + "syn", + "thiserror 2.0.18", + "unicode-segmentation", +] + +[[package]] +name = "jacquard-repo" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98986367bb78dadaa0f2f07196bab357786c0e3670d8311b350585b91f84d6eb" +dependencies = [ + "bytes", + "cid", + "ed25519-dalek", + "iroh-car", + "jacquard-api", + "jacquard-common", + "jacquard-derive", + "k256", + "miette", + "multihash", + "n0-future", + "p256", + "serde", + "serde_bytes", + "serde_ipld_dagcbor", + "sha2 0.10.9", + "smol_str", + "thiserror 2.0.18", + "tokio", + "trait-variant", +] + +[[package]] +name = "jiff" +version = "0.2.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4603d3033e49e2b0e31229fcab20a5d40089c607d975cd9c80551dc69eed9102" +dependencies = [ + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link", +] + +[[package]] +name = "jiff-static" +version = "0.2.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "782d32378dddf207193ac91cefb848ad41abb58195c95168e1291227a0832b47" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c900ef84826f1338a557697dc8fc601df9ca9af4ac137c7fb61d4c6f2dfd3076" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" +dependencies = [ + "cfg-if", + "futures-util", + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "k256" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" +dependencies = [ + "cfg-if", + "ecdsa", + "elliptic-curve", + "once_cell", + "sha2 0.10.9", + "signature", +] + +[[package]] +name = "knot-edge" +version = "0.1.0" +dependencies = [ + "arc-swap", + "async-trait", + "axum", + "base64", + "bytes", + "futures", + "governor", + "h3", + "h3-quinn", + "http", + "http-body", + "hyper", + "hyper-util", + "knot-types", + "quinn", + "rustls", + "rustls-acme", + "rustls-pemfile", + "sha2 0.11.0", + "thiserror 2.0.18", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http 0.7.0", + "tower_governor", + "tracing", + "x509-parser 0.18.1", +] + +[[package]] +name = "knot-git" +version = "0.1.0" +dependencies = [ + "base64", + "dashmap", + "flate2", + "gix", + "gix-archive", + "gix-bitmap", + "gix-hash", + "gix-pack", + "knot-resource", + "knot-types", + "moka", + "thiserror 2.0.18", + "walkdir", +] + +[[package]] +name = "knot-pack" +version = "0.1.0" +dependencies = [ + "axum", + "dashmap", + "flate2", + "gix", + "gix-hash", + "gix-pack", + "gix-packetline", + "knot-edge", + "knot-git", + "knot-resource", + "knot-runtime", + "knot-types", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "url", + "walkdir", +] + +[[package]] +name = "knot-pack-fuzz" +version = "0.0.0" +dependencies = [ + "knot-pack", + "libfuzzer-sys", +] + +[[package]] +name = "knot-resource" +version = "0.1.0" +dependencies = [ + "rustix", +] + +[[package]] +name = "knot-runtime" +version = "0.1.0" +dependencies = [ + "bytes", + "futures", + "getrandom 0.4.2", + "http", + "k256", + "reqwest 0.13.1", + "thiserror 2.0.18", + "tokio", + "url", +] + +[[package]] +name = "knot-types" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "chrono", + "cid", + "gix-hash", + "http", + "jacquard-common", + "jacquard-derive", + "jacquard-lexicon", + "jacquard-repo", + "miette", + "serde", + "serde_json", + "thiserror 2.0.18", + "walkdir", +] + +[[package]] +name = "kstring" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "558bf9508a558512042d3095138b1f7b8fe90c5467d94f9f1da28b3731c5dbd1" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" + +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "maitake-sync" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6816ab14147f80234c675b80ed6dc4f440d8a1cefc158e766067aedb84c0bcd5" +dependencies = [ + "cordyceps", + "loom", + "mycelium-bitfield", + "pin-project", + "portable-atomic", +] + +[[package]] +name = "match-lookup" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "757aee279b8bdbb9f9e676796fd459e4207a1f986e87886700abf589f5abf771" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "maybe-async" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "746873a384ad60adc5db74471dfaba74bd278afbdcfd81db93fafcdfc8b5ca0c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "memchr" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" + +[[package]] +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +dependencies = [ + "libc", +] + +[[package]] +name = "miette" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" +dependencies = [ + "cfg-if", + "miette-derive", + "unicode-width", +] + +[[package]] +name = "miette-derive" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "moka" +version = "0.12.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "event-listener", + "futures-util", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "multibase" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8694bb4835f452b0e3bb06dbebb1d6fc5385b6ca1caf2e55fd165c042390ec77" +dependencies = [ + "base-x", + "base256emoji", + "data-encoding", + "data-encoding-macro", +] + +[[package]] +name = "multihash" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "577c63b00ad74d57e8c9aa870b5fccebf2fd64a308a5aee9f1bb88e4aea19447" +dependencies = [ + "serde", + "unsigned-varint 0.8.0", +] + +[[package]] +name = "mycelium-bitfield" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24e0cc5e2c585acbd15c5ce911dff71e1f4d5313f43345873311c4f5efd741cc" + +[[package]] +name = "n0-future" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb0e5d99e681ab3c938842b96fcb41bf8a7bb4bfdb11ccbd653a7e83e06c794" +dependencies = [ + "cfg_aliases", + "derive_more", + "futures-buffered", + "futures-lite", + "futures-util", + "js-sys", + "pin-project", + "send_wrapper", + "tokio", + "tokio-util", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nonempty" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9e591e719385e6ebaeb5ce5d3887f7d5676fceca6411d1925ccc95745f3d6f7" + +[[package]] +name = "nonempty" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9737e026353e5cd0736f98eddae28665118eb6f6600902a7f50db585621fecb6" + +[[package]] +name = "nonzero_ext" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.60.2", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "oid-registry" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d8034d9489cdaf79228eb9f6a3b8d7bb32ba00d6645ebd48eef4077ceb5bd9" +dependencies = [ + "asn1-rs 0.6.2", +] + +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs 0.7.2", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "oxilangtag" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d3b4eb570abd4a1dcb062c31fd37b832264d9dc7292c3e69acfe926c87b063f" +dependencies = [ + "serde", +] + +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2 0.10.9", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64", + "serde_core", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand 0.8.6", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "piper" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1" +dependencies = [ + "atomic-waker", + "fastrand", + "futures-io", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "heapless 0.7.17", + "serde", +] + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prodash" +version = "31.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "962200e2d7d551451297d9fdce85138374019ada198e30ea9ede38034e27604c" +dependencies = [ + "bytesize", + "human_format", + "parking_lot", +] + +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "futures-io", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.4", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags", +] + +[[package]] +name = "rawzip" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d9575f44c8cf85bc843ad666dcdf20d05a7753772bef56eb2a5140282b32150" + +[[package]] +name = "rcgen" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" +dependencies = [ + "aws-lc-rs", + "pem", + "rustls-pki-types", + "time", + "yasna", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "mime", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http 0.6.11", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots 1.0.7", +] + +[[package]] +name = "reqwest" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04e9018c9d814e5f30cc16a0f03271aeab3571e609612d9fe78c1aa8d11c2f62" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http 0.6.11", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots 1.0.7", +] + +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "aws-lc-rs", + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-acme" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9c70a17ecb067d5067565a16a2e0f26a4a2ea0924f49739d558c45186facc75" +dependencies = [ + "async-io", + "async-trait", + "async-web-client", + "aws-lc-rs", + "base64", + "blocking", + "chrono", + "futures", + "futures-rustls", + "http", + "log", + "pem", + "rcgen", + "serde", + "serde_json", + "thiserror 2.0.18", + "webpki-roots 1.0.7", + "x509-parser 0.16.0", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "send_wrapper" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_html_form" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acf96b1d9364968fce46ebb548f1c0e1d7eceae27bdff73865d42e6c7369d94" +dependencies = [ + "form_urlencoded", + "indexmap", + "itoa", + "serde_core", +] + +[[package]] +name = "serde_ipld_dagcbor" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46182f4f08349a02b45c998ba3215d3f9de826246ba02bb9dddfe9a2a2100778" +dependencies = [ + "cbor4ii", + "ipld-core", + "scopeguard", + "serde", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" +dependencies = [ + "base64", + "bs58", + "chrono", + "hex", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha1-checked" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89f599ac0c323ebb1c6082821a54962b839832b03984598375bff3975b804423" +dependencies = [ + "digest 0.10.7", + "sha1", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shell-words" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a0c28ca5908dbdbcd52e6fdaa00358ab88637f8ab33e1f188dd510eb44b53d" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "smol_str" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa7368fcf4852a4c2dd92df0cace6a71f2091ca0a23391ce7f3a31833f1523" +dependencies = [ + "borsh", + "serde_core", +] + +[[package]] +name = "socket2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" + +[[package]] +name = "spinning_top" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d96d2d1d716fb500937168cc09353ffdc7a012be8475ac7308e1bdf0e3923300" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.52.0", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" +dependencies = [ + "futures-util", + "log", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tungstenite", +] + +[[package]] +name = "tokio-tungstenite-wasm" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e21a5c399399c3db9f08d8297ac12b500e86bca82e930253fdc62eaf9c0de6ae" +dependencies = [ + "futures-channel", + "futures-util", + "http", + "httparse", + "js-sys", + "rustls", + "thiserror 1.0.69", + "tokio", + "tokio-tungstenite", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "async-compression", + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tokio", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-http" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233" +dependencies = [ + "async-compression", + "bitflags", + "bytes", + "futures-core", + "http", + "http-body", + "percent-encoding", + "pin-project-lite", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tower_governor" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44de9b94d849d3c46e06a883d72d408c2de6403367b39df2b1c9d9e7b6736fe6" +dependencies = [ + "axum", + "forwarded-header-value", + "governor", + "http", + "pin-project", + "thiserror 2.0.18", + "tower", + "tracing", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "trait-variant" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.8.6", + "rustls", + "rustls-pki-types", + "sha1", + "thiserror 1.0.69", + "utf-8", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "uluru" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c8a2469e56e6e5095c82ccd3afb98dad95f7af7929aab6d8ba8d6e0f73657da" +dependencies = [ + "arrayvec", +] + +[[package]] +name = "unicode-bom" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eec5d1121208364f6793f7d2e222bf75a915c19557537745b195b253dd64217" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "unsigned-varint" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" + +[[package]] +name = "unsigned-varint" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", + "serde_derive", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" +dependencies = [ + "getrandom 0.4.2", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.3+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +dependencies = [ + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.122" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "web-sys" +version = "0.3.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.7", +] + +[[package]] +name = "webpki-roots" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "x509-parser" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcbc162f30700d6f3f82a24bf7cc62ffe7caea42c0b2cba8bf7f3ae50cf51f69" +dependencies = [ + "asn1-rs 0.6.2", + "data-encoding", + "der-parser 9.0.0", + "lazy_static", + "nom", + "oid-registry 0.7.1", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs 0.7.2", + "data-encoding", + "der-parser 10.0.0", + "lazy_static", + "nom", + "oid-registry 0.8.1", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zlib-rs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/knot2/crates/knot-pack/fuzz/Cargo.toml b/knot2/crates/knot-pack/fuzz/Cargo.toml new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/fuzz/Cargo.toml @@ -0,0 +1,47 @@ +[package] +name = "knot-pack-fuzz" +version = "0.0.0" +publish = false +edition = "2024" + +[package.metadata] +cargo-fuzz = true + +[workspace] + +[dependencies] +libfuzzer-sys = "0.4" + +[dependencies.knot-pack] +path = ".." + +[[bin]] +name = "pkt" +path = "fuzz_targets/pkt.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "pack" +path = "fuzz_targets/pack.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "receive_commands" +path = "fuzz_targets/receive_commands.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "upload_args" +path = "fuzz_targets/upload_args.rs" +test = false +doc = false +bench = false + +[patch.crates-io] +gix-pack = { path = "../../../third_party/gix-pack" } diff --git a/knot2/crates/knot-pack/src/archive.rs b/knot2/crates/knot-pack/src/archive.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/src/archive.rs @@ -0,0 +1,139 @@ +use std::io::{self, Read, Seek, SeekFrom}; + +use knot_git::{ArchiveFormat, ArchivePrefix, Repo}; +use knot_types::Oid; + +use crate::error::PackError; +use crate::pkt; + +struct Request { + treeish: String, + format: ArchiveFormat, + prefix: Option, +} + +pub fn stream( + repo: &Repo, + request: &[u8], + sink: &mut dyn FnMut(&[u8]) -> io::Result<()>, +) -> Result<(), PackError> { + let args = parse_arguments(request)?; + match build(repo, &args) { + Ok(mut spool) => { + let mut head = Vec::new(); + pkt::write_data(&mut head, b"ACK\n")?; + pkt::write_flush(&mut head)?; + emit(sink, &head)?; + std::iter::from_fn(|| { + let mut chunk = vec![0u8; pkt::MAX_BAND]; + match spool.read(&mut chunk) { + Ok(0) => None, + Ok(read) => { + chunk.truncate(read); + Some(Ok(chunk)) + } + Err(error) => Some(Err(error)), + } + }) + .try_for_each(|chunk| -> Result<(), PackError> { + let chunk = chunk.map_err(|error| PackError::Pack(error.to_string()))?; + let mut framed = Vec::new(); + pkt::write_band(&mut framed, &chunk)?; + emit(sink, &framed) + })?; + let mut tail = Vec::new(); + pkt::write_flush(&mut tail)?; + emit(sink, &tail) + } + Err(error) => { + let mut buf = Vec::new(); + pkt::write_data( + &mut buf, + format!("NACK {}\n", error.to_string().replace('\n', " ")).as_bytes(), + )?; + pkt::write_flush(&mut buf)?; + emit(sink, &buf) + } + } +} + +fn emit(sink: &mut dyn FnMut(&[u8]) -> io::Result<()>, bytes: &[u8]) -> Result<(), PackError> { + sink(bytes).map_err(|error| PackError::Pack(error.to_string())) +} + +fn parse_arguments(request: &[u8]) -> Result, PackError> { + Ok(pkt::data_payloads(request)? + .iter() + .filter_map(|line| { + std::str::from_utf8(line) + .ok()? + .trim_end_matches('\n') + .strip_prefix("argument ") + .map(str::to_string) + }) + .collect()) +} + +fn interpret(args: &[String]) -> Result { + let format = args + .iter() + .find_map(|arg| arg.strip_prefix("--format=")) + .map(format_from) + .unwrap_or(ArchiveFormat::Tar); + let prefix = args + .iter() + .find_map(|arg| arg.strip_prefix("--prefix=")) + .map(|raw| { + ArchivePrefix::new(raw).map_err(|_| { + PackError::Protocol("archive prefix must not escape archive root".to_string()) + }) + }) + .transpose()?; + let treeish = args + .iter() + .find(|arg| !arg.starts_with('-')) + .cloned() + .ok_or_else(|| PackError::Protocol("archive request has no tree-ish".to_string()))?; + Ok(Request { + treeish, + format, + prefix, + }) +} + +fn format_from(value: &str) -> ArchiveFormat { + match value { + "zip" => ArchiveFormat::Zip, + "tar.gz" | "tgz" => ArchiveFormat::TarGz, + _ => ArchiveFormat::Tar, + } +} + +fn build(repo: &Repo, args: &[String]) -> Result { + let request = interpret(args)?; + let id = repo + .resolve_revision(&request.treeish) + .ok_or_else(|| PackError::Protocol(format!("cannot resolve {}", request.treeish)))?; + let commit = ensure_public_commit(repo, id)?; + let tree = repo + .peel_to_tree(commit) + .map_err(|error| PackError::Pack(error.to_string()))?; + let mut spool = tempfile::tempfile().map_err(|error| PackError::Pack(error.to_string()))?; + repo.write_archive(tree, request.format, request.prefix.as_ref(), &mut spool) + .map_err(|error| PackError::Pack(error.to_string()))?; + spool + .seek(SeekFrom::Start(0)) + .map_err(|error| PackError::Pack(error.to_string()))?; + Ok(spool) +} + +fn ensure_public_commit(repo: &Repo, id: Oid) -> Result { + let unreachable = + || PackError::Protocol("tree-ish is not reachable from public ref".to_string()); + let commit = repo.peel_to_commit(id).map_err(|_| unreachable())?; + match repo.reachable_from_public(commit) { + Ok(true) => Ok(commit), + Ok(false) => Err(unreachable()), + Err(error) => Err(PackError::Pack(error.to_string())), + } +} diff --git a/knot2/crates/knot-pack/src/cache.rs b/knot2/crates/knot-pack/src/cache.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/src/cache.rs @@ -0,0 +1,511 @@ +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use axum::body::Bytes; +use knot_cache::{Cache, EntryCount, Lru, Reclaimable, Weight}; +use knot_runtime::Clock; +use tokio::sync::watch; + +const MAX_ENTRIES: usize = 1024; + +knot_types::scalar_newtype! { + pub struct MaxEntryBytes(usize); + pub struct MaxCacheBytes(usize); +} + +#[derive(Debug, Clone, Copy)] +pub struct CacheConfig { + pub enabled: bool, + pub ttl: Duration, + pub max_entry_bytes: MaxEntryBytes, + pub max_total_bytes: MaxCacheBytes, +} + +impl Default for CacheConfig { + fn default() -> Self { + Self { + enabled: true, + ttl: Duration::from_secs(60), + max_entry_bytes: MaxEntryBytes::new(64 * 1024 * 1024), + max_total_bytes: MaxCacheBytes::new(512 * 1024 * 1024), + } + } +} + +#[derive(Clone, PartialEq, Eq, Hash)] +pub(crate) struct RequestKey { + objects_dir: PathBuf, + digest: [u8; 32], +} + +impl RequestKey { + pub(crate) fn new(objects_dir: &Path, ref_token: &crate::ids::RefsDigest, body: &[u8]) -> Self { + let mut hasher = gix::hash::hasher(gix::hash::Kind::Sha256); + hasher.update(ref_token.as_bytes()); + hasher.update(body); + let id = hasher.try_finalize().expect("sha256 digest finalizes"); + let mut digest = [0u8; 32]; + digest.copy_from_slice(id.as_slice()); + Self { + objects_dir: objects_dir.to_path_buf(), + digest, + } + } +} + +#[derive(Clone)] +pub(crate) enum Signal { + Pending, + Ready(Bytes), + Retry, + Regenerate, +} + +#[derive(Clone)] +enum Slot { + Ready(Bytes), + TooLarge, +} + +fn weigh(slot: &Slot) -> Weight { + match slot { + Slot::Ready(bytes) => Weight::new(bytes.len() as u64), + Slot::TooLarge => Weight::new(0), + } +} + +enum Settlement { + Ready(Bytes), + TooLarge, + Regenerate, + Retry, +} + +pub(crate) struct PackCache { + config: CacheConfig, + store: Lru>, + inflight: Mutex>>, +} + +pub(crate) enum Decision { + Serve(Bytes), + Await(watch::Receiver), + Lead(Lease), + Stream, + Off, +} + +impl PackCache { + pub(crate) fn new(mut config: CacheConfig, clock: Arc) -> Arc { + config.max_entry_bytes = MaxEntryBytes::new( + config + .max_entry_bytes + .get() + .min(config.max_total_bytes.get()), + ); + let store = Lru::by_weight_with_ttl( + Weight::new(config.max_total_bytes.get() as u64), + config.ttl, + clock, + weigh, + ) + .with_entry_cap(EntryCount::new(MAX_ENTRIES as u64)); + let cache = Arc::new(Self { + config, + store, + inflight: Mutex::new(HashMap::new()), + }); + knot_cache::register(&cache); + cache + } + + pub(crate) fn decide(self: &Arc, key: RequestKey) -> Decision { + if !self.config.enabled { + return Decision::Off; + } + if let Some(decision) = self.serve_cached(&key) { + return decision; + } + let mut inflight = self.inflight_lock(); + if let Some(sender) = inflight.get(&key) { + return Decision::Await(sender.subscribe()); + } + // `store` & `inflight` are separate locks, + // such that a leader can finish settling in the gap between + // a fast-path miss and this lock acquisition, + // by which point it has inserted the pack and taken its sender away. + // Reading the store a second time will cover that gap, + // since `settle` always inserts before it removes. + // + // Without it, + // the race-losing caller would elect itself and rebuild a pack + // the store does in fact already have. + if let Some(decision) = self.serve_cached(&key) { + return decision; + } + let (sender, _) = watch::channel(Signal::Pending); + inflight.insert(key.clone(), sender); + Decision::Lead(Lease { + key, + cache: Arc::clone(self), + max_entry_bytes: self.config.max_entry_bytes, + settled: false, + }) + } + + fn serve_cached(&self, key: &RequestKey) -> Option { + match self.store.get(key) { + Some(Slot::Ready(bytes)) => Some(Decision::Serve(bytes)), + Some(Slot::TooLarge) => Some(Decision::Stream), + None => None, + } + } + + fn settle(&self, key: &RequestKey, settlement: Settlement) { + let signal = match settlement { + Settlement::Ready(bytes) => { + self.store.insert(key.clone(), Slot::Ready(bytes.clone())); + Signal::Ready(bytes) + } + Settlement::TooLarge => { + self.store.insert(key.clone(), Slot::TooLarge); + Signal::Regenerate + } + Settlement::Regenerate => Signal::Regenerate, + Settlement::Retry => Signal::Retry, + }; + if let Some(sender) = self.inflight_lock().remove(key) { + let _ = sender.send(signal); + } + } + + fn inflight_lock( + &self, + ) -> std::sync::MutexGuard<'_, HashMap>> { + self.inflight + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + #[cfg(test)] + fn entry_count(&self) -> u64 { + self.store.entry_count().get() + } +} + +impl Reclaimable for PackCache { + fn footprint(&self) -> Weight { + self.store.footprint() + } + + fn reclaim(&self) { + self.store.reclaim(); + } +} + +pub(crate) struct Lease { + key: RequestKey, + cache: Arc, + max_entry_bytes: MaxEntryBytes, + settled: bool, +} + +impl Lease { + pub(crate) fn max_entry_bytes(&self) -> MaxEntryBytes { + self.max_entry_bytes + } + + pub(crate) fn ready(mut self, bytes: Bytes) { + self.cache.settle(&self.key, Settlement::Ready(bytes)); + self.settled = true; + } + + pub(crate) fn too_large(mut self) { + self.cache.settle(&self.key, Settlement::TooLarge); + self.settled = true; + } + + pub(crate) fn regenerate(mut self) { + self.cache.settle(&self.key, Settlement::Regenerate); + self.settled = true; + } + + pub(crate) fn retry(mut self) { + self.cache.settle(&self.key, Settlement::Retry); + self.settled = true; + } +} + +impl Drop for Lease { + fn drop(&mut self) { + if !self.settled { + self.cache.settle(&self.key, Settlement::Retry); + } + } +} + +pub(crate) enum Resolved { + Bytes(Bytes), + Retry, + Regenerate, +} + +pub(crate) async fn wait(mut receiver: watch::Receiver) -> Resolved { + let resolved = classify(&receiver.borrow_and_update()); + match resolved { + Some(resolved) => resolved, + None => match receiver.changed().await { + Err(_) => Resolved::Regenerate, + Ok(()) => Box::pin(wait(receiver)).await, + }, + } +} + +fn classify(signal: &Signal) -> Option { + match signal { + Signal::Pending => None, + Signal::Ready(bytes) => Some(Resolved::Bytes(bytes.clone())), + Signal::Retry => Some(Resolved::Retry), + Signal::Regenerate => Some(Resolved::Regenerate), + } +} + +pub(crate) enum Capture { + Buffering { buffer: Vec, limit: usize }, + Overflow, + Off, +} + +impl Capture { + pub(crate) fn new(limit: Option) -> Self { + match limit { + Some(limit) => Capture::Buffering { + buffer: Vec::new(), + limit: limit.get(), + }, + None => Capture::Off, + } + } + + pub(crate) fn record(&mut self, chunk: &[u8]) { + match self { + Capture::Buffering { buffer, limit } if buffer.len() + chunk.len() <= *limit => { + buffer.extend_from_slice(chunk) + } + Capture::Buffering { .. } => *self = Capture::Overflow, + _ => {} + } + } + + pub(crate) fn into_bytes(self) -> Option> { + match self { + Capture::Buffering { buffer, .. } => Some(buffer), + _ => None, + } + } +} + +#[cfg(test)] +mod tests { + use knot_runtime::{ManualClock, SystemClock, UnixMicros}; + + use super::*; + + fn built(config: CacheConfig) -> Arc { + PackCache::new(config, Arc::new(SystemClock)) + } + + fn key(body: &[u8]) -> RequestKey { + let mut token = [0u8; 32]; + token[..4].copy_from_slice(b"refs"); + RequestKey::new( + Path::new("/scan/did:plc:squid/objects"), + &crate::ids::RefsDigest::new(token), + body, + ) + } + + fn lead(cache: &Arc, body: &[u8]) -> Lease { + match cache.decide(key(body)) { + Decision::Lead(lease) => lease, + _ => panic!("the first request for a key leads"), + } + } + + #[tokio::test] + async fn a_second_identical_request_serves_the_cached_pack() { + let cache = built(CacheConfig::default()); + lead(&cache, b"want").ready(Bytes::from_static(b"PACK-bytes")); + match cache.decide(key(b"want")) { + Decision::Serve(bytes) => assert_eq!(bytes.as_ref(), b"PACK-bytes"), + _ => panic!("second identical request hits the cache"), + } + } + + #[tokio::test] + async fn a_concurrent_request_awaits_the_leader_then_shares_its_bytes() { + let cache = built(CacheConfig::default()); + let leader = lead(&cache, b"clone"); + let receiver = match cache.decide(key(b"clone")) { + Decision::Await(receiver) => receiver, + _ => panic!("concurrent request awaits the inflight leader"), + }; + leader.ready(Bytes::from_static(b"shared")); + match wait(receiver).await { + Resolved::Bytes(bytes) => assert_eq!(bytes.as_ref(), b"shared"), + _ => panic!("the follower shares the leader's bytes"), + } + } + + #[tokio::test] + async fn a_regenerating_leader_wakes_followers_to_regenerate() { + let cache = built(CacheConfig::default()); + let leader = lead(&cache, b"err"); + let receiver = match cache.decide(key(b"err")) { + Decision::Await(receiver) => receiver, + _ => panic!("follower awaits"), + }; + leader.regenerate(); + assert!( + matches!(wait(receiver).await, Resolved::Regenerate), + "an errored leader tells the follower to regenerate in parallel" + ); + match cache.decide(key(b"err")) { + Decision::Lead(_) => {} + _ => panic!("a regenerated key isn't remembered, so the next request leads afresh"), + } + } + + #[tokio::test] + async fn a_retrying_leader_tells_followers_to_re_elect() { + let cache = built(CacheConfig::default()); + let leader = lead(&cache, b"vanish"); + let receiver = match cache.decide(key(b"vanish")) { + Decision::Await(receiver) => receiver, + _ => panic!("follower awaits"), + }; + leader.retry(); + assert!( + matches!(wait(receiver).await, Resolved::Retry), + "a vanished leader tells the follower to re-elect a fresh leader" + ); + match cache.decide(key(b"vanish")) { + Decision::Lead(_) => {} + _ => panic!("a retried key isn't remembered, so the next request leads afresh"), + } + } + + #[tokio::test] + async fn a_dropped_lease_re_elects_rather_than_stranding_the_follower() { + let cache = built(CacheConfig::default()); + let leader = lead(&cache, b"dropped"); + let receiver = match cache.decide(key(b"dropped")) { + Decision::Await(receiver) => receiver, + _ => panic!("follower awaits"), + }; + drop(leader); + assert!( + matches!(wait(receiver).await, Resolved::Retry), + "an unsettled lease that drops re-elects instead of stranding the follower" + ); + } + + #[tokio::test] + async fn an_oversized_leader_marks_the_key_for_direct_streaming() { + let cache = built(CacheConfig::default()); + let leader = lead(&cache, b"huge"); + let receiver = match cache.decide(key(b"huge")) { + Decision::Await(receiver) => receiver, + _ => panic!("follower awaits"), + }; + leader.too_large(); + assert!( + matches!(wait(receiver).await, Resolved::Regenerate), + "an oversized leader sends its followers to stream directly" + ); + match cache.decide(key(b"huge")) { + Decision::Stream => {} + _ => panic!("an oversized key streams directly without re-buffering"), + } + } + + #[tokio::test] + async fn an_expired_entry_is_regenerated() { + let clock = Arc::new(ManualClock::new(UnixMicros::new(0))); + let cache = PackCache::new( + CacheConfig { + ttl: Duration::from_secs(60), + ..CacheConfig::default() + }, + Arc::clone(&clock) as Arc, + ); + lead(&cache, b"stale").ready(Bytes::from_static(b"old")); + clock.advance(Duration::from_secs(61)); + match cache.decide(key(b"stale")) { + Decision::Lead(_) => {} + _ => panic!("an expired entry forces a fresh generation"), + } + } + + #[tokio::test] + async fn the_total_byte_limit_evicts_the_oldest_entry() { + let cache = built(CacheConfig { + max_total_bytes: MaxCacheBytes::new(8), + ..CacheConfig::default() + }); + lead(&cache, b"a").ready(Bytes::from(vec![0u8; 5])); + lead(&cache, b"b").ready(Bytes::from(vec![0u8; 5])); + match cache.decide(key(b"a")) { + Decision::Lead(_) => {} + _ => panic!("the oldest entry is evicted once the total limit is exceeded"), + } + match cache.decide(key(b"b")) { + Decision::Serve(_) => {} + _ => panic!("the newest entry survives eviction"), + } + } + + #[tokio::test] + async fn the_entry_limit_never_exceeds_the_total_cache_size() { + let cache = built(CacheConfig { + max_entry_bytes: MaxEntryBytes::new(64), + max_total_bytes: MaxCacheBytes::new(16), + ..CacheConfig::default() + }); + let lease = lead(&cache, b"probe"); + assert_eq!( + lease.max_entry_bytes().get(), + 16, + "a per-entry limit above the whole-cache size is clamped so a full entry can be retained" + ); + } + + #[tokio::test] + async fn oversized_entries_cannot_grow_the_cache_without_bound() { + let cache = built(CacheConfig::default()); + (0..(MAX_ENTRIES + 64)).for_each(|nonce| { + lead(&cache, format!("oversized-{nonce}").as_bytes()).too_large(); + }); + assert!( + cache.entry_count() <= MAX_ENTRIES as u64, + "a flood of distinct oversized requests stays within the entry bound" + ); + } + + #[test] + fn capture_stops_buffering_once_the_limit_is_passed() { + let mut capture = Capture::new(Some(MaxEntryBytes::new(4))); + capture.record(b"abc"); + capture.record(b"de"); + assert!(capture.into_bytes().is_none(), "overflow drops the buffer"); + } + + #[test] + fn capture_keeps_bytes_under_the_limit() { + let mut capture = Capture::new(Some(MaxEntryBytes::new(8))); + capture.record(b"abcd"); + assert_eq!(capture.into_bytes().unwrap(), b"abcd"); + } +} diff --git a/knot2/crates/knot-pack/src/error.rs b/knot2/crates/knot-pack/src/error.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/src/error.rs @@ -0,0 +1,90 @@ +use std::fmt; + +use axum::http::StatusCode; +use axum::response::{IntoResponse, Response}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PackLimit { + Objects, + ObjectBytes, + TotalBytes, + DeltaDepth, +} + +impl fmt::Display for PackLimit { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + PackLimit::Objects => "object count", + PackLimit::ObjectBytes => "per-object size", + PackLimit::TotalBytes => "total decompressed size", + PackLimit::DeltaDepth => "delta chain depth", + }) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum PackError { + #[error("repository not found")] + NotFound, + #[error("repository index is warming, retry shortly")] + Unavailable, + #[error("invalid request path: {0}")] + BadPath(String), + #[error("unsupported service")] + UnsupportedService, + #[error("push is served over SSH, not HTTP")] + PushOverSsh, + #[error("protocol: {0}")] + Protocol(String), + #[error("unsupported request content-encoding: {0}")] + UnsupportedEncoding(String), + #[error("pkt-line: {0}")] + PktLine(#[from] std::io::Error), + #[error("pack: {0}")] + Pack(String), + #[error("pack exceeds {0} limit")] + LimitExceeded(PackLimit), + #[error("upload-pack selection exceeded its object-set limit")] + SelectionTooLarge, + #[error("upload-pack selection exceeded its time budget")] + SelectionTimeout, + #[error("insufficient memory to ingest this push, retry when the server is less busy")] + InsufficientMemory, + #[error(transparent)] + Git(knot_git::GitError), +} + +impl From for PackError { + fn from(error: knot_git::GitError) -> Self { + use knot_git::SelectionLimit; + match error { + knot_git::GitError::Selection(SelectionLimit::Objects) => PackError::SelectionTooLarge, + knot_git::GitError::Selection(SelectionLimit::Time) => PackError::SelectionTimeout, + other => PackError::Git(other), + } + } +} + +impl PackError { + pub fn http_status(&self) -> StatusCode { + match self { + PackError::NotFound => StatusCode::NOT_FOUND, + PackError::Unavailable => StatusCode::SERVICE_UNAVAILABLE, + PackError::PushOverSsh => StatusCode::FORBIDDEN, + PackError::BadPath(_) | PackError::UnsupportedService => StatusCode::BAD_REQUEST, + PackError::Protocol(_) | PackError::PktLine(_) => StatusCode::BAD_REQUEST, + PackError::UnsupportedEncoding(_) => StatusCode::UNSUPPORTED_MEDIA_TYPE, + PackError::LimitExceeded(_) => StatusCode::PAYLOAD_TOO_LARGE, + PackError::SelectionTooLarge + | PackError::SelectionTimeout + | PackError::InsufficientMemory => StatusCode::SERVICE_UNAVAILABLE, + PackError::Pack(_) | PackError::Git(_) => StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + +impl IntoResponse for PackError { + fn into_response(self) -> Response { + (self.http_status(), self.to_string()).into_response() + } +} diff --git a/knot2/crates/knot-pack/src/fetch.rs b/knot2/crates/knot-pack/src/fetch.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/src/fetch.rs @@ -0,0 +1,477 @@ +use std::io::Write; + +use axum::http::{HeaderMap, HeaderValue, Method, header}; +use knot_git::{Filter, RefRecord, Repo}; +use knot_runtime::{HttpRequest, HttpResponse, HttpTransport, NetworkError}; +use knot_types::{HttpStatus, Oid, RefName}; +use url::Url; + +use crate::error::PackError; +use crate::pkt::{self, Frame}; +use crate::{HaveOids, WantOids}; + +#[derive(Debug, thiserror::Error)] +pub enum FetchError { + #[error("upstream url: {0}")] + Url(String), + #[error("upstream network: {0}")] + Network(#[from] NetworkError), + #[error("upstream returned http status {0}")] + Status(HttpStatus), + #[error("upstream protocol: {0}")] + Protocol(String), + #[error("upstream reported: {0}")] + Remote(String), + #[error("fetched pack exceeds {limit} bytes")] + PackTooLarge { limit: u64 }, + #[error(transparent)] + Pack(#[from] PackError), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UpstreamRefs { + pub head_symref: Option, + pub refs: Vec, +} + +impl UpstreamRefs { + pub fn tips(&self) -> Vec { + let mut tips: Vec = self.refs.iter().map(|record| record.target).collect(); + tips.sort_unstable(); + tips.dedup(); + tips + } + + pub fn find(&self, name: &RefName) -> Option { + self.refs + .iter() + .find(|record| record.name == *name) + .map(|record| record.target) + } +} + +fn protocol(message: impl Into) -> FetchError { + FetchError::Protocol(message.into()) +} + +fn endpoint(base: &Url, suffix: &str) -> Result { + let trimmed = base.as_str().trim_end_matches('/'); + Url::parse(&format!("{trimmed}/{suffix}")).map_err(|error| FetchError::Url(error.to_string())) +} + +fn headers_v2(content_type: Option<&'static str>) -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert("git-protocol", HeaderValue::from_static("version=2")); + if let Some(content_type) = content_type { + headers.insert(header::CONTENT_TYPE, HeaderValue::from_static(content_type)); + } + headers +} + +async fn execute( + http: &dyn HttpTransport, + request: HttpRequest, +) -> Result { + let response = http.execute(request).await?; + if !response.status.is_success() { + return Err(FetchError::Status(HttpStatus::new( + response.status.as_u16(), + ))); + } + Ok(response) +} + +pub fn parse_advertisement(body: &[u8]) -> Result<(), FetchError> { + let lines = pkt::data_payloads_all(body).map_err(|error| protocol(error.to_string()))?; + let lines: Vec<&str> = lines + .iter() + .map(|line| std::str::from_utf8(line).unwrap_or_default().trim_end()) + .collect(); + let has = |name: &str| { + lines + .iter() + .any(|line| *line == name || line.starts_with(&format!("{name}="))) + }; + if !has("version 2") { + return Err(protocol("upstream doesn't speak git protocol v2")); + } + if !has("ls-refs") || !has("fetch") { + return Err(protocol("upstream is missing ls-refs or fetch v2 command")); + } + Ok(()) +} + +pub fn ls_refs_request(prefixes: &[&str]) -> Result, PackError> { + let mut buf = Vec::new(); + pkt::write_data(&mut buf, b"command=ls-refs\n")?; + pkt::write_data(&mut buf, b"agent=knot/0\n")?; + pkt::write_delim(&mut buf)?; + pkt::write_data(&mut buf, b"symrefs\n")?; + prefixes.iter().try_for_each(|prefix| { + pkt::write_data(&mut buf, format!("ref-prefix {prefix}\n").as_bytes()) + })?; + pkt::write_flush(&mut buf)?; + Ok(buf) +} + +pub fn parse_ls_refs(body: &[u8]) -> Result { + let lines = pkt::data_payloads(body).map_err(|error| protocol(error.to_string()))?; + lines.iter().try_fold( + UpstreamRefs { + head_symref: None, + refs: Vec::new(), + }, + |mut refs, line| { + let text = std::str::from_utf8(line) + .map_err(|_| protocol("ref line isn't utf-8"))? + .trim_end(); + if let Some(message) = text.strip_prefix("ERR ") { + return Err(FetchError::Remote(message.to_string())); + } + let (oid, rest) = text + .split_once(' ') + .ok_or_else(|| protocol(format!("malformed ref line: {text}")))?; + let target = + Oid::from_hex(oid).map_err(|_| protocol(format!("malformed ref oid: {oid}")))?; + let mut attributes = rest.split(' '); + match attributes.next() { + Some("HEAD") => { + refs.head_symref = attributes + .find_map(|attribute| attribute.strip_prefix("symref-target:")) + .and_then(|symref| RefName::new(symref).ok()); + } + Some(name) => { + if let Ok(name) = RefName::new(name) { + refs.refs.push(RefRecord { name, target }); + } + } + None => return Err(protocol(format!("malformed ref line: {text}"))), + } + Ok(refs) + }, + ) +} + +pub fn fetch_request(wants: &WantOids, haves: &HaveOids) -> Result, PackError> { + let mut buf = Vec::new(); + pkt::write_data(&mut buf, b"command=fetch\n")?; + pkt::write_data(&mut buf, b"agent=knot/0\n")?; + pkt::write_delim(&mut buf)?; + pkt::write_data(&mut buf, b"no-progress\n")?; + pkt::write_data(&mut buf, b"ofs-delta\n")?; + wants + .iter() + .try_for_each(|want| pkt::write_data(&mut buf, format!("want {want}\n").as_bytes()))?; + haves + .iter() + .try_for_each(|have| pkt::write_data(&mut buf, format!("have {have}\n").as_bytes()))?; + pkt::write_data(&mut buf, b"done\n")?; + pkt::write_flush(&mut buf)?; + Ok(buf) +} + +pub fn parse_fetch_response(body: &[u8], max_pack_bytes: u64) -> Result, FetchError> { + let (pack, in_packfile) = pkt::frames(body, None) + .map(|frame| frame.map_err(|error| protocol(error.to_string()))) + .try_fold( + (Vec::new(), false), + |(mut pack, in_packfile), frame| match (frame?.0, in_packfile) { + (Frame::Data(payload), false) => { + if let Some(message) = payload + .strip_prefix(b"ERR ".as_slice()) + .map(|rest| String::from_utf8_lossy(rest).trim_end().to_string()) + { + return Err(FetchError::Remote(message)); + } + let entered = + payload.strip_suffix(b"\n".as_slice()).unwrap_or(payload) == b"packfile"; + Ok((pack, entered)) + } + (Frame::Data(payload), true) => match payload.split_first() { + Some((1, data)) => { + if pack.len() as u64 + data.len() as u64 > max_pack_bytes { + return Err(FetchError::PackTooLarge { + limit: max_pack_bytes, + }); + } + pack.extend_from_slice(data); + Ok((pack, true)) + } + Some((2, _)) => Ok((pack, true)), + Some((3, message)) => Err(FetchError::Remote( + String::from_utf8_lossy(message).trim_end().to_string(), + )), + _ => Err(protocol("empty sideband frame in packfile section")), + }, + (_, in_packfile) => Ok((pack, in_packfile)), + }, + )?; + if !in_packfile { + return Err(protocol("upstream response has no packfile section")); + } + Ok(pack) +} + +pub async fn remote_refs( + http: &dyn HttpTransport, + base: &Url, + prefixes: &[&str], +) -> Result { + let advertise = endpoint(base, "info/refs?service=git-upload-pack")?; + let response = execute( + http, + HttpRequest { + method: Method::GET, + url: advertise, + headers: headers_v2(None), + body: None, + }, + ) + .await?; + parse_advertisement(&response.body)?; + + let upload = endpoint(base, "git-upload-pack")?; + let response = execute( + http, + HttpRequest { + method: Method::POST, + url: upload, + headers: headers_v2(Some("application/x-git-upload-pack-request")), + body: Some(ls_refs_request(prefixes)?.into()), + }, + ) + .await?; + parse_ls_refs(&response.body) +} + +pub async fn remote_pack( + http: &dyn HttpTransport, + base: &Url, + wants: &WantOids, + haves: &HaveOids, + max_pack_bytes: u64, +) -> Result, FetchError> { + if wants.is_empty() { + return Ok(Vec::new()); + } + let upload = endpoint(base, "git-upload-pack")?; + let response = execute( + http, + HttpRequest { + method: Method::POST, + url: upload, + headers: headers_v2(Some("application/x-git-upload-pack-request")), + body: Some(fetch_request(wants, haves)?.into()), + }, + ) + .await?; + parse_fetch_response(&response.body, max_pack_bytes) +} + +pub fn local_refs(source: &Repo, prefixes: &[&str]) -> Result { + let refs = source + .advertised_refs() + .map_err(PackError::from)? + .iter() + .filter(|record| crate::upload::matches_prefix(record.name.as_str(), prefixes)) + .cloned() + .collect(); + Ok(UpstreamRefs { + head_symref: source.head().map(|head| head.name), + refs, + }) +} + +struct BoundedPack { + buf: Vec, + limit: u64, + overflowed: bool, +} + +impl Write for BoundedPack { + fn write(&mut self, data: &[u8]) -> std::io::Result { + if self.buf.len() as u64 + data.len() as u64 > self.limit { + self.overflowed = true; + return Err(std::io::Error::other("pack byte limit exceeded")); + } + self.buf.extend_from_slice(data); + Ok(data.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +pub fn local_pack( + source: &Repo, + wants: &WantOids, + haves: &HaveOids, + max_pack_bytes: u64, +) -> Result, FetchError> { + if wants.is_empty() { + return Ok(Vec::new()); + } + let oids = source + .select_pack_objects_filtered( + wants.wants(), + haves.haves(), + Filter::None, + crate::upload::selection_budget(), + ) + .map_err(PackError::from)? + .send; + let mut out = BoundedPack { + buf: Vec::new(), + limit: max_pack_bytes, + overflowed: false, + }; + match crate::objects::write_pack( + &source.objects_dir(), + oids, + None, + &mut out, + source.object_format().kind(), + ) { + Ok(()) => Ok(out.buf), + Err(_) if out.overflowed => Err(FetchError::PackTooLarge { + limit: max_pack_bytes, + }), + Err(error) => Err(FetchError::Pack(error)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn data(buf: &mut Vec, line: &[u8]) { + pkt::write_data(buf, line).unwrap(); + } + + #[test] + fn the_v2_advertisement_is_accepted_and_v0_is_refused() { + let scan = tempfile::tempdir().unwrap(); + let repo = knot_git::Layout::new(scan.path()) + .create(&knot_types::RepoDid::new("did:plc:squid").unwrap()) + .unwrap(); + let v2 = crate::upload::advertise(&repo).unwrap(); + assert!(parse_advertisement(&v2).is_ok()); + + let mut v0 = Vec::new(); + data(&mut v0, b"# service=git-upload-pack\n"); + pkt::write_flush(&mut v0).unwrap(); + data( + &mut v0, + b"95d09f2b10159347eece71399a7e2e907ea3df4f HEAD\0side-band-64k\n", + ); + pkt::write_flush(&mut v0).unwrap(); + assert!(matches!( + parse_advertisement(&v0), + Err(FetchError::Protocol(_)) + )); + } + + #[test] + fn ls_refs_lines_parse_with_symref_and_skip_head() { + let mut body = Vec::new(); + data( + &mut body, + b"95d09f2b10159347eece71399a7e2e907ea3df4f HEAD symref-target:refs/heads/main\n", + ); + data( + &mut body, + b"95d09f2b10159347eece71399a7e2e907ea3df4f refs/heads/main\n", + ); + pkt::write_flush(&mut body).unwrap(); + let refs = parse_ls_refs(&body).unwrap(); + assert_eq!( + refs.head_symref.as_ref().map(RefName::as_str), + Some("refs/heads/main") + ); + assert_eq!(refs.refs.len(), 1); + assert_eq!(refs.refs[0].name.as_str(), "refs/heads/main"); + assert_eq!(refs.tips().len(), 1); + } + + #[test] + fn a_malformed_ref_oid_is_a_protocol_error() { + let mut body = Vec::new(); + data(&mut body, b"zzzz refs/heads/main\n"); + pkt::write_flush(&mut body).unwrap(); + assert!(matches!(parse_ls_refs(&body), Err(FetchError::Protocol(_)))); + } + + #[test] + fn an_err_line_is_surfaced_as_remote() { + let mut body = Vec::new(); + data(&mut body, b"ERR access denied\n"); + pkt::write_flush(&mut body).unwrap(); + assert!(matches!( + parse_ls_refs(&body), + Err(FetchError::Remote(message)) if message == "access denied" + )); + } + + #[test] + fn the_packfile_section_demuxes_data_and_drops_progress() { + let mut body = Vec::new(); + data(&mut body, b"packfile\n"); + data(&mut body, b"\x01PACKDATA"); + data(&mut body, b"\x02counting objects\n"); + data(&mut body, b"\x01MORE"); + pkt::write_flush(&mut body).unwrap(); + let pack = parse_fetch_response(&body, 1024).unwrap(); + assert_eq!(pack, b"PACKDATAMORE"); + } + + #[test] + fn a_sideband_error_band_is_remote_and_the_limit_holds() { + let mut body = Vec::new(); + data(&mut body, b"packfile\n"); + data(&mut body, b"\x03out of disk\n"); + pkt::write_flush(&mut body).unwrap(); + assert!(matches!( + parse_fetch_response(&body, 1024), + Err(FetchError::Remote(message)) if message == "out of disk" + )); + + let mut big = Vec::new(); + data(&mut big, b"packfile\n"); + data(&mut big, b"\x01PACKDATA"); + pkt::write_flush(&mut big).unwrap(); + assert!(matches!( + parse_fetch_response(&big, 4), + Err(FetchError::PackTooLarge { limit: 4 }) + )); + } + + #[test] + fn a_response_without_a_packfile_section_is_refused() { + let mut body = Vec::new(); + data(&mut body, b"acknowledgments\n"); + data(&mut body, b"NAK\n"); + pkt::write_flush(&mut body).unwrap(); + assert!(matches!( + parse_fetch_response(&body, 1024), + Err(FetchError::Protocol(_)) + )); + } + + #[test] + fn the_fetch_request_includes_wants_haves_and_done() { + let want = Oid::from_hex("95d09f2b10159347eece71399a7e2e907ea3df4f").unwrap(); + let have = Oid::from_hex("2222222222222222222222222222222222222222").unwrap(); + let body = fetch_request(&WantOids::new(vec![want]), &HaveOids::new(vec![have])).unwrap(); + let lines = pkt::data_payloads_all(&body).unwrap(); + let text: Vec<&str> = lines + .iter() + .map(|line| std::str::from_utf8(line).unwrap().trim_end()) + .collect(); + assert!(text.contains(&"command=fetch")); + assert!(text.contains(&"want 95d09f2b10159347eece71399a7e2e907ea3df4f")); + assert!(text.contains(&"have 2222222222222222222222222222222222222222")); + assert!(text.contains(&"done")); + assert!(text.contains(&"no-progress")); + } +} diff --git a/knot2/crates/knot-pack/src/frame.rs b/knot2/crates/knot-pack/src/frame.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/src/frame.rs @@ -0,0 +1,455 @@ +use std::collections::HashMap; + +use flate2::{Decompress, FlushDecompress, Status}; +use gix_pack::data::{Entry, entry::Header}; + +use knot_types::ObjectCount; + +use crate::error::{PackError, PackLimit}; +use crate::ids::PackOffset; +use crate::meter::{PackLimits, check_depth, malformed, pack_object_count}; +use crate::pkt::{self, Frame}; + +const HEADER_SLACK: usize = 64; + +fn pack_start(buf: &[u8]) -> Option { + let caps = pkt::first_command(buf) + .map(pkt::parse_caps) + .unwrap_or_default(); + let boundary = if caps.push_options { 2 } else { 1 }; + pkt::frames(buf, Some(boundary)) + .filter_map(|item| match item { + Ok((Frame::Flush, at)) => Some(at), + _ => None, + }) + .nth(boundary - 1) +} + +fn new_oid_field(line: &[u8]) -> Option<&[u8]> { + let line = line.split(|byte| *byte == 0).next().unwrap_or(line); + line.split(|byte| *byte == b' ').nth(1) +} + +fn no_pack_needed(buf: &[u8]) -> bool { + pkt::frames(buf, Some(1)) + .filter_map(|item| match item { + Ok((Frame::Data(payload), _)) => Some(payload), + _ => None, + }) + .all(|line| { + new_oid_field(line) + .map(|oid| oid.iter().all(|byte| *byte == b'0')) + .unwrap_or(false) + }) +} + +const PREAMBLE_SCAN_LIMIT: usize = 16 * 1024 * 1024; + +trait PackSource { + fn len(&self) -> u64; + fn read_at(&self, offset: u64, buf: &mut [u8]) -> std::io::Result; +} + +impl PackSource for [u8] { + fn len(&self) -> u64 { + <[u8]>::len(self) as u64 + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> std::io::Result { + let start = usize::try_from(offset) + .unwrap_or(usize::MAX) + .min(<[u8]>::len(self)); + let read = (<[u8]>::len(self) - start).min(buf.len()); + buf[..read].copy_from_slice(&self[start..start + read]); + Ok(read) + } +} + +struct FileSource<'a> { + file: &'a std::fs::File, + len: u64, +} + +impl PackSource for FileSource<'_> { + fn len(&self) -> u64 { + self.len + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> std::io::Result { + use std::os::unix::fs::FileExt; + let read = self.len.saturating_sub(offset).min(buf.len() as u64) as usize; + self.file.read_exact_at(&mut buf[..read], offset)?; + Ok(read) + } +} + +fn read_head(source: &S, upto: u64) -> Result, PackError> { + let limit = upto.min(PREAMBLE_SCAN_LIMIT as u64) as usize; + let mut head = vec![0u8; limit]; + let read = source + .read_at(0, &mut head) + .map_err(|error| PackError::Pack(format!("pack read: {error}")))?; + head.truncate(read); + Ok(head) +} + +struct EntryInflate { + data_offset: PackOffset, + decompressed_size: u64, + decompress: Decompress, + produced: u64, +} + +impl EntryInflate { + fn feed( + &mut self, + pack: &S, + scratch: &mut [u8], + chunk: &mut [u8], + ) -> Result, PackError> { + loop { + let consumed = self.decompress.total_in(); + let out_before = self.decompress.total_out(); + let read = pack + .read_at(self.data_offset.get() + consumed, chunk) + .map_err(|error| PackError::Pack(format!("pack read: {error}")))?; + let status = self + .decompress + .decompress(&chunk[..read], scratch, FlushDecompress::None) + .map_err(|error| PackError::Pack(format!("inflate: {error}")))?; + self.produced += self.decompress.total_out() - out_before; + if self.produced > self.decompressed_size { + return Err(malformed("object inflates beyond its declared size")); + } + match status { + Status::StreamEnd => { + return if self.produced == self.decompressed_size { + self.data_offset + .get() + .checked_add(self.decompress.total_in()) + .map(|offset| Some(PackOffset::new(offset))) + .ok_or_else(|| malformed("pack offset overflow")) + } else { + Err(malformed("object decompressed size mismatch")) + }; + } + Status::Ok | Status::BufError => { + if self.decompress.total_in() == consumed + && self.decompress.total_out() == out_before + { + return Ok(None); + } + } + } + } + } +} + +#[derive(Default)] +struct PackProgress { + num_objects: ObjectCount, + objects_done: ObjectCount, + next_offset: PackOffset, + total_decompressed: u64, + base_of: HashMap, + current: Option, + depth_checked: bool, +} + +impl PackProgress { + fn scan( + &mut self, + pack: &S, + limits: &PackLimits, + kind: gix::hash::Kind, + ) -> Result, PackError> { + let hash_len = kind.len_in_bytes(); + let len = pack.len(); + let mut scratch = [0u8; 8192]; + let mut chunk = [0u8; 8192]; + let mut header = [0u8; HEADER_SLACK]; + loop { + if self.objects_done == self.num_objects { + if !self.depth_checked { + check_depth(&self.base_of, limits.max_delta_depth)?; + self.depth_checked = true; + } + let total_len = (self.next_offset.get() as usize) + .checked_add(hash_len) + .ok_or_else(|| malformed("pack length overflow"))?; + return Ok((len as usize >= total_len).then_some(total_len)); + } + match self.current.as_mut() { + Some(entry) => match entry.feed(pack, &mut scratch, &mut chunk)? { + Some(next_offset) => { + self.next_offset = next_offset; + self.objects_done = self.objects_done.succ(); + self.current = None; + } + None => return Ok(None), + }, + None => { + let start = self.next_offset; + if start.get() >= len { + return Ok(None); + } + let read = pack + .read_at(start.get(), &mut header) + .map_err(|error| PackError::Pack(format!("pack read: {error}")))?; + let mut reader: &[u8] = &header[..read]; + let entry = match Entry::from_read(&mut reader, start.get(), hash_len) { + Ok(entry) => entry, + Err(error) => { + return if len.saturating_sub(start.get()) < HEADER_SLACK as u64 { + Ok(None) + } else { + Err(PackError::Pack(error.to_string())) + }; + } + }; + if limits.max_object_bytes.exceeded_by(entry.decompressed_size) { + return Err(PackError::LimitExceeded(PackLimit::ObjectBytes)); + } + self.total_decompressed = self + .total_decompressed + .checked_add(entry.decompressed_size) + .ok_or_else(|| malformed("decompressed size overflow"))?; + if limits.max_total_bytes.exceeded_by(self.total_decompressed) { + return Err(PackError::LimitExceeded(PackLimit::TotalBytes)); + } + if let Header::OfsDelta { base_distance } = entry.header { + let base = entry + .checked_base_pack_offset(base_distance) + .ok_or_else(|| malformed("ofs-delta base out of range"))?; + self.base_of.insert(self.next_offset, PackOffset::new(base)); + } + self.current = Some(EntryInflate { + data_offset: PackOffset::new(entry.data_offset), + decompressed_size: entry.decompressed_size, + decompress: Decompress::new(true), + produced: 0, + }); + } + } + } + } +} + +pub struct ReceiveFramer { + limits: PackLimits, + kind: gix::hash::Kind, + pack_start: Option, + pack: Option, +} + +impl ReceiveFramer { + pub fn new(limits: PackLimits, kind: gix::hash::Kind) -> Self { + Self { + limits, + kind, + pack_start: None, + pack: None, + } + } + + pub fn pack_start(&self) -> Option { + self.pack_start + } + + pub fn advance_bytes(&mut self, buf: &[u8]) -> Result, PackError> { + self.advance(buf) + } + + pub fn advance_file( + &mut self, + file: &std::fs::File, + len: u64, + ) -> Result, PackError> { + self.advance(&FileSource { file, len }) + } + + fn advance(&mut self, source: &S) -> Result, PackError> { + let len = source.len(); + let pack_start = match self.pack_start { + Some(start) => start, + None => { + let head = read_head(source, len)?; + match pack_start(&head) { + Some(start) => { + self.pack_start = Some(start); + start + } + None => return Ok(None), + } + } + }; + if self.pack.is_none() { + let pack_len = len.saturating_sub(pack_start as u64); + if pack_len == 0 { + let head = read_head(source, pack_start as u64)?; + return Ok(no_pack_needed(&head).then_some(pack_start)); + } + if pack_len < 12 { + return Ok(None); + } + let mut header = [0u8; 12]; + source + .read_at(pack_start as u64, &mut header) + .map_err(|error| PackError::Pack(format!("pack read: {error}")))?; + if &header[..4] != b"PACK" { + return Err(malformed("packfile is missing its PACK signature")); + } + let num_objects = pack_object_count(&header)?; + if num_objects > self.limits.max_objects { + return Err(PackError::LimitExceeded(PackLimit::Objects)); + } + self.pack = Some(PackProgress { + num_objects, + next_offset: PackOffset::new(pack_start as u64 + 12), + ..PackProgress::default() + }); + } + let kind = self.kind; + self.pack + .as_mut() + .expect("pack progress initialized") + .scan(source, &self.limits, kind) + } +} + +pub fn receive_request_complete( + buf: &[u8], + limits: &PackLimits, + kind: gix::hash::Kind, +) -> Result, PackError> { + ReceiveFramer::new(*limits, kind).advance_bytes(buf) +} + +pub fn archive_request_complete(buf: &[u8]) -> Option { + pkt::frames(buf, Some(1)).find_map(|item| match item { + Ok((Frame::Flush, end)) => Some(end), + _ => None, + }) +} + +#[derive(Default)] +pub struct UploadFramer { + scanned: usize, + v2: bool, + flushes: usize, + complete: Option, +} + +impl UploadFramer { + pub fn new() -> Self { + Self::default() + } + + pub fn advance(&mut self, buf: &[u8]) -> Option { + if self.complete.is_some() { + return self.complete; + } + let base = self.scanned; + for item in pkt::frames(&buf[base..], None) { + let Ok((frame, at)) = item else { break }; + let boundary = base + at; + match frame { + Frame::Data(payload) => { + if payload.starts_with(b"command=") { + self.v2 = true; + } + let trimmed = payload + .iter() + .rposition(|byte| !byte.is_ascii_whitespace()) + .map(|end| &payload[..=end]) + .unwrap_or(payload); + if !self.v2 && trimmed == b"done" { + self.complete = Some(boundary); + return self.complete; + } + } + Frame::Flush => { + self.flushes += 1; + if self.v2 { + self.complete = Some(boundary); + return self.complete; + } + } + _ => {} + } + self.scanned = boundary; + } + None + } + + pub fn unanswered_flushes(&self) -> usize { + self.flushes.saturating_sub(1) + } +} + +pub fn upload_v0_nak() -> Vec { + let mut buf = Vec::new(); + pkt::write_data(&mut buf, b"NAK\n").expect("write to in-memory buffer never fails"); + buf +} + +#[cfg(test)] +mod tests { + use super::*; + + fn v2_fetch_request() -> Vec { + let mut buf = Vec::new(); + pkt::write_data(&mut buf, b"command=fetch\n").unwrap(); + pkt::write_delim(&mut buf).unwrap(); + pkt::write_data(&mut buf, b"want 1111111111111111111111111111111111111111\n").unwrap(); + pkt::write_data(&mut buf, b"want 2222222222222222222222222222222222222222\n").unwrap(); + pkt::write_data(&mut buf, b"done\n").unwrap(); + pkt::write_flush(&mut buf).unwrap(); + buf + } + + #[test] + fn upload_framer_completes_a_v2_request_at_the_terminating_flush() { + let request = v2_fetch_request(); + assert_eq!(UploadFramer::new().advance(&request), Some(request.len())); + } + + #[test] + fn upload_framer_fed_one_byte_at_a_time_never_overruns_the_buffer() { + let request = v2_fetch_request(); + let mut framer = UploadFramer::new(); + let mut buf = Vec::new(); + let mut completed = None; + for byte in &request { + buf.push(*byte); + if let Some(len) = framer.advance(&buf) { + assert!( + len <= buf.len(), + "advance returned {len} past buffer of {}", + buf.len() + ); + completed = Some(len); + break; + } + } + assert_eq!( + completed, + Some(request.len()), + "incrementally fed request completes exactly once the whole buffer has arrived" + ); + } + + #[test] + fn upload_framer_counts_v0_have_batch_flushes_without_completing() { + let mut buf = Vec::new(); + pkt::write_data(&mut buf, b"want 1111111111111111111111111111111111111111\n").unwrap(); + pkt::write_flush(&mut buf).unwrap(); + pkt::write_data(&mut buf, b"have 2222222222222222222222222222222222222222\n").unwrap(); + pkt::write_flush(&mut buf).unwrap(); + let mut framer = UploadFramer::new(); + assert_eq!(framer.advance(&buf), None, "v0 request is open until done"); + assert_eq!( + framer.unanswered_flushes(), + 1, + "two flushes seen, one have-batch awaits NAK" + ); + } +} diff --git a/knot2/crates/knot-pack/src/guard.rs b/knot2/crates/knot-pack/src/guard.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/src/guard.rs @@ -0,0 +1,58 @@ +use std::sync::Arc; + +use knot_cob::{CobHome, CobStore}; +use knot_git::Repo; +use knot_messages::{Catalog, ErrorKey}; +use knot_types::{ActorId, RefName}; + +use crate::{ReceiveCommand, ReceiveGuard, RefDecision}; + +pub struct PushGuard { + pub cob_authority: ActorId, + pub home: CobHome, + pub messages: Arc, +} + +impl ReceiveGuard for PushGuard { + fn authorize(&self, staged: &Repo, commands: &[ReceiveCommand]) -> Vec { + commands + .iter() + .map(|command| self.decide(staged, command)) + .collect() + } +} + +impl PushGuard { + fn decide(&self, staged: &Repo, command: &ReceiveCommand) -> RefDecision { + match command.name() { + None => RefDecision::Reject( + self.messages + .reject + .cob_verification + .line(|ErrorKey::Error| crate::receive::invalid_refname(command.refname())), + ), + Some(name) if knot_git::is_public_ref(name) => RefDecision::Allow, + Some(name) if knot_git::is_reserved(name) => { + if command.is_delete() { + RefDecision::Reject(self.messages.reject.cob_delete.text()) + } else { + self.verify_cob(staged, name) + } + } + Some(_) => RefDecision::Reject(self.messages.reject.hidden_reserved.text()), + } + } + + fn verify_cob(&self, staged: &Repo, name: &RefName) -> RefDecision { + let store = CobStore::new(staged); + match knot_cobs::verify_cob_ref(&store, &self.home, name, &self.cob_authority) { + Ok(_) => RefDecision::Allow, + Err(error) => RefDecision::Reject( + self.messages + .reject + .cob_verification + .line(|ErrorKey::Error| error.to_string()), + ), + } + } +} diff --git a/knot2/crates/knot-pack/src/ids.rs b/knot2/crates/knot-pack/src/ids.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/src/ids.rs @@ -0,0 +1,108 @@ +knot_types::scalar_newtype! { + pub(crate) struct Crc32(u32); + pub struct MaxWireBytes(usize); +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] +pub(crate) struct PackOffset(u64); + +impl PackOffset { + pub(crate) fn new(value: u64) -> Self { + Self(value) + } + + pub(crate) fn get(self) -> u64 { + self.0 + } + + // Hostile packs will ask to go back past the start of a file, + // so just making sure. + pub(crate) fn checked_sub_distance(self, distance: u64) -> Option { + self.0.checked_sub(distance).map(Self) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct DeltaDepth(usize); + +impl DeltaDepth { + pub(crate) const ZERO: Self = Self(0); + + pub const fn new(depth: usize) -> Self { + Self(depth) + } + + pub(crate) const fn get(self) -> usize { + self.0 + } + + pub(crate) fn deeper(self) -> Self { + Self(self.0 + 1) + } + + pub(crate) fn exceeds(self, max: DeltaDepth) -> bool { + self.0 > max.0 + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct MaxObjectBytes(u64); + +impl MaxObjectBytes { + pub const fn new(bytes: u64) -> Self { + Self(bytes) + } + + pub const fn get(self) -> u64 { + self.0 + } + + pub(crate) const fn exceeded_by(self, size: u64) -> bool { + size > self.0 + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct MaxTotalBytes(u64); + +impl MaxTotalBytes { + pub const fn new(bytes: u64) -> Self { + Self(bytes) + } + + pub const fn get(self) -> u64 { + self.0 + } + + pub(crate) const fn exceeded_by(self, size: u64) -> bool { + size > self.0 + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Rounds(usize); + +impl Rounds { + pub(crate) fn new(rounds: usize) -> Self { + Self(rounds) + } + + // Ensuring chains deeper than the limit return `None` instead of + // wrapping and doing like 18 quintillion more passes. + pub(crate) fn next(self) -> Option { + self.0.checked_sub(1).map(Self) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) struct RefsDigest([u8; 32]); + +impl RefsDigest { + pub(crate) fn new(bytes: [u8; 32]) -> Self { + Self(bytes) + } + + pub(crate) fn as_bytes(&self) -> &[u8] { + &self.0 + } +} diff --git a/knot2/crates/knot-pack/src/idxwrite.rs b/knot2/crates/knot-pack/src/idxwrite.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/src/idxwrite.rs @@ -0,0 +1,211 @@ +use std::io::{self, BufWriter, Write}; +use std::os::unix::fs::FileExt; +use std::sync::Mutex; + +use gix::ObjectId; + +use crate::error::PackError; +use crate::ids::{Crc32, PackOffset}; + +const V2_SIGNATURE: &[u8] = &[0xff, 0x74, 0x4f, 0x63]; +const V2_VERSION: u32 = 2; +const HIGH_BIT: u32 = 0x8000_0000; +const LARGE_OFFSET_THRESHOLD: u64 = 0x7fff_ffff; +const BUCKETS: usize = 256; +const BUCKET_BUF: usize = 64 * 1024; +const CRC_LEN: usize = 4; +const OFFSET_LEN: usize = 8; + +struct Record { + id: ObjectId, + crc32: Crc32, + offset: PackOffset, +} + +pub(crate) struct Spool { + buckets: Vec>>, + record_len: usize, + hash_len: usize, +} + +impl Spool { + pub(crate) fn new(kind: gix::hash::Kind) -> io::Result { + let hash_len = kind.len_in_bytes(); + let buckets = (0..BUCKETS) + .map(|_| { + tempfile::tempfile() + .map(|file| Mutex::new(BufWriter::with_capacity(BUCKET_BUF, file))) + }) + .collect::>>()?; + Ok(Self { + buckets, + record_len: hash_len + CRC_LEN + OFFSET_LEN, + hash_len, + }) + } + + pub(crate) fn push(&self, id: ObjectId, crc32: Crc32, offset: PackOffset) -> io::Result<()> { + let mut guard = self.buckets[id.first_byte() as usize] + .lock() + .expect("spool bucket poisoned"); + guard.write_all(id.as_slice())?; + guard.write_all(&crc32.get().to_be_bytes())?; + guard.write_all(&offset.get().to_be_bytes()) + } + + fn cumulative_fanout(&self) -> Result<[u32; 256], PackError> { + let mut fanout = [0u32; 256]; + self.buckets.iter().enumerate().try_for_each( + |(bucket, cell)| -> Result<(), PackError> { + let mut guard = cell.lock().expect("spool bucket poisoned"); + guard.flush()?; + let len = guard.get_ref().metadata()?.len() as usize; + fanout[bucket] = (len / self.record_len) as u32; + Ok(()) + }, + )?; + fanout.iter_mut().fold(0u32, |acc, count| { + *count += acc; + *count + }); + Ok(fanout) + } + + fn visit_sorted( + &self, + mut visit: impl FnMut(&Record) -> Result<(), PackError>, + ) -> Result<(), PackError> { + self.buckets.iter().try_for_each(|cell| { + let mut records = self.read_bucket(cell)?; + records.sort_unstable_by_key(|record| record.id); + records.iter().try_for_each(&mut visit) + }) + } + + fn read_bucket( + &self, + cell: &Mutex>, + ) -> Result, PackError> { + let mut guard = cell.lock().expect("spool bucket poisoned"); + guard.flush()?; + let file = guard.get_ref(); + let len = file.metadata()?.len() as usize; + let mut bytes = vec![0u8; len]; + file.read_exact_at(&mut bytes, 0)?; + drop(guard); + bytes + .chunks_exact(self.record_len) + .map(|chunk| { + let (id, rest) = chunk.split_at(self.hash_len); + Ok(Record { + id: ObjectId::try_from(id) + .map_err(|error| PackError::Pack(format!("spool record oid: {error}")))?, + crc32: Crc32::new(u32::from_be_bytes( + rest[..CRC_LEN].try_into().expect("crc slice"), + )), + offset: PackOffset::new(u64::from_be_bytes( + rest[CRC_LEN..].try_into().expect("offset slice"), + )), + }) + }) + .collect() + } +} + +fn feed(out: &mut dyn Write, hasher: &mut gix_hash::Hasher, buf: &[u8]) -> io::Result<()> { + hasher.update(buf); + out.write_all(buf) +} + +pub(crate) fn write_v2_index( + out: &mut dyn Write, + records: &Spool, + pack_hash: &ObjectId, + kind: gix::hash::Kind, +) -> Result { + let mut hasher = gix_hash::hasher(kind); + feed(out, &mut hasher, V2_SIGNATURE)?; + feed(out, &mut hasher, &V2_VERSION.to_be_bytes())?; + + records + .cumulative_fanout()? + .iter() + .try_for_each(|count| feed(out, &mut hasher, &count.to_be_bytes()))?; + records.visit_sorted(|record| Ok(feed(out, &mut hasher, record.id.as_slice())?))?; + records + .visit_sorted(|record| Ok(feed(out, &mut hasher, &record.crc32.get().to_be_bytes())?))?; + + let mut large_offsets = Vec::::new(); + records.visit_sorted(|record| { + let encoded = if record.offset.get() > LARGE_OFFSET_THRESHOLD { + let position = large_offsets.len() as u32; + large_offsets.push(record.offset.get()); + position | HIGH_BIT + } else { + record.offset.get() as u32 + }; + Ok(feed(out, &mut hasher, &encoded.to_be_bytes())?) + })?; + large_offsets + .iter() + .try_for_each(|offset| feed(out, &mut hasher, &offset.to_be_bytes()))?; + + feed(out, &mut hasher, pack_hash.as_slice())?; + + let index_hash = hasher + .try_finalize() + .map_err(|error| PackError::Pack(format!("finalize index hash: {error}")))?; + out.write_all(index_hash.as_slice())?; + out.flush()?; + Ok(index_hash) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn oid(seed: u8) -> ObjectId { + let mut raw = [0u8; 20]; + raw[0] = seed; + raw[19] = seed; + ObjectId::try_from(raw.as_slice()).unwrap() + } + + #[test] + fn large_offsets_round_trip_through_the_index_reader() { + let records = [ + (oid(0x02), 0x1111_1111u32, 12u64), + (oid(0x40), 0x2222_2222, LARGE_OFFSET_THRESHOLD), + (oid(0x80), 0x3333_3333, 0x1_2345_6789), + (oid(0xc0), 0x4444_4444, LARGE_OFFSET_THRESHOLD + 1), + ]; + let spool = Spool::new(gix::hash::Kind::Sha1).unwrap(); + records.iter().for_each(|(id, crc32, offset)| { + spool + .push(*id, Crc32::new(*crc32), PackOffset::new(*offset)) + .unwrap() + }); + let pack_hash = oid(0xaa); + + let mut buf = Vec::new(); + let index_hash = + write_v2_index(&mut buf, &spool, &pack_hash, gix::hash::Kind::Sha1).unwrap(); + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("pack-under-test.idx"); + std::fs::write(&path, &buf).unwrap(); + let index = gix_pack::index::File::at(&path, gix::hash::Kind::Sha1).unwrap(); + + assert_eq!(index.num_objects(), records.len() as u32); + assert_eq!(index.index_checksum(), index_hash); + assert_eq!(index.pack_checksum(), pack_hash); + records.iter().for_each(|(id, crc32, offset)| { + let at = index + .lookup(*id) + .expect("written oid resolves in the index"); + assert_eq!(index.oid_at_index(at), id.as_ref()); + assert_eq!(index.pack_offset_at_index(at), *offset); + assert_eq!(index.crc32_at_index(at), Some(*crc32)); + }); + } +} diff --git a/knot2/crates/knot-pack/src/lib.rs b/knot2/crates/knot-pack/src/lib.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/src/lib.rs @@ -0,0 +1,911 @@ +mod archive; +mod cache; +mod error; +mod fetch; +mod frame; +mod guard; +mod ids; +mod idxwrite; +mod meter; +mod objects; +mod oids; +mod pkt; +mod quarantine; +mod receive; +mod receiver; +mod resolve; +mod upload; + +use std::collections::HashMap; +use std::io::{self, Read}; + +use axum::Router; +use axum::body::{Body, Bytes}; +use axum::extract::{DefaultBodyLimit, Path, Query, State}; +use axum::http::{HeaderMap, header}; +use axum::response::Response; +use axum::routing::post; +use knot_git::{Layout, Repo}; +use knot_messages::{Catalog, ErrorKey, FetchMessages}; +use knot_resource::{PackSlots, SlotPermit}; +use knot_runtime::Clock; +use knot_types::{ + AccountDid, Handle, KnotHostname, OwnerDid, OwnerRef, ParseError, RepoDid, RepoRkey, +}; +use std::sync::Arc; +use tokio::sync::mpsc; +use tokio_stream::wrappers::ReceiverStream; + +pub use cache::{CacheConfig, MaxCacheBytes, MaxEntryBytes}; +pub use error::{PackError, PackLimit}; +pub use fetch::{FetchError, UpstreamRefs, local_pack, local_refs, remote_pack, remote_refs}; +pub use frame::{ + ReceiveFramer, UploadFramer, archive_request_complete, receive_request_complete, upload_v0_nak, +}; +pub use guard::PushGuard; +pub use ids::{DeltaDepth, MaxObjectBytes, MaxTotalBytes, MaxWireBytes}; +pub use meter::PackLimits; +pub use objects::{ExpandedPack, count_expanded, write_expanded, write_pack}; +pub use oids::{HaveOids, WantOids}; +pub use pkt::frame_report; +pub use quarantine::sweep_incoming; +pub use receive::{Preflight, ReceiveCommand, ReceiveGuard, ReceiveOutcome, RefDecision}; +pub use receiver::{PackReceiver, ReceiveReadError, ReceivedPack}; +pub use upload::{SelectionLimits, init_selection_limits, selection_budget}; + +use upload::UploadOutcome; + +pub use knot_messages::default_catalog; + +pub fn default_hostname() -> &'static KnotHostname { + static HOSTNAME: std::sync::LazyLock = std::sync::LazyLock::new(|| { + KnotHostname::new("knot.invalid").expect("static hostname parses") + }); + &HOSTNAME +} + +pub fn advertise_upload(repo: &Repo) -> Result, PackError> { + upload::advertise(repo) +} + +pub fn advertise_upload_v0(repo: &Repo) -> Result, PackError> { + upload::advertise_v0(repo) +} + +pub fn upload_pack(repo: &Repo, request: &[u8]) -> Result, PackError> { + upload::buffered(repo, request, &default_catalog().fetch, default_hostname()) +} + +pub fn upload_pack_streamed( + repo: &Repo, + request: &[u8], + messages: &FetchMessages, + knot: &KnotHostname, + sink: &mut dyn FnMut(&[u8]) -> io::Result<()>, +) -> Result<(), PackError> { + upload::streamed(repo, request, messages, knot, sink) +} + +pub fn advertise_receive(repo: &Repo) -> Result, PackError> { + receive::advertise(repo) +} + +pub fn advertise_upload_ssh(repo: &Repo) -> Result, PackError> { + upload::advertise_ssh(repo) +} + +pub fn advertise_upload_v0_ssh(repo: &Repo) -> Result, PackError> { + upload::advertise_v0_ssh(repo) +} + +pub fn advertise_receive_ssh(repo: &Repo) -> Result, PackError> { + receive::advertise_ssh(repo) +} + +#[doc(hidden)] +pub fn receive_pack(repo: &Repo, request: &[u8]) -> Result, PackError> { + receive::handle_bytes(repo, request, &PackLimits::default()) +} + +#[doc(hidden)] +pub fn receive_pack_with_limits( + repo: &Repo, + request: &[u8], + limits: &PackLimits, +) -> Result, PackError> { + receive::handle_bytes(repo, request, limits) +} + +#[doc(hidden)] +pub fn bench_ingest_fresh( + objects_dir: &std::path::Path, + pack_path: &std::path::Path, + kind: gix::hash::Kind, +) -> Result { + let pack = gix_pack::data::File::at(pack_path, kind) + .map_err(|error| PackError::Pack(error.to_string()))?; + objects::ingest_and_close( + objects_dir, + &pack, + &PackLimits::default(), + kind, + knot_resource::ingest_base_budget(), + false, + ) + .map(|closure| closure.is_some()) +} + +#[doc(hidden)] +pub fn bench_ingest_external( + objects_dir: &std::path::Path, + pack_path: &std::path::Path, + kind: gix::hash::Kind, +) -> Result, PackError> { + let pack = gix_pack::data::File::at(pack_path, kind) + .map_err(|error| PackError::Pack(error.to_string()))?; + objects::admit_ingest(&pack, kind)?; + objects::ingest_and_close( + objects_dir, + &pack, + &PackLimits::default(), + kind, + knot_resource::ingest_base_budget(), + true, + ) + .map(|closure| closure.map(|closure| closure.self_contained)) +} + +#[doc(hidden)] +pub fn bench_admit_and_ingest( + objects_dir: &std::path::Path, + pack_path: &std::path::Path, + kind: gix::hash::Kind, +) -> Result { + bench_ingest_with_base_budget( + objects_dir, + pack_path, + kind, + knot_resource::ingest_base_budget(), + ) +} + +#[doc(hidden)] +pub fn bench_ingest_with_base_budget( + objects_dir: &std::path::Path, + pack_path: &std::path::Path, + kind: gix::hash::Kind, + base_budget: Option, +) -> Result { + let pack = gix_pack::data::File::at(pack_path, kind) + .map_err(|error| PackError::Pack(error.to_string()))?; + objects::admit_ingest(&pack, kind)?; + objects::ingest_and_close( + objects_dir, + &pack, + &PackLimits::default(), + kind, + base_budget, + false, + ) + .map(|closure| closure.is_some()) +} + +pub fn receive_pack_guarded( + repo: &Repo, + request: &[u8], + limits: &PackLimits, + guard: &dyn ReceiveGuard, + seal: &dyn Fn(&[knot_git::RefUpdate]), + messages: &knot_messages::RejectMessages, +) -> Result { + receive::handle_guarded_bytes(repo, request, limits, guard, seal, messages) +} + +pub fn receive_pack_guarded_streamed( + repo: &Repo, + received: &ReceivedPack, + limits: &PackLimits, + guard: &dyn ReceiveGuard, + seal: &dyn Fn(&[knot_git::RefUpdate]), + messages: &knot_messages::RejectMessages, +) -> Result { + receive::handle_guarded_streamed(repo, received, limits, guard, seal, messages) +} + +pub fn receive_preflight(request: &[u8]) -> Preflight { + receive::preflight(request) +} + +pub fn upload_archive_streamed( + repo: &Repo, + request: &[u8], + sink: &mut dyn FnMut(&[u8]) -> io::Result<()>, +) -> Result<(), PackError> { + archive::stream(repo, request, sink) +} + +pub fn upload_archive(repo: &Repo, request: &[u8]) -> Result, PackError> { + let mut buf = Vec::new(); + upload_archive_streamed(repo, request, &mut |chunk| { + buf.extend_from_slice(chunk); + Ok(()) + })?; + Ok(buf) +} + +pub fn meter_pack( + pack: &[u8], + limits: &PackLimits, + kind: gix::hash::Kind, +) -> Result<(), PackError> { + meter::meter(pack, limits, kind) +} + +pub fn ingest_pack( + objects_dir: &std::path::Path, + pack: &[u8], + limits: &PackLimits, + kind: gix::hash::Kind, +) -> Result<(), PackError> { + objects::index_pack(objects_dir, pack, limits, kind) +} + +#[doc(hidden)] +pub mod fuzz { + pub fn pkt(data: &[u8]) { + let _ = crate::pkt::data_payloads(data); + let _ = crate::pkt::data_payloads_all(data); + let _ = crate::pkt::split_receive(data); + } + + pub fn pack(data: &[u8]) { + let _ = crate::meter::meter(data, &crate::PackLimits::default(), gix::hash::Kind::Sha1); + } + + pub fn receive_commands(data: &[u8]) { + crate::receive::fuzz(data); + } + + pub fn upload_args(data: &[u8]) { + crate::upload::fuzz(data); + } +} + +const MAX_REQUEST_BYTES: usize = 16 * 1024 * 1024; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RepoTarget { + Did(RepoDid), + OwnerRkey(OwnerDid, RepoRkey), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RepoLookup { + Hosted(RepoDid), + Unhosted, + Unavailable, +} + +impl RepoLookup { + pub fn or_else(self, next: impl FnOnce() -> RepoLookup) -> RepoLookup { + match self { + RepoLookup::Unhosted => next(), + decided => decided, + } + } + + pub fn first( + candidates: impl IntoIterator, + resolve: impl Fn(RepoRkey) -> RepoLookup, + ) -> RepoLookup { + candidates + .into_iter() + .fold(RepoLookup::Unhosted, |acc, rkey| { + acc.or_else(|| resolve(rkey)) + }) + } + + pub fn from_resolved( + resolved: knot_index::Resolved>, + found: impl FnOnce(T) -> RepoDid, + ) -> RepoLookup { + match resolved { + knot_index::Resolved::Ready(Some(value)) => RepoLookup::Hosted(found(value)), + knot_index::Resolved::Ready(None) => RepoLookup::Unhosted, + knot_index::Resolved::Warming => RepoLookup::Unavailable, + } + } +} + +pub trait RepoResolver: Send + Sync + 'static { + fn resolve(&self, target: &RepoTarget) -> RepoLookup; +} + +pub trait HandleResolver: Send + Sync + 'static { + fn resolve( + &self, + handle: Handle, + ) -> std::pin::Pin> + Send + '_>>; +} + +pub use knot_edge::SocketPeer; + +pub trait ReceiveAdvertiser: Send + Sync + 'static { + fn advertise( + &self, + repo: RepoDid, + peer: knot_edge::SocketPeer, + headers: HeaderMap, + ) -> std::pin::Pin + Send + '_>>; +} + +impl RepoResolver for F +where + F: Fn(&RepoTarget) -> RepoLookup + Send + Sync + 'static, +{ + fn resolve(&self, target: &RepoTarget) -> RepoLookup { + self(target) + } +} + +#[derive(Clone)] +struct PackState { + layout: Layout, + resolver: Arc, + receive: Option>, + handle_resolver: Option>, + pack_slots: PackSlots, + cache: Arc, + catalog: Arc, + hostname: KnotHostname, +} + +pub fn router(layout: Layout, resolver: Arc, clock: Arc) -> Router { + router_with_pack_slots( + layout, + resolver, + PackSlots::new(knot_resource::threads().get()), + clock, + ) +} + +pub fn router_with_pack_slots( + layout: Layout, + resolver: Arc, + pack_slots: PackSlots, + clock: Arc, +) -> Router { + let state = pack_state( + layout, + resolver, + None, + None, + pack_slots, + CacheConfig::default(), + Arc::new(Catalog::defaults()), + default_hostname().clone(), + clock, + ); + write_routes(state.clone()).merge(advertisement_routes(state).into_router()) +} + +#[allow(clippy::too_many_arguments)] +pub fn edge_routes( + layout: Layout, + resolver: Arc, + receive: Option>, + handle_resolver: Option>, + pack_slots: PackSlots, + cache: CacheConfig, + catalog: Arc, + hostname: KnotHostname, + clock: Arc, +) -> (Router, knot_edge::ZeroRttRoutes) { + let state = pack_state( + layout, + resolver, + receive, + handle_resolver, + pack_slots, + cache, + catalog, + hostname, + clock, + ); + (write_routes(state.clone()), advertisement_routes(state)) +} + +#[allow(clippy::too_many_arguments)] +fn pack_state( + layout: Layout, + resolver: Arc, + receive: Option>, + handle_resolver: Option>, + pack_slots: PackSlots, + cache: CacheConfig, + catalog: Arc, + hostname: KnotHostname, + clock: Arc, +) -> PackState { + PackState { + layout, + resolver, + receive, + handle_resolver, + pack_slots, + cache: cache::PackCache::new(cache, clock), + catalog, + hostname, + } +} + +fn write_routes(state: PackState) -> Router { + Router::new() + .route("/{did}/{name}/git-upload-pack", post(upload_named)) + .route( + "/{did}/{name}/git-upload-archive", + post(upload_archive_named), + ) + .route("/{did}/git-upload-pack", post(upload_did)) + .route("/{did}/git-upload-archive", post(upload_archive_did)) + .layer(DefaultBodyLimit::max(MAX_REQUEST_BYTES)) + .with_state(state) +} + +fn advertisement_routes(state: PackState) -> knot_edge::ZeroRttRoutes { + let named_state = state.clone(); + let did_state = state; + knot_edge::ZeroRttRoutes::new() + .get( + "/{did}/{name}/info/refs", + knot_edge::ZeroRttSafe::new( + move |Path((did, name)): Path<(String, String)>, + Query(query): Query>, + peer: knot_edge::SocketPeer, + headers: HeaderMap| { + let state = named_state.clone(); + async move { + let owner = resolve_owner(&state, &did).await?; + let repo_did = resolve_named_did(&state, &owner, &name)?; + read_advertisement( + &state, + repo_did, + query.get("service").map(String::as_str), + peer, + &headers, + ) + .await + } + }, + ), + ) + .get( + "/{did}/info/refs", + knot_edge::ZeroRttSafe::new( + move |Path(did): Path, + Query(query): Query>, + peer: knot_edge::SocketPeer, + headers: HeaderMap| { + let state = did_state.clone(); + async move { + let repo_did = resolve_did_did(&state, &did)?; + read_advertisement( + &state, + repo_did, + query.get("service").map(String::as_str), + peer, + &headers, + ) + .await + } + }, + ), + ) +} + +fn lookup_did(lookup: RepoLookup) -> Result { + match lookup { + RepoLookup::Hosted(did) => Ok(did), + RepoLookup::Unhosted => Err(PackError::NotFound), + RepoLookup::Unavailable => Err(PackError::Unavailable), + } +} + +fn bad_path(error: ParseError) -> PackError { + PackError::BadPath(error.to_string()) +} + +async fn resolve_owner(state: &PackState, owner: &str) -> Result { + match OwnerRef::parse(owner).ok_or(PackError::NotFound)? { + OwnerRef::Did(did) => Ok(did), + OwnerRef::Handle(handle) => { + let resolver = state.handle_resolver.as_ref().ok_or(PackError::NotFound)?; + let did = resolver.resolve(handle).await.ok_or(PackError::NotFound)?; + Ok(did.into()) + } + } +} + +fn resolve_named_did( + state: &PackState, + owner: &OwnerDid, + name: &str, +) -> Result { + lookup_did(RepoLookup::first( + RepoRkey::clone_path_candidates(name), + |rkey| { + state + .resolver + .resolve(&RepoTarget::OwnerRkey(owner.clone(), rkey)) + }, + )) +} + +fn resolve_did_did(state: &PackState, did: &str) -> Result { + let did = RepoDid::new(did).map_err(bad_path)?; + lookup_did(state.resolver.resolve(&RepoTarget::Did(did))) +} + +fn open_named(state: &PackState, owner: &OwnerDid, name: &str) -> Result { + let did = resolve_named_did(state, owner, name)?; + state.layout.open(&did).map_err(|_| PackError::NotFound) +} + +fn open_did(state: &PackState, did: &str) -> Result { + let did = resolve_did_did(state, did)?; + state.layout.open(&did).map_err(|_| PackError::NotFound) +} + +async fn read_advertisement( + state: &PackState, + repo_did: RepoDid, + service: Option<&str>, + peer: knot_edge::SocketPeer, + headers: &HeaderMap, +) -> Result { + match service { + Some("git-upload-pack") => { + let repo = state + .layout + .open(&repo_did) + .map_err(|_| PackError::NotFound)?; + let body = if wants_v2(headers) { + advertise_upload(&repo)? + } else { + upload::advertise_v0(&repo)? + }; + Ok(git_response( + "application/x-git-upload-pack-advertisement", + body, + )) + } + Some("git-receive-pack") => match &state.receive { + Some(advertiser) => Ok(advertiser.advertise(repo_did, peer, headers.clone()).await), + None => Err(PackError::PushOverSsh), + }, + _ => Err(PackError::UnsupportedService), + } +} + +fn decode_request(headers: &HeaderMap, body: Bytes) -> Result, PackError> { + let codings: Vec<&str> = headers + .get(header::CONTENT_ENCODING) + .and_then(|value| value.to_str().ok()) + .map(|value| { + value + .split(',') + .map(str::trim) + .filter(|token| !token.is_empty() && !token.eq_ignore_ascii_case("identity")) + .collect() + }) + .unwrap_or_default(); + match codings.as_slice() { + [] => Ok(body.to_vec()), + [token] if token.eq_ignore_ascii_case("gzip") => { + let mut out = Vec::new(); + flate2::read::GzDecoder::new(body.as_ref()) + .take(MAX_REQUEST_BYTES as u64 + 1) + .read_to_end(&mut out) + .map_err(|error| PackError::Protocol(format!("gzip request body: {error}")))?; + match out.len() > MAX_REQUEST_BYTES { + true => Err(PackError::LimitExceeded(PackLimit::TotalBytes)), + false => Ok(out), + } + } + unsupported => Err(PackError::UnsupportedEncoding(unsupported.join(", "))), + } +} + +fn wants_v2(headers: &HeaderMap) -> bool { + headers + .get("git-protocol") + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.split(':').any(|token| token.trim() == "version=2")) +} + +fn refs_digest(repo: &Repo) -> Option { + let refs = repo + .advertised_refs_for(knot_git::AdvertScope::Upload) + .ok()?; + let mut hasher = gix::hash::hasher(gix::hash::Kind::Sha256); + refs.iter().for_each(|record| { + hasher.update(record.name.as_str().as_bytes()); + hasher.update(b"\0"); + hasher.update(record.target.to_string().as_bytes()); + hasher.update(b"\n"); + }); + let id = hasher.try_finalize().ok()?; + let mut token = [0u8; 32]; + token.copy_from_slice(id.as_slice()); + Some(ids::RefsDigest::new(token)) +} + +const CACHE_RETRY_BUDGET: usize = 4; + +async fn upload_dispatch( + state: &PackState, + repo: Repo, + body: &[u8], +) -> Result { + upload_dispatch_within(state, repo, body, CACHE_RETRY_BUDGET).await +} + +async fn upload_dispatch_within( + state: &PackState, + repo: Repo, + body: &[u8], + retries: usize, +) -> Result { + let Some(token) = refs_digest(&repo) else { + return dispatch_plan(state, repo, body, None).await; + }; + let key = cache::RequestKey::new(&repo.objects_dir(), &token, body); + match state.cache.decide(key) { + cache::Decision::Serve(bytes) => Ok(cached_response(bytes)), + cache::Decision::Await(receiver) => match cache::wait(receiver).await { + cache::Resolved::Bytes(bytes) => Ok(cached_response(bytes)), + cache::Resolved::Retry => match retries { + 0 => dispatch_plan(state, repo, body, None).await, + _ => Box::pin(upload_dispatch_within(state, repo, body, retries - 1)).await, + }, + cache::Resolved::Regenerate => dispatch_plan(state, repo, body, None).await, + }, + cache::Decision::Lead(lease) => dispatch_plan(state, repo, body, Some(lease)).await, + cache::Decision::Stream | cache::Decision::Off => { + dispatch_plan(state, repo, body, None).await + } + } +} + +async fn dispatch_plan( + state: &PackState, + repo: Repo, + body: &[u8], + lease: Option, +) -> Result { + let permit = state.pack_slots.acquire().await; + let owned_body = body.to_vec(); + let planned = tokio::task::spawn_blocking(move || { + let outcome = upload::plan(&repo, &owned_body); + (repo, outcome) + }) + .await; + let (repo, plan) = match planned { + Ok((repo, Ok(plan))) => (repo, plan), + Ok((_repo, Err(error))) => { + if let Some(lease) = lease { + lease.regenerate(); + } + return Err(error); + } + Err(_join) => { + if let Some(lease) = lease { + lease.regenerate(); + } + return Err(PackError::Pack("upload planning task panicked".to_string())); + } + }; + match plan { + UploadOutcome::Buffered(bytes) => { + drop(permit); + if let Some(lease) = lease { + lease.regenerate(); + } + Ok(git_response("application/x-git-upload-pack-result", bytes)) + } + UploadOutcome::Streaming { + preamble, + wants, + haves, + opts, + } => Ok(stream_response( + repo, + preamble, + wants, + haves, + opts, + permit, + lease, + Arc::clone(&state.catalog), + state.hostname.clone(), + )), + } +} + +fn cached_response(bytes: Bytes) -> Response { + nocache( + Response::builder().header(header::CONTENT_TYPE, "application/x-git-upload-pack-result"), + ) + .body(Body::from(bytes)) + .expect("valid response") +} + +#[allow(clippy::too_many_arguments)] +fn stream_response( + repo: Repo, + preamble: Vec, + wants: WantOids, + haves: HaveOids, + opts: upload::StreamOpts, + permit: SlotPermit, + lease: Option, + catalog: Arc, + hostname: KnotHostname, +) -> Response { + let side_band = opts.side_band; + let limit = lease.as_ref().map(cache::Lease::max_entry_bytes); + let (tx, rx) = mpsc::channel::>(16); + tokio::task::spawn_blocking(move || { + let _permit = permit; + let capture = std::cell::RefCell::new(cache::Capture::new(limit)); + let mut emit = |chunk: &[u8]| -> io::Result<()> { + capture.borrow_mut().record(chunk); + tx.blocking_send(Ok(Bytes::copy_from_slice(chunk))) + .map_err(|_| io::Error::other("client disconnected")) + }; + if emit(&preamble).is_err() { + if let Some(lease) = lease { + lease.retry(); + } + return; + } + let result = upload::stream_pack( + &repo, + &wants, + &haves, + &opts, + &catalog.fetch, + &hostname, + &mut emit, + ); + match result { + Ok(()) => { + if side_band { + let mut flush = Vec::new(); + if pkt::write_flush(&mut flush).is_ok() { + let _ = emit(&flush); + } + } + if let Some(lease) = lease { + match capture.into_inner().into_bytes() { + Some(bytes) => lease.ready(Bytes::from(bytes)), + None => lease.too_large(), + } + } + } + Err(error) if side_band => { + let mut tail = Vec::new(); + let line = catalog + .fetch + .fatal + .line(|ErrorKey::Error| error.to_string().replace('\n', " ")); + let message = format!("{line}\n"); + if pkt::write_band_error(&mut tail, message.as_bytes()).is_ok() { + let _ = pkt::write_flush(&mut tail); + let _ = emit(&tail); + } + if let Some(lease) = lease { + lease.regenerate(); + } + } + Err(error) => { + let _ = tx.blocking_send(Err(io::Error::other(error.to_string()))); + if let Some(lease) = lease { + lease.regenerate(); + } + } + } + }); + nocache( + Response::builder().header(header::CONTENT_TYPE, "application/x-git-upload-pack-result"), + ) + .body(Body::from_stream(ReceiverStream::new(rx))) + .expect("valid response") +} + +async fn upload_named( + State(state): State, + Path((did, name)): Path<(String, String)>, + headers: HeaderMap, + body: Bytes, +) -> Result { + let owner = resolve_owner(&state, &did).await?; + let repo = open_named(&state, &owner, &name)?; + upload_dispatch(&state, repo, &decode_request(&headers, body)?).await +} + +async fn upload_did( + State(state): State, + Path(did): Path, + headers: HeaderMap, + body: Bytes, +) -> Result { + let repo = open_did(&state, &did)?; + upload_dispatch(&state, repo, &decode_request(&headers, body)?).await +} + +async fn archive_dispatch( + state: &PackState, + repo: Repo, + body: Vec, +) -> Result { + let permit = state.pack_slots.acquire().await; + Ok(archive_response(repo, body, permit)) +} + +fn archive_response(repo: Repo, body: Vec, permit: SlotPermit) -> Response { + let (tx, rx) = mpsc::channel::>(16); + tokio::task::spawn_blocking(move || { + let _permit = permit; + let mut sink = |chunk: &[u8]| -> io::Result<()> { + tx.blocking_send(Ok(Bytes::copy_from_slice(chunk))) + .map_err(|_| io::Error::other("client disconnected")) + }; + if let Err(error) = upload_archive_streamed(&repo, &body, &mut sink) { + let _ = tx.blocking_send(Err(io::Error::other(error.to_string()))); + } + }); + nocache(Response::builder().header( + header::CONTENT_TYPE, + "application/x-git-upload-archive-result", + )) + .body(Body::from_stream(ReceiverStream::new(rx))) + .expect("valid response") +} + +async fn upload_archive_named( + State(state): State, + Path((did, name)): Path<(String, String)>, + headers: HeaderMap, + body: Bytes, +) -> Result { + let owner = resolve_owner(&state, &did).await?; + let repo = open_named(&state, &owner, &name)?; + archive_dispatch(&state, repo, decode_request(&headers, body)?).await +} + +async fn upload_archive_did( + State(state): State, + Path(did): Path, + headers: HeaderMap, + body: Bytes, +) -> Result { + let repo = open_did(&state, &did)?; + archive_dispatch(&state, repo, decode_request(&headers, body)?).await +} + +fn nocache(builder: axum::http::response::Builder) -> axum::http::response::Builder { + builder + .header(header::EXPIRES, "Fri, 01 Jan 1980 00:00:00 GMT") + .header(header::PRAGMA, "no-cache") + .header( + header::CACHE_CONTROL, + "no-cache, max-age=0, must-revalidate", + ) +} + +fn git_response(content_type: &'static str, body: Vec) -> Response { + nocache(Response::builder().header(header::CONTENT_TYPE, content_type)) + .body(Body::from(body)) + .expect("valid response") +} diff --git a/knot2/crates/knot-pack/src/meter.rs b/knot2/crates/knot-pack/src/meter.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/src/meter.rs @@ -0,0 +1,257 @@ +use std::collections::HashMap; +use std::io::{self, Write}; + +use flate2::{Decompress, FlushDecompress, Status}; +use gix_pack::data::input; +use gix_pack::data::{entry::Header, header}; + +use knot_types::ObjectCount; + +use crate::error::{PackError, PackLimit}; +use crate::ids::{DeltaDepth, MaxObjectBytes, MaxTotalBytes, PackOffset}; + +const MAX_OBJECTS: ObjectCount = ObjectCount::new(16_000_000); +const MAX_OBJECT_BYTES: MaxObjectBytes = MaxObjectBytes::new(512 * 1024 * 1024); +const MAX_TOTAL_BYTES: MaxTotalBytes = MaxTotalBytes::new(16 * 1024 * 1024 * 1024); +const MAX_DELTA_DEPTH: DeltaDepth = DeltaDepth::new(50); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PackLimits { + pub max_objects: ObjectCount, + pub max_object_bytes: MaxObjectBytes, + pub max_total_bytes: MaxTotalBytes, + pub max_delta_depth: DeltaDepth, +} + +impl Default for PackLimits { + fn default() -> Self { + Self { + max_objects: MAX_OBJECTS, + max_object_bytes: MAX_OBJECT_BYTES, + max_total_bytes: MAX_TOTAL_BYTES, + max_delta_depth: MAX_DELTA_DEPTH, + } + } +} + +pub(crate) fn malformed(message: &str) -> PackError { + PackError::Pack(message.to_string()) +} + +pub(crate) fn pack_object_count(pack: &[u8]) -> Result { + let head: [u8; 12] = pack + .get(..12) + .and_then(|slice| slice.try_into().ok()) + .ok_or_else(|| malformed("packfile header is truncated"))?; + header::decode(&head) + .map(|(_version, num_objects)| ObjectCount::from(num_objects)) + .map_err(|error| PackError::Pack(error.to_string())) +} + +pub fn meter(pack: &[u8], limits: &PackLimits, kind: gix::hash::Kind) -> Result<(), PackError> { + if pack.len() < 12 + kind.len_in_bytes() { + return Err(malformed("packfile is truncated")); + } + meter_entries( + io::Cursor::new(pack), + pack_object_count(pack)?, + limits, + kind, + ) + .map(|_| ()) +} + +pub(crate) fn meter_file( + pack: &gix_pack::data::File, + limits: &PackLimits, + kind: gix::hash::Kind, +) -> Result { + let reader = io::BufReader::new(std::fs::File::open(pack.path())?); + meter_entries(reader, ObjectCount::from(pack.num_objects()), limits, kind) +} + +fn meter_entries( + reader: R, + num_objects: ObjectCount, + limits: &PackLimits, + kind: gix::hash::Kind, +) -> Result { + if num_objects > limits.max_objects { + return Err(PackError::LimitExceeded(PackLimit::Objects)); + } + let mut entries = input::BytesToEntriesIter::new_from_header( + reader, + input::Mode::Verify, + input::EntryDataMode::Keep, + kind, + ) + .map_err(|error| PackError::Pack(error.to_string()))?; + + let mut total = 0u64; + let mut thin = false; + let mut base_of: HashMap = HashMap::new(); + entries.try_for_each(|entry| -> Result<(), PackError> { + let entry = entry.map_err(|error| PackError::Pack(error.to_string()))?; + if limits.max_object_bytes.exceeded_by(entry.decompressed_size) { + return Err(PackError::LimitExceeded(PackLimit::ObjectBytes)); + } + total = total + .checked_add(entry.decompressed_size) + .ok_or_else(|| malformed("decompressed size overflow"))?; + if limits.max_total_bytes.exceeded_by(total) { + return Err(PackError::LimitExceeded(PackLimit::TotalBytes)); + } + match entry.header { + Header::OfsDelta { base_distance } => { + let pack_offset = PackOffset::new(entry.pack_offset); + let base = pack_offset + .checked_sub_distance(base_distance) + .ok_or_else(|| malformed("ofs-delta base out of range"))?; + base_of.insert(pack_offset, base); + check_delta_result(&entry, limits.max_object_bytes)?; + } + Header::RefDelta { .. } => { + thin = true; + check_delta_result(&entry, limits.max_object_bytes)?; + } + _ => {} + } + Ok(()) + })?; + + check_depth(&base_of, limits.max_delta_depth)?; + Ok(thin) +} + +fn check_delta_result( + entry: &input::Entry, + max_object_bytes: MaxObjectBytes, +) -> Result<(), PackError> { + let compressed = entry + .compressed + .as_deref() + .ok_or_else(|| malformed("delta entry missing compressed data"))?; + let mut peek = HeaderPeek::new(); + inflate_into(compressed, entry.decompressed_size, &mut peek)?; + if max_object_bytes.exceeded_by(delta_result_size(peek.filled())?) { + return Err(PackError::LimitExceeded(PackLimit::ObjectBytes)); + } + Ok(()) +} + +pub(crate) fn inflate_into( + input: &[u8], + expected: u64, + out: &mut dyn Write, +) -> Result { + let mut decompress = Decompress::new(true); + let mut scratch = [0u8; 8192]; + let mut produced = 0u64; + loop { + let consumed = decompress.total_in() as usize; + let out_before = decompress.total_out(); + let status = decompress + .decompress( + input.get(consumed..).unwrap_or_default(), + &mut scratch, + FlushDecompress::None, + ) + .map_err(|error| PackError::Pack(format!("inflate: {error}")))?; + let written = (decompress.total_out() - out_before) as usize; + produced += written as u64; + if produced > expected { + return Err(malformed("object inflates beyond its declared size")); + } + out.write_all(&scratch[..written]) + .map_err(|error| PackError::Pack(format!("inflate sink: {error}")))?; + match status { + Status::StreamEnd => break, + Status::Ok | Status::BufError => { + if decompress.total_in() as usize == consumed + && decompress.total_out() == out_before + { + return Err(malformed("inflate stalled or pack truncated")); + } + } + } + } + if produced != expected { + return Err(malformed("object decompressed size mismatch")); + } + Ok(decompress.total_in()) +} + +struct HeaderPeek { + bytes: [u8; 32], + len: usize, +} + +impl HeaderPeek { + fn new() -> Self { + Self { + bytes: [0u8; 32], + len: 0, + } + } + + fn filled(&self) -> &[u8] { + &self.bytes[..self.len] + } +} + +impl Write for HeaderPeek { + fn write(&mut self, data: &[u8]) -> io::Result { + let take = (self.bytes.len() - self.len).min(data.len()); + self.bytes[self.len..self.len + take].copy_from_slice(&data[..take]); + self.len += take; + Ok(data.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +fn read_delta_varint( + data: &[u8], + pos: usize, + shift: u32, + acc: u64, +) -> Result<(u64, usize), PackError> { + if shift >= u64::BITS { + return Err(malformed("delta size header overflows")); + } + let byte = *data + .get(pos) + .ok_or_else(|| malformed("delta size header truncated"))?; + let acc = acc | (u64::from(byte & 0x7f) << shift); + if byte & 0x80 == 0 { + Ok((acc, pos + 1)) + } else { + read_delta_varint(data, pos + 1, shift + 7, acc) + } +} + +fn delta_result_size(header: &[u8]) -> Result { + let (_base_size, after_base) = read_delta_varint(header, 0, 0, 0)?; + let (result_size, _) = read_delta_varint(header, after_base, 0, 0)?; + Ok(result_size) +} + +pub(crate) fn check_depth( + base_of: &HashMap, + max: DeltaDepth, +) -> Result<(), PackError> { + base_of.keys().try_for_each(|start| { + let mut depth = DeltaDepth::ZERO; + let mut cursor = *start; + while let Some(&base) = base_of.get(&cursor) { + depth = depth.deeper(); + if depth.exceeds(max) { + return Err(PackError::LimitExceeded(PackLimit::DeltaDepth)); + } + cursor = base; + } + Ok(()) + }) +} diff --git a/knot2/crates/knot-pack/src/objects.rs b/knot2/crates/knot-pack/src/objects.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/src/objects.rs @@ -0,0 +1,993 @@ +use std::collections::{HashMap, HashSet}; +use std::error::Error; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, mpsc}; +use std::time::{Duration, Instant}; + +use gix::ObjectId; +use gix::prelude::FindExt; +use gix::progress::Discard; +use gix_pack::data::{Version, output}; +use knot_types::{ObjectCount, Oid}; + +use crate::error::{PackError, PackLimit}; +use crate::ids::{Crc32, MaxObjectBytes, PackOffset}; +use crate::meter::{self, PackLimits, check_depth}; +use crate::resolve; + +type OidStream = Box>> + Send>; + +fn odb_at(objects_dir: &Path, kind: gix::hash::Kind) -> Result { + gix::odb::at_opts( + objects_dir, + std::iter::empty(), + gix::odb::store::init::Options { + object_hash: kind, + ..Default::default() + }, + ) + .map_err(|error| PackError::Pack(error.to_string())) +} + +pub fn write_pack( + objects_dir: &Path, + oids: Vec, + thin_bases: Option<&HashSet>, + out: &mut dyn Write, + kind: gix::hash::Kind, +) -> Result<(), PackError> { + let mut odb = odb_at(objects_dir, kind)?; + odb.prevent_pack_unload(); + odb.refresh_never(); + + let interrupt = AtomicBool::new(false); + let permitted_bases: Option> = + thin_bases.map(|bases| bases.iter().map(|oid| oid.object_id()).collect()); + let oids: OidStream = Box::new(oids.into_iter().map(|oid| Ok(oid.object_id()))); + + let (counts, _) = output::count::objects( + odb.clone(), + oids, + &Discard, + &interrupt, + output::count::objects::Options { + thread_limit: knot_resource::gix_thread_limit().map(knot_resource::ThreadCount::get), + chunk_size: 50, + input_object_expansion: output::count::objects::ObjectExpansion::AsIs, + }, + ) + .map_err(|error| PackError::Pack(error.to_string()))?; + + write_counts(counts, odb, permitted_bases, out, kind) +} + +pub struct ExpandedPack { + counts: Vec, + odb: gix::odb::Handle, + kind: gix::hash::Kind, +} + +impl ExpandedPack { + pub fn len(&self) -> usize { + self.counts.len() + } + + pub fn is_empty(&self) -> bool { + self.counts.is_empty() + } +} + +pub fn count_expanded( + objects_dir: &Path, + roots: Vec, + max_objects: ObjectCount, + stall: Duration, + kind: gix::hash::Kind, +) -> Result { + let mut odb = odb_at(objects_dir, kind)?; + odb.prevent_pack_unload(); + odb.refresh_never(); + + let interrupt = Arc::new(AtomicBool::new(false)); + let counter = Arc::new(AtomicUsize::new(0)); + let progress = SharedCount(Arc::clone(&counter)); + let oids: OidStream = Box::new(roots.into_iter().map(|oid| Ok(oid.object_id()))); + let result = { + let _watchdog = Watchdog::arm(interrupt.clone(), Arc::clone(&counter), max_objects, stall); + output::count::objects( + Interruptible { + inner: odb.clone(), + flag: Arc::clone(&interrupt), + }, + oids, + &progress, + &interrupt, + output::count::objects::Options { + thread_limit: knot_resource::gix_thread_limit() + .map(knot_resource::ThreadCount::get), + chunk_size: 50, + input_object_expansion: output::count::objects::ObjectExpansion::TreeContents, + }, + ) + }; + let over_limit = counter.load(Ordering::Relaxed) > max_objects.get(); + let timed_out = interrupt.load(Ordering::Relaxed); + let counts = match result { + Ok((counts, _)) => counts, + Err(_) if over_limit => return Err(PackError::SelectionTooLarge), + Err(_) if timed_out => return Err(PackError::SelectionTimeout), + Err(error) => return Err(PackError::Pack(error.to_string())), + }; + if counts.len() > max_objects.get() { + return Err(PackError::SelectionTooLarge); + } + if timed_out { + return Err(PackError::SelectionTimeout); + } + Ok(ExpandedPack { counts, odb, kind }) +} + +pub fn write_expanded(pack: ExpandedPack, out: &mut dyn Write) -> Result<(), PackError> { + write_counts(pack.counts, pack.odb, None, out, pack.kind) +} + +fn write_counts( + counts: Vec, + odb: gix::odb::Handle, + permitted_bases: Option>, + out: &mut dyn Write, + kind: gix::hash::Kind, +) -> Result<(), PackError> { + let num_entries = counts.len() as u32; + let allow_thin_pack = permitted_bases.is_some(); + let counted = output::entry::iter_from_counts( + counts, + odb.clone(), + Box::new(Discard), + output::entry::iter_from_counts::Options { + thread_limit: knot_resource::gix_thread_limit().map(knot_resource::ThreadCount::get), + mode: output::entry::iter_from_counts::Mode::PackCopyAndBaseObjects, + allow_thin_pack, + chunk_size: 50, + version: Version::V2, + }, + ); + + let entries = gix::parallel::InOrderIter::from(counted).map( + move |chunk| -> Result, PackError> { + let entries = chunk.map_err(|error| PackError::Pack(error.to_string()))?; + entries + .into_iter() + .map(|entry| restrict_thin_base(&odb, permitted_bases.as_ref(), entry)) + .collect() + }, + ); + + let mut writer = + output::bytes::FromEntriesIter::new(entries, out, num_entries, Version::V2, kind); + writer + .try_fold((), |(), written| written.map(|_| ())) + .map_err(|error| PackError::Pack(error.to_string()))?; + Ok(()) +} + +struct SharedCount(Arc); + +impl gix::progress::Count for SharedCount { + fn set(&self, step: usize) { + self.0.store(step, Ordering::Relaxed); + } + + fn step(&self) -> usize { + self.0.load(Ordering::Relaxed) + } + + fn inc_by(&self, step: usize) { + self.0.fetch_add(step, Ordering::Relaxed); + } + + fn counter(&self) -> Arc { + Arc::clone(&self.0) + } +} + +#[derive(Debug)] +struct Halted; + +impl std::fmt::Display for Halted { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("object enumeration halted by selection budget") + } +} + +impl Error for Halted {} + +#[derive(Clone)] +struct Interruptible { + inner: gix::odb::Handle, + flag: Arc, +} + +impl gix_pack::Find for Interruptible { + fn contains(&self, id: &gix::hash::oid) -> bool { + self.inner.contains(id) + } + + fn try_find_cached<'a>( + &self, + id: &gix::hash::oid, + buffer: &'a mut Vec, + pack_cache: &mut dyn gix_pack::cache::DecodeEntry, + ) -> Result< + Option<(gix::objs::Data<'a>, Option)>, + gix::objs::find::Error, + > { + if self.flag.load(Ordering::Relaxed) { + return Err(Box::new(Halted)); + } + self.inner.try_find_cached(id, buffer, pack_cache) + } + + fn location_by_oid( + &self, + id: &gix::hash::oid, + buf: &mut Vec, + ) -> Option { + self.inner.location_by_oid(id, buf) + } + + fn pack_offsets_and_oid( + &self, + pack_id: u32, + ) -> Option> { + self.inner.pack_offsets_and_oid(pack_id) + } + + fn entry_by_location( + &self, + location: &gix_pack::data::entry::Location, + ) -> Option { + self.inner.entry_by_location(location) + } +} + +struct Watchdog { + idle: Option>, + watch: Option>, +} + +impl Watchdog { + fn arm( + flag: Arc, + counter: Arc, + max_objects: ObjectCount, + stall: Duration, + ) -> Self { + let (idle, wake) = mpsc::channel::<()>(); + let watch = std::thread::spawn(move || { + let poll = Duration::from_millis(25).min(stall); + let mut last = counter.load(Ordering::Relaxed); + let mut since = Instant::now(); + loop { + let now = counter.load(Ordering::Relaxed); + if now != last { + last = now; + since = Instant::now(); + } + if since.elapsed() >= stall || now > max_objects.get() { + flag.store(true, Ordering::Relaxed); + return; + } + if !matches!( + wake.recv_timeout(poll), + Err(mpsc::RecvTimeoutError::Timeout) + ) { + return; + } + } + }); + Self { + idle: Some(idle), + watch: Some(watch), + } + } +} + +impl Drop for Watchdog { + fn drop(&mut self) { + self.idle.take(); + if let Some(watch) = self.watch.take() { + let _ = watch.join(); + } + } +} + +fn restrict_thin_base( + odb: &gix::odb::Handle, + permitted: Option<&HashSet>, + entry: output::Entry, +) -> Result { + let base = match &entry.kind { + output::entry::Kind::DeltaOid { id } => *id, + _ => return Ok(entry), + }; + if permitted.is_some_and(|bases| bases.contains(&base)) { + return Ok(entry); + } + let mut buf = Vec::new(); + let object = odb + .find(&entry.id, &mut buf) + .map_err(|error| PackError::Pack(error.to_string()))?; + let count = output::Count::from_data(entry.id, None); + output::Entry::from_data(&count, &object).map_err(|error| PackError::Pack(error.to_string())) +} + +pub fn index_pack( + objects_dir: &Path, + pack: &[u8], + limits: &PackLimits, + kind: gix::hash::Kind, +) -> Result<(), PackError> { + if pack.is_empty() { + return Ok(()); + } + if !pack.starts_with(b"PACK") { + return Err(PackError::Pack( + "packfile is missing its PACK signature".to_string(), + )); + } + std::fs::create_dir_all(objects_dir)?; + let mut tmp = tempfile::NamedTempFile::new_in(objects_dir)?; + tmp.write_all(pack)?; + tmp.flush()?; + let file = gix_pack::data::File::at(tmp.path(), kind) + .map_err(|error| PackError::Pack(error.to_string()))?; + index_pack_bounded(objects_dir, &file, limits, kind) +} + +pub(crate) fn index_pack_bounded( + objects_dir: &Path, + pack: &gix_pack::data::File, + limits: &PackLimits, + kind: gix::hash::Kind, +) -> Result<(), PackError> { + if pack.data_len() < 12 + kind.len_in_bytes() { + return Ok(()); + } + let thin = meter::meter_file(pack, limits, kind)?; + + let pack_dir = objects_dir.join("pack"); + std::fs::create_dir_all(&pack_dir)?; + + inline_and_index(objects_dir, pack, &pack_dir, limits, kind, thin).or_else(|_| { + let bytes = std::fs::read(pack.path())?; + resolve::resolve(objects_dir, &bytes, limits, kind) + }) +} + +fn inline_and_index( + objects_dir: &Path, + pack: &gix_pack::data::File, + pack_dir: &Path, + limits: &PackLimits, + kind: gix::hash::Kind, + thin: bool, +) -> Result<(), PackError> { + let (pack_path, pack_hash) = if thin { + let pack_hash = inline_thin_bases(objects_dir, pack, pack_dir, kind)?; + ( + pack_dir.join(format!("pack-{}.pack", pack_hash.to_hex())), + pack_hash, + ) + } else { + let pack_hash = pack.checksum(); + let pack_path = pack_dir.join(format!("pack-{}.pack", pack_hash.to_hex())); + let source = pack.path().to_owned(); + persist_atomic(&pack_path, |writer| { + std::io::copy(&mut std::fs::File::open(&source)?, writer)?; + Ok(()) + })?; + (pack_path, pack_hash) + }; + let pack_cleanup = RemoveOnDrop::arm(&pack_path); + + let nodes = scan_offsets(&pack_path, kind)?; + let spool = spool_pack( + &pack_path, + nodes, + kind, + knot_resource::ingest_base_budget(), + limits.max_object_bytes, + None, + )?; + let (_present, idx_cleanup) = persist_index(pack_dir, &pack_hash, spool, kind)?; + pack_cleanup.disarm(); + idx_cleanup.disarm(); + Ok(()) +} + +fn inline_thin_bases( + objects_dir: &Path, + pack: &gix_pack::data::File, + pack_dir: &Path, + kind: gix::hash::Kind, +) -> Result { + let odb = odb_at(objects_dir, kind)?; + let staged = tempfile::NamedTempFile::new_in(pack_dir)?; + let writer = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(staged.path())?; + let reader = std::io::BufReader::new(std::fs::File::open(pack.path())?); + let entries = gix_pack::data::input::BytesToEntriesIter::new_from_header( + reader, + gix_pack::data::input::Mode::Verify, + gix_pack::data::input::EntryDataMode::KeepAndCrc32, + kind, + ) + .map_err(|error| PackError::Pack(error.to_string()))?; + let version = entries.version(); + let lookup = gix_pack::data::input::LookupRefDeltaObjectsIter::new(entries, odb); + let mut sink = gix_pack::data::input::EntriesToBytesIter::new(lookup, writer, version, kind); + sink.try_for_each(|entry| { + entry + .map(|_| ()) + .map_err(|error| PackError::Pack(error.to_string())) + })?; + let pack_hash = sink + .digest() + .ok_or_else(|| PackError::Pack("resolved pack has no trailer".to_string()))?; + drop(sink); + + let pack_path = pack_dir.join(format!("pack-{}.pack", pack_hash.to_hex())); + staged + .persist(&pack_path) + .map_err(|error| PackError::Pack(error.to_string()))?; + Ok(pack_hash) +} + +fn scan_offsets(pack_path: &Path, kind: gix::hash::Kind) -> Result, PackError> { + let reader = std::io::BufReader::new(std::fs::File::open(pack_path)?); + let mut entries = gix_pack::data::input::BytesToEntriesIter::new_from_header( + reader, + gix_pack::data::input::Mode::Verify, + gix_pack::data::input::EntryDataMode::Crc32, + kind, + ) + .map_err(|error| PackError::Pack(error.to_string()))?; + let mut nodes: Vec = Vec::new(); + entries.try_for_each(|entry| -> Result<(), PackError> { + let entry = entry.map_err(|error| PackError::Pack(error.to_string()))?; + nodes.push(Node { + offset: PackOffset::new(entry.pack_offset), + crc32: Crc32::new( + entry + .crc32 + .ok_or_else(|| PackError::Pack("entry crc32 not computed".to_string()))?, + ), + }); + Ok(()) + })?; + Ok(nodes) +} + +pub(crate) struct PresentSet(gix_pack::index::File); + +impl PresentSet { + fn open(idx_path: &Path, kind: gix::hash::Kind) -> Result { + gix_pack::index::File::at(idx_path, kind) + .map(Self) + .map_err(|error| PackError::Pack(error.to_string())) + } + + pub(crate) fn contains(&self, oid: &Oid) -> bool { + self.0.lookup(oid.object_id()).is_some() + } + + fn sorted_offsets(&self) -> Vec { + self.0 + .sorted_offsets() + .into_iter() + .map(PackOffset::new) + .collect() + } +} + +pub(crate) struct FreshClosure { + pub self_contained: bool, + pub present: PresentSet, +} + +#[derive(Clone)] +struct Node { + offset: PackOffset, + crc32: Crc32, +} + +const PRESENT: u8 = 1; +const REFERENCED: u8 = 2; + +struct Connectivity { + objects: scc::HashMap, + empty_tree: Oid, +} + +impl Connectivity { + fn mark_present(&self, oid: Oid) { + self.objects + .entry_sync(oid) + .and_modify(|flags| *flags |= PRESENT) + .or_insert(PRESENT); + } + + fn check(&self, reference: Oid) { + if reference != self.empty_tree { + self.objects + .entry_sync(reference) + .and_modify(|flags| *flags |= REFERENCED) + .or_insert(REFERENCED); + } + } + + fn self_contained(&self) -> bool { + self.objects + .any_sync(|_, flags| *flags & REFERENCED != 0 && *flags & PRESENT == 0) + .is_none() + } +} + +enum Scan { + Thin, + Limit(PackLimit), + Pack(String), +} + +const INGEST_BYTES_PER_OBJECT: u64 = 128; + +fn fits_in_memory(num_objects: ObjectCount) -> bool { + knot_resource::ingest_admits(knot_resource::PayloadBytes::new( + (num_objects.get() as u64).saturating_mul(INGEST_BYTES_PER_OBJECT), + )) +} + +pub(crate) fn admit_ingest( + pack: &gix_pack::data::File, + kind: gix::hash::Kind, +) -> Result<(), PackError> { + if pack.data_len() < 12 + kind.len_in_bytes() { + return Ok(()); + } + if fits_in_memory(ObjectCount::from(pack.num_objects())) { + Ok(()) + } else { + Err(PackError::InsufficientMemory) + } +} + +struct RemoveOnDrop(Option); + +impl RemoveOnDrop { + fn arm(path: &Path) -> Self { + Self(Some(path.to_owned())) + } + + fn disarm(mut self) { + self.0 = None; + } +} + +impl Drop for RemoveOnDrop { + fn drop(&mut self) { + if let Some(path) = self.0.take() { + let _ = std::fs::remove_file(path); + } + } +} + +pub(crate) fn ingest_and_close( + objects_dir: &Path, + pack: &gix_pack::data::File, + limits: &PackLimits, + kind: gix::hash::Kind, + base_budget: Option, + force_external: bool, +) -> Result, PackError> { + let hash_len = kind.len_in_bytes(); + if pack.data_len() < 12 + hash_len { + return Ok(None); + } + if ObjectCount::from(pack.num_objects()) > limits.max_objects { + return Err(PackError::LimitExceeded(PackLimit::Objects)); + } + + let reader = std::io::BufReader::new(std::fs::File::open(pack.path())?); + let mut entries = gix_pack::data::input::BytesToEntriesIter::new_from_header( + reader, + gix_pack::data::input::Mode::Verify, + gix_pack::data::input::EntryDataMode::Crc32, + kind, + ) + .map_err(|error| PackError::Pack(error.to_string()))?; + let mut nodes: Vec = Vec::with_capacity(pack.num_objects() as usize); + let mut base_of: HashMap = HashMap::new(); + let mut total_decompressed: u64 = 0; + let mut max_object: u64 = 0; + let scan = entries.try_for_each(|entry| -> Result<(), Scan> { + let entry = entry.map_err(|error| Scan::Pack(error.to_string()))?; + match entry.header { + gix_pack::data::entry::Header::RefDelta { .. } => return Err(Scan::Thin), + gix_pack::data::entry::Header::OfsDelta { base_distance } => { + let pack_offset = PackOffset::new(entry.pack_offset); + let base = pack_offset + .checked_sub_distance(base_distance) + .ok_or_else(|| Scan::Pack("ofs-delta base out of range".to_string()))?; + base_of.insert(pack_offset, base); + } + _ => {} + } + if limits.max_object_bytes.exceeded_by(entry.decompressed_size) { + return Err(Scan::Limit(PackLimit::ObjectBytes)); + } + max_object = max_object.max(entry.decompressed_size); + total_decompressed = total_decompressed + .checked_add(entry.decompressed_size) + .ok_or_else(|| Scan::Pack("decompressed size overflow".to_string()))?; + if limits.max_total_bytes.exceeded_by(total_decompressed) { + return Err(Scan::Limit(PackLimit::TotalBytes)); + } + nodes.push(Node { + offset: PackOffset::new(entry.pack_offset), + crc32: Crc32::new( + entry + .crc32 + .ok_or_else(|| Scan::Pack("entry crc32 not computed".to_string()))?, + ), + }); + Ok(()) + }); + match scan { + Ok(()) => {} + Err(Scan::Thin) => return Ok(None), + Err(Scan::Limit(limit)) => return Err(PackError::LimitExceeded(limit)), + Err(Scan::Pack(message)) => return Err(PackError::Pack(message)), + } + check_depth(&base_of, limits.max_delta_depth)?; + drop(base_of); + let entry_count = nodes.len(); + + let base_cache = knot_resource::ingest_base_budget().unwrap_or(0) as u64; + let working_set = (entry_count as u64) + .saturating_mul(INGEST_BYTES_PER_OBJECT) + .saturating_add(base_cache) + .saturating_add(max_object); + if !knot_resource::ingest_admits_churn( + knot_resource::WorkingSetBytes::new(working_set), + knot_resource::ChurnBytes::new(total_decompressed), + ) { + return Err(PackError::InsufficientMemory); + } + + let pack_hash = pack.checksum(); + let pack_dir = objects_dir.join("pack"); + std::fs::create_dir_all(&pack_dir)?; + let stem = format!("pack-{}", pack_hash.to_hex()); + let pack_path = pack_dir.join(format!("{stem}.pack")); + let source = pack.path().to_owned(); + persist_atomic(&pack_path, |writer| { + let mut reader = std::fs::File::open(&source)?; + std::io::copy(&mut reader, writer)?; + Ok(()) + })?; + let pack_cleanup = RemoveOnDrop::arm(&pack_path); + + let externalize = force_external + || knot_resource::externalize_connectivity(knot_resource::ConnectivityObjects::new( + entry_count as u64, + )); + let connectivity = (!externalize).then(|| Connectivity { + objects: scc::HashMap::with_capacity(entry_count), + empty_tree: Oid::from(ObjectId::empty_tree(kind)), + }); + let spool = spool_pack( + &pack_path, + nodes, + kind, + base_budget, + limits.max_object_bytes, + connectivity.as_ref(), + )?; + let self_contained_in_ram = connectivity.as_ref().map(Connectivity::self_contained); + drop(connectivity); + let (present, idx_cleanup) = persist_index(&pack_dir, &pack_hash, spool, kind)?; + + let self_contained = match self_contained_in_ram { + Some(value) => value, + None => verify_connectivity_retraverse( + &pack_path, + &present, + kind, + base_budget, + limits.max_object_bytes, + )?, + }; + + pack_cleanup.disarm(); + idx_cleanup.disarm(); + Ok(Some(FreshClosure { + self_contained, + present, + })) +} + +fn spool_pack( + pack_path: &Path, + nodes: Vec, + kind: gix::hash::Kind, + base_budget: Option, + max_object_bytes: MaxObjectBytes, + connectivity: Option<&Connectivity>, +) -> Result { + let interrupt = AtomicBool::new(false); + let tree = gix_pack::cache::delta::Tree::from_offsets_in_pack( + pack_path, + nodes.into_iter(), + &|node: &Node| node.offset.get(), + &|_id| None, + &mut Discard, + &interrupt, + kind, + ) + .map_err(|error| PackError::Pack(error.to_string()))?; + let stored = gix_pack::data::File::at(pack_path, kind) + .map_err(|error| PackError::Pack(error.to_string()))?; + let spool = crate::idxwrite::Spool::new(kind)?; + run_ingest_traverse( + tree, + &stored, + &interrupt, + base_budget, + max_object_bytes, + kind, + |node: &mut Node, _progress, context| harvest(node, context, kind, connectivity, &spool), + )?; + Ok(spool) +} + +fn persist_index( + pack_dir: &Path, + pack_hash: &ObjectId, + spool: crate::idxwrite::Spool, + kind: gix::hash::Kind, +) -> Result<(PresentSet, RemoveOnDrop), PackError> { + let idx_path = pack_dir.join(format!("pack-{}.idx", pack_hash.to_hex())); + persist_atomic(&idx_path, |writer| { + crate::idxwrite::write_v2_index(writer, &spool, pack_hash, kind).map(|_| ()) + })?; + let idx_cleanup = RemoveOnDrop::arm(&idx_path); + drop(spool); + let present = PresentSet::open(&idx_path, kind)?; + Ok((present, idx_cleanup)) +} + +// he wishes he was on the farm already +fn harvest( + node: &Node, + context: gix_pack::cache::delta::traverse::Context<'_>, + kind: gix::hash::Kind, + connectivity: Option<&Connectivity>, + spool: &crate::idxwrite::Spool, +) -> Result<(), PackError> { + let id = gix::objs::compute_hash(kind, context.object_kind, context.decompressed) + .map_err(|error| PackError::Pack(error.to_string()))?; + spool.push(id, node.crc32, node.offset)?; + if let Some(connectivity) = connectivity { + connectivity.mark_present(Oid::from(id)); + parse_references( + context.object_kind, + context.decompressed, + kind, + &mut |reference| connectivity.check(reference), + )?; + } + Ok(()) +} + +fn parse_references( + object_kind: gix::object::Kind, + bytes: &[u8], + kind: gix::hash::Kind, + check: &mut dyn FnMut(Oid), +) -> Result<(), PackError> { + match object_kind { + gix::object::Kind::Blob => Ok(()), + gix::object::Kind::Tree => harvest_tree(bytes, kind, check), + gix::object::Kind::Commit => harvest_commit(bytes, kind, check), + gix::object::Kind::Tag => harvest_tag(bytes, kind, check), + } +} + +fn harvest_tree( + bytes: &[u8], + kind: gix::hash::Kind, + check: &mut dyn FnMut(Oid), +) -> Result<(), PackError> { + gix::objs::TreeRefIter::from_bytes(bytes, kind).try_for_each(|entry| { + let entry = entry.map_err(|error| PackError::Pack(error.to_string()))?; + if !entry.mode.is_commit() { + check(Oid::from(entry.oid.to_owned())); + } + Ok(()) + }) +} + +fn harvest_commit( + bytes: &[u8], + kind: gix::hash::Kind, + check: &mut dyn FnMut(Oid), +) -> Result<(), PackError> { + let mut iter = gix::objs::CommitRefIter::from_bytes(bytes, kind); + let tree = iter + .tree_id() + .map_err(|error| PackError::Pack(error.to_string()))?; + std::iter::once(tree) + .chain(iter.parent_ids()) + .for_each(|oid| check(Oid::from(oid))); + Ok(()) +} + +fn harvest_tag( + bytes: &[u8], + kind: gix::hash::Kind, + check: &mut dyn FnMut(Oid), +) -> Result<(), PackError> { + let target = gix::objs::TagRefIter::from_bytes(bytes, kind) + .target_id() + .map_err(|error| PackError::Pack(error.to_string()))?; + check(Oid::from(target)); + Ok(()) +} + +fn verify_connectivity_retraverse( + pack_path: &Path, + present: &PresentSet, + kind: gix::hash::Kind, + base_budget: Option, + max_object_bytes: MaxObjectBytes, +) -> Result { + let interrupt = AtomicBool::new(false); + let tree = gix_pack::cache::delta::Tree::from_offsets_in_pack( + pack_path, + present.sorted_offsets().into_iter(), + &|offset: &PackOffset| offset.get(), + &|_id| None, + &mut Discard, + &interrupt, + kind, + ) + .map_err(|error| PackError::Pack(error.to_string()))?; + let stored = gix_pack::data::File::at(pack_path, kind) + .map_err(|error| PackError::Pack(error.to_string()))?; + let empty_tree = Oid::from(ObjectId::empty_tree(kind)); + let missing = AtomicBool::new(false); + run_ingest_traverse( + tree, + &stored, + &interrupt, + base_budget, + max_object_bytes, + kind, + |_offset: &mut PackOffset, _progress, context| { + parse_references( + context.object_kind, + context.decompressed, + kind, + &mut |reference| { + if reference != empty_tree && !present.contains(&reference) { + missing.store(true, Ordering::Relaxed); + } + }, + ) + }, + )?; + Ok(!missing.load(Ordering::Relaxed)) +} + +fn run_ingest_traverse( + tree: gix_pack::cache::delta::Tree, + stored: &gix_pack::data::File, + interrupt: &AtomicBool, + base_budget: Option, + max_object_bytes: MaxObjectBytes, + kind: gix::hash::Kind, + harvest: H, +) -> Result<(), PackError> +where + T: Send + Sync + Clone, + H: FnMut( + &mut T, + &dyn gix::progress::Progress, + gix_pack::cache::delta::traverse::Context<'_>, + ) -> Result<(), PackError> + + Send + + Clone, +{ + let base_spill = match base_budget { + Some(budget) => Some(Arc::new(gix_pack::cache::delta::traverse::BaseSpill::new( + tempfile::tempfile()?, + budget, + ))), + None => None, + }; + tree.traverse( + |range: gix_pack::data::EntryRange, source: &gix_pack::data::File, buf: &mut Vec| { + source.read_into(range, buf) + }, + stored, + stored.pack_end() as u64, + harvest, + gix_pack::cache::delta::traverse::Options { + object_progress: Box::new(Discard), + size_progress: &mut Discard, + thread_limit: Some(knot_resource::ingest_thread_limit()), + should_interrupt: interrupt, + object_hash: kind, + base_spill, + collect_items: false, + max_object_bytes: Some(max_object_bytes.get()), + }, + ) + .map(|_| ()) + .map_err(|error| PackError::Pack(error.to_string())) +} + +fn persist_atomic( + path: &Path, + write: impl FnOnce(&mut dyn Write) -> Result<(), PackError>, +) -> Result<(), PackError> { + let dir = path + .parent() + .ok_or_else(|| PackError::Pack("object path has no parent".to_string()))?; + let mut tmp = tempfile::NamedTempFile::new_in(dir)?; + let mut writer = std::io::BufWriter::new(tmp.as_file_mut()); + write(&mut writer)?; + writer.flush()?; + drop(writer); + tmp.as_file().sync_all()?; + tmp.persist(path) + .map_err(|error| PackError::Pack(error.to_string()))?; + std::fs::File::open(dir)?.sync_all()?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use gix_pack::Find; + + use super::*; + + #[test] + fn interruptible_find_errors_once_the_flag_is_set() { + let dir = tempfile::tempdir().unwrap(); + let flag = Arc::new(AtomicBool::new(false)); + let find = Interruptible { + inner: gix::odb::at(dir.path()).unwrap(), + flag: Arc::clone(&flag), + }; + let absent = Oid::null().object_id(); + let mut buf = Vec::new(); + + assert!( + find.try_find(&absent, &mut buf).unwrap().is_none(), + "before budget trips, lookup of an absent object is a plain miss" + ); + + flag.store(true, Ordering::Relaxed); + assert!( + find.try_find(&absent, &mut buf).is_err(), + "once tripped, every decode errors, which is what aborts a breadthfirst tree walk \ + mid-closure instead of waiting for the next commit-root boundary" + ); + } +} diff --git a/knot2/crates/knot-pack/src/oids.rs b/knot2/crates/knot-pack/src/oids.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/src/oids.rs @@ -0,0 +1,71 @@ +use knot_types::Oid; + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct WantOids(Vec); + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct HaveOids(Vec); + +impl WantOids { + pub fn new(oids: Vec) -> Self { + Self(oids) + } + + pub fn as_slice(&self) -> &[Oid] { + &self.0 + } + + pub fn wants(&self) -> knot_git::Wants<'_> { + knot_git::Wants::new(&self.0) + } + + pub fn iter(&self) -> std::slice::Iter<'_, Oid> { + self.0.iter() + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl FromIterator for WantOids { + fn from_iter>(iter: I) -> Self { + Self(iter.into_iter().collect()) + } +} + +impl HaveOids { + pub fn new(oids: Vec) -> Self { + Self(oids) + } + + pub fn as_slice(&self) -> &[Oid] { + &self.0 + } + + pub fn haves(&self) -> knot_git::Haves<'_> { + knot_git::Haves::new(&self.0) + } + + pub fn iter(&self) -> std::slice::Iter<'_, Oid> { + self.0.iter() + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl FromIterator for HaveOids { + fn from_iter>(iter: I) -> Self { + Self(iter.into_iter().collect()) + } +} diff --git a/knot2/crates/knot-pack/src/pkt.rs b/knot2/crates/knot-pack/src/pkt.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/src/pkt.rs @@ -0,0 +1,310 @@ +use std::io; + +use gix_packetline::{Channel, blocking_io::encode}; + +pub const MAX_BAND: usize = 65515; + +pub enum Frame<'a> { + Data(&'a [u8]), + Flush, + Delim, + ResponseEnd, +} + +fn hex4(prefix: &[u8]) -> io::Result { + std::str::from_utf8(prefix) + .ok() + .and_then(|text| u16::from_str_radix(text, 16).ok()) + .ok_or_else(|| io::Error::other("invalid pkt-line length prefix")) +} + +pub fn frames( + input: &[u8], + stop_after_flushes: Option, +) -> impl Iterator, usize)>> + '_ { + let mut pos = 0usize; + let mut flushes = 0usize; + let mut stopped = false; + std::iter::from_fn(move || { + (!stopped && pos + 4 <= input.len()).then(|| { + let frame = hex4(&input[pos..pos + 4]).and_then(|len| { + pos += 4; + match len { + 0 => { + flushes += 1; + stopped = stop_after_flushes == Some(flushes); + Ok((Frame::Flush, pos)) + } + 1 => Ok((Frame::Delim, pos)), + 2 => Ok((Frame::ResponseEnd, pos)), + 3 => Err(io::Error::other("invalid pkt-line length 3")), + n => { + let end = pos - 4 + usize::from(n); + (end <= input.len()) + .then(|| { + let payload = &input[pos..end]; + pos = end; + (Frame::Data(payload), end) + }) + .ok_or_else(|| io::Error::other("truncated pkt-line")) + } + } + }); + stopped |= frame.is_err(); + frame + }) + }) +} + +pub fn data_payloads(input: &[u8]) -> io::Result> { + collect_data(input, Some(1)) +} + +pub fn data_payloads_all(input: &[u8]) -> io::Result> { + collect_data(input, None) +} + +fn collect_data(input: &[u8], stop_after_flushes: Option) -> io::Result> { + frames(input, stop_after_flushes) + .filter_map(|item| match item { + Ok((Frame::Data(payload), _)) => Some(Ok(payload)), + Ok(_) => None, + Err(err) => Some(Err(err)), + }) + .collect() +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct Caps { + pub atomic: bool, + pub side_band_64k: bool, + pub push_options: bool, +} + +pub(crate) fn first_command(input: &[u8]) -> Option<&[u8]> { + frames(input, Some(1)).find_map(|item| match item { + Ok((Frame::Data(payload), _)) if !is_preamble(payload) => Some(payload), + _ => None, + }) +} + +fn is_preamble(line: &[u8]) -> bool { + line.starts_with(b"shallow ") +} + +pub(crate) fn parse_caps(first_command: &[u8]) -> Caps { + let caps = first_command + .split(|byte| *byte == 0) + .nth(1) + .and_then(|caps| std::str::from_utf8(caps).ok()) + .unwrap_or_default(); + let has = |needle: &str| caps.split_whitespace().any(|cap| cap == needle); + Caps { + atomic: has("atomic"), + side_band_64k: has("side-band-64k"), + push_options: has("push-options"), + } +} + +pub struct Receive<'a> { + pub commands: Vec<&'a [u8]>, + pub options: Vec<&'a [u8]>, + pub pack: &'a [u8], + pub caps: Caps, +} + +pub fn split_receive(input: &[u8]) -> io::Result> { + let caps = first_command(input).map(parse_caps).unwrap_or_default(); + let boundary = if caps.push_options { 2 } else { 1 }; + frames(input, Some(boundary)) + .try_fold( + (Vec::new(), Vec::new(), 0usize, None), + |(mut commands, mut options, flushes, end), item| { + item.map(|(frame, at)| match frame { + Frame::Data(payload) if flushes == 0 && !is_preamble(payload) => { + commands.push(payload); + (commands, options, flushes, end) + } + Frame::Data(payload) if flushes == 1 && caps.push_options => { + options.push(payload); + (commands, options, flushes, end) + } + Frame::Data(_) => (commands, options, flushes, end), + Frame::Flush => (commands, options, flushes + 1, Some(at)), + _ => (commands, options, flushes, end), + }) + }, + ) + .map(|(commands, options, _flushes, end)| Receive { + commands, + options, + caps, + pack: &input[end.unwrap_or(input.len())..], + }) +} + +pub fn write_data(buf: &mut Vec, payload: &[u8]) -> io::Result<()> { + encode::data_to_write(payload, buf).map(|_| ()) +} + +pub fn write_flush(buf: &mut Vec) -> io::Result<()> { + encode::flush_to_write(buf).map(|_| ()) +} + +pub fn write_delim(buf: &mut Vec) -> io::Result<()> { + encode::delim_to_write(buf).map(|_| ()) +} + +pub fn write_band(buf: &mut Vec, chunk: &[u8]) -> io::Result<()> { + encode::band_to_write(Channel::Data, chunk, buf).map(|_| ()) +} + +pub fn write_band_progress(buf: &mut Vec, message: &[u8]) -> io::Result<()> { + encode::band_to_write(Channel::Progress, message, buf).map(|_| ()) +} + +pub fn write_band_error(buf: &mut Vec, message: &[u8]) -> io::Result<()> { + encode::band_to_write(Channel::Error, message, buf).map(|_| ()) +} + +pub fn frame_report(report: &[u8], messages: &[String], side_band: bool) -> Vec { + if !side_band { + return report.to_vec(); + } + let mut buf = Vec::new(); + report + .chunks(MAX_BAND) + .for_each(|chunk| write_band(&mut buf, chunk).expect("band write to vec never fails")); + messages.iter().for_each(|message| { + format!("{message}\n") + .into_bytes() + .chunks(MAX_BAND) + .for_each(|chunk| { + write_band_progress(&mut buf, chunk).expect("band write to vec never fails") + }); + }); + write_flush(&mut buf).expect("flush write to vec never fails"); + buf +} + +#[cfg(test)] +mod tests { + use super::*; + + fn command_line(caps: &str) -> Vec { + let mut line = b"\ + 0000000000000000000000000000000000000000 \ + 1111111111111111111111111111111111111111 refs/heads/main" + .to_vec(); + line.push(0); + line.extend_from_slice(caps.as_bytes()); + line.push(b'\n'); + line + } + + #[test] + fn a_malformed_length_prefix_terminates_instead_of_spinning() { + let garbage = b"zzzz this is not a pkt-line stream at all"; + assert_eq!( + frames(garbage, None).count(), + 1, + "bad length prefix yields one error frame then the stream ends" + ); + assert!( + first_command(garbage).is_none(), + "no command is parsed out of garbage, and scan does not loop" + ); + assert!( + split_receive(garbage).is_err(), + "malformed prefix is a parse error, never an infinite loop" + ); + } + + #[test] + fn split_receive_skips_the_push_options_section_before_the_pack() { + let mut body = Vec::new(); + write_data( + &mut body, + &command_line("report-status side-band-64k push-options"), + ) + .unwrap(); + write_flush(&mut body).unwrap(); + write_data(&mut body, b"ci-skip").unwrap(); + write_data(&mut body, b"verbose-ci").unwrap(); + write_flush(&mut body).unwrap(); + body.extend_from_slice(b"PACKreal-pack-bytes"); + + let parsed = split_receive(&body).unwrap(); + assert!(parsed.caps.push_options); + assert!(parsed.caps.side_band_64k); + assert_eq!(parsed.commands.len(), 1); + assert_eq!(parsed.options, vec![&b"ci-skip"[..], &b"verbose-ci"[..]]); + assert_eq!(parsed.pack, b"PACKreal-pack-bytes"); + } + + #[test] + fn split_receive_reads_caps_past_a_shallow_preamble_line() { + let mut body = Vec::new(); + write_data( + &mut body, + b"shallow 1111111111111111111111111111111111111111\n", + ) + .unwrap(); + write_data(&mut body, &command_line("report-status side-band-64k")).unwrap(); + write_flush(&mut body).unwrap(); + body.extend_from_slice(b"PACKbytes"); + + let parsed = split_receive(&body).unwrap(); + assert!( + parsed.caps.side_band_64k, + "capabilities come from the command line, not the shallow preamble" + ); + assert_eq!( + parsed.commands.len(), + 1, + "the shallow line is not a command" + ); + assert_eq!(parsed.pack, b"PACKbytes"); + } + + #[test] + fn split_receive_without_push_options_starts_the_pack_after_the_command_flush() { + let mut body = Vec::new(); + write_data(&mut body, &command_line("report-status side-band-64k")).unwrap(); + write_flush(&mut body).unwrap(); + body.extend_from_slice(b"PACKbytes"); + + let parsed = split_receive(&body).unwrap(); + assert!(!parsed.caps.push_options); + assert!(parsed.options.is_empty()); + assert_eq!(parsed.pack, b"PACKbytes"); + } + + #[test] + fn frame_report_muxes_the_report_on_band_one_and_messages_on_band_two() { + let report = b"unpack ok\n"; + let messages = vec!["hello there".to_string()]; + let framed = frame_report(report, &messages, true); + + let bands: Vec<(u8, Vec)> = frames(&framed, None) + .filter_map(|item| match item { + Ok((Frame::Data(payload), _)) => Some((payload[0], payload[1..].to_vec())), + _ => None, + }) + .collect(); + assert_eq!(bands[0].0, 1, "report rides band 1"); + assert_eq!(bands[0].1, report); + assert_eq!(bands[1].0, 2, "message rides band 2"); + assert_eq!(bands[1].1, b"hello there\n"); + assert!(framed.ends_with(b"0000"), "outer flush closes the stream"); + } + + #[test] + fn frame_report_passes_through_raw_without_side_band() { + let report = b"unpack ok\n0000"; + assert_eq!( + frame_report(report, &["dropped".to_string()], false), + report + ); + } +} diff --git a/knot2/crates/knot-pack/src/quarantine.rs b/knot2/crates/knot-pack/src/quarantine.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/src/quarantine.rs @@ -0,0 +1,72 @@ +//! The same way one would do this for uploading images onto a server, +//! we stage pushes such that objects unpack into a separate bare repo +//! under the incoming prefix, because we treat a push as real only once +//! every object it referenced actually made it over. +//! +//! Hence aborting is as simple as removal of the directory that +//! represents the push in flight. +//! +//! "Why not use `GIT_QUARANTINE_PATH`?" - because we never forked `receive-pack`. +//! +//! If you were wondering, this `sweep_incoming` is for a crash between a stage +//! and migration that would otherwise leak a staging dir. + +use std::path::Path; + +use knot_git::{INCOMING_PREFIX, Repo, Staging}; + +use crate::error::PackError; +use crate::meter::PackLimits; +use crate::objects; + +pub(crate) struct Quarantine { + staging: Staging, +} + +impl Quarantine { + pub(crate) fn stage( + live: &Repo, + pack: Option<&gix_pack::data::File>, + limits: &PackLimits, + kind: gix::hash::Kind, + live_empty: bool, + ) -> Result<(Self, Option), PackError> { + let staging = Staging::new(live)?; + let (unpack, closure) = crate::receive::ingest( + &staging.repo().objects_dir(), + pack, + limits, + kind, + live_empty, + ); + unpack?; + Ok((Self { staging }, closure)) + } + + pub(crate) fn repo(&self) -> &Repo { + self.staging.repo() + } + + pub(crate) fn migrate_into(&self, live: &Repo) -> Result<(), PackError> { + self.staging.migrate_into(live).map_err(PackError::from) + } +} + +pub fn sweep_incoming(scan_path: &Path) -> usize { + walkdir::WalkDir::new(scan_path) + .into_iter() + .filter_entry(|entry| entry.file_name().to_str() != Some("objects")) + .filter_map(Result::ok) + .filter(|entry| entry.file_type().is_dir()) + .filter(|entry| { + entry + .file_name() + .to_str() + .is_some_and(|name| name.starts_with(INCOMING_PREFIX)) + }) + .map(|entry| entry.into_path()) + .collect::>() + .into_iter() + .filter(|path| std::fs::remove_dir_all(path).is_ok()) + .count() +} diff --git a/knot2/crates/knot-pack/src/receive.rs b/knot2/crates/knot-pack/src/receive.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/src/receive.rs @@ -0,0 +1,861 @@ +use std::collections::HashMap; +use std::io::Write; +use std::path::Path; + +use knot_git::{Filter, RefTxn, RefUpdate, Repo}; +use knot_messages::{RefKey, RejectMessages}; +use knot_types::{ObjectFormat, Oid, PushOption, PushOptions, RefName}; + +use crate::error::PackError; +use crate::meter::PackLimits; +use crate::objects; +use crate::pkt; +use crate::quarantine::Quarantine; +use crate::receiver::ReceivedPack; +use crate::{HaveOids, WantOids}; + +fn stage_pack_bytes( + dir: &Path, + pack: &[u8], + kind: gix::hash::Kind, +) -> Result, PackError> { + if pack.is_empty() { + return Ok(None); + } + if !pack.starts_with(b"PACK") { + return Err(PackError::Pack( + "packfile is missing its PACK signature".to_string(), + )); + } + let mut tmp = tempfile::NamedTempFile::new_in(dir)?; + tmp.write_all(pack)?; + tmp.flush()?; + let file = gix_pack::data::File::at(tmp.path(), kind) + .map_err(|error| PackError::Pack(error.to_string()))?; + Ok(Some((tmp, file))) +} + +pub fn handle_bytes(repo: &Repo, body: &[u8], limits: &PackLimits) -> Result, PackError> { + let kind = repo.object_format().kind(); + let messages = &crate::default_catalog().reject; + match stage_pack_bytes(&repo.objects_dir(), pkt::split_receive(body)?.pack, kind) { + Ok(staged) => handle( + repo, + body, + Ok(staged.as_ref().map(|(_, file)| file)), + limits, + messages, + ), + Err(error) => handle(repo, body, Err(error), limits, messages), + } +} + +pub fn handle_guarded_bytes( + live: &Repo, + body: &[u8], + limits: &PackLimits, + guard: &dyn ReceiveGuard, + seal: &dyn Fn(&[RefUpdate]), + messages: &RejectMessages, +) -> Result { + let kind = live.object_format().kind(); + match stage_pack_bytes(&live.objects_dir(), pkt::split_receive(body)?.pack, kind) { + Ok(staged) => handle_guarded( + live, + body, + Ok(staged.as_ref().map(|(_, file)| file)), + limits, + guard, + seal, + messages, + ), + Err(error) => handle_guarded(live, body, Err(error), limits, guard, seal, messages), + } +} + +pub fn handle_guarded_streamed( + live: &Repo, + received: &ReceivedPack, + limits: &PackLimits, + guard: &dyn ReceiveGuard, + seal: &dyn Fn(&[RefUpdate]), + messages: &RejectMessages, +) -> Result { + match received.open_pack() { + Ok(pack) => handle_guarded( + live, + received.preamble(), + Ok(pack.as_ref()), + limits, + guard, + seal, + messages, + ), + Err(error) => handle_guarded( + live, + received.preamble(), + Err(error), + limits, + guard, + seal, + messages, + ), + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RefDecision { + Allow, + Reject(String), +} + +pub struct ReceiveOutcome { + pub report: Vec, + pub side_band: bool, + pub push_options: PushOptions, +} + +pub trait ReceiveGuard { + fn authorize(&self, staged: &Repo, commands: &[ReceiveCommand]) -> Vec; +} + +const CAPS_BASE: &str = + "report-status delete-refs atomic ofs-delta side-band-64k push-options agent=knot/0"; + +fn caps(format: ObjectFormat) -> String { + format!("{CAPS_BASE} object-format={}", format.capability()) +} + +pub fn advertise(repo: &Repo) -> Result, PackError> { + let mut buf = Vec::new(); + pkt::write_data(&mut buf, b"# service=git-receive-pack\n")?; + pkt::write_flush(&mut buf)?; + write_advert(&mut buf, repo)?; + Ok(buf) +} + +pub fn advertise_ssh(repo: &Repo) -> Result, PackError> { + let mut buf = Vec::new(); + write_advert(&mut buf, repo)?; + Ok(buf) +} + +fn write_advert(buf: &mut Vec, repo: &Repo) -> Result<(), PackError> { + let format = repo.object_format(); + let caps = caps(format); + let refs = repo.advertised_refs_for(knot_git::AdvertScope::Receive)?; + match refs.split_first() { + Some((first, rest)) => { + let mut line = format!("{} {}", first.target, first.name).into_bytes(); + line.push(0); + line.extend_from_slice(caps.as_bytes()); + line.push(b'\n'); + pkt::write_data(buf, &line)?; + rest.iter().try_fold(&mut *buf, |buf, record| { + pkt::write_data( + buf, + format!("{} {}\n", record.target, record.name).as_bytes(), + )?; + Ok::<_, PackError>(buf) + })?; + } + None => { + let mut line = format!("{} capabilities^{{}}", format.null_oid()).into_bytes(); + line.push(0); + line.extend_from_slice(caps.as_bytes()); + line.push(b'\n'); + pkt::write_data(buf, &line)?; + } + } + pkt::write_flush(buf)?; + Ok(()) +} + +enum CommandRef { + Named(RefName), + Unparsed(String), +} + +pub struct ReceiveCommand { + old: Oid, + new: Oid, + name: CommandRef, +} + +impl ReceiveCommand { + pub fn refname(&self) -> &str { + match &self.name { + CommandRef::Named(name) => name.as_str(), + CommandRef::Unparsed(raw) => raw, + } + } + + pub fn name(&self) -> Option<&RefName> { + match &self.name { + CommandRef::Named(name) => Some(name), + CommandRef::Unparsed(_) => None, + } + } + + pub fn is_delete(&self) -> bool { + self.new.is_null() + } + + pub fn is_create(&self) -> bool { + self.old.is_null() + } + + fn parse(line: &[u8], first: bool) -> Option { + let line = if first { + line.split(|byte| *byte == 0).next().unwrap_or(line) + } else { + line + }; + let text = std::str::from_utf8(line).ok()?; + let mut parts = text.trim_end().split(' '); + let old = Oid::from_hex(parts.next()?).ok()?; + let new = Oid::from_hex(parts.next()?).ok()?; + let raw = parts.next()?.to_string(); + let name = match RefName::new(raw.as_str()) { + Ok(name) => CommandRef::Named(name), + Err(_) => CommandRef::Unparsed(raw), + }; + Some(ReceiveCommand { old, new, name }) + } + + fn to_update(&self) -> Result { + let name = self + .name() + .cloned() + .ok_or_else(|| PackError::Protocol(invalid_refname(self.refname())))?; + Ok(match (self.old.is_null(), self.new.is_null()) { + (_, true) => RefUpdate::Delete { + name, + old: self.old, + }, + (true, false) => RefUpdate::Create { + name, + new: self.new, + }, + (false, false) => RefUpdate::Update { + name, + old: self.old, + new: self.new, + }, + }) + } +} + +pub(crate) fn invalid_refname(raw: &str) -> String { + format!("invalid ref name {raw:?}") +} + +fn forbidden_ref(command: &ReceiveCommand, messages: &RejectMessages) -> Option { + match command.name() { + Some(name) => (!knot_git::is_public_ref(name)).then(|| messages.reserved_refs.text()), + None => Some(invalid_refname(command.refname())), + } +} + +fn reserved_create_only(command: &ReceiveCommand, messages: &RejectMessages) -> Option { + (command.name().is_some_and(knot_git::is_reserved) && !command.is_create()) + .then(|| messages.cob_create_only.text()) +} + +struct RefSnapshot { + by_name: HashMap, +} + +impl RefSnapshot { + fn capture(repo: &Repo) -> Result { + let by_name = repo + .references()? + .into_iter() + .map(|record| (record.name, record.target)) + .collect(); + Ok(RefSnapshot { by_name }) + } + + fn tips(&self) -> HaveOids { + self.by_name.values().copied().collect() + } + + fn conflict(&self, command: &ReceiveCommand, messages: &RejectMessages) -> Option { + match ( + command.old.is_null(), + command + .name() + .and_then(|name| self.by_name.get(name).copied()), + ) { + (true, Some(_)) => Some(messages.ref_exists.text()), + (false, found) if found != Some(command.old) => Some(messages.stale_old_value.text()), + _ => None, + } + } +} + +struct RefResult { + refname: String, + failure: Option, +} + +impl RefResult { + fn of(command: &ReceiveCommand, failure: Option) -> RefResult { + RefResult { + refname: command.refname().to_string(), + failure, + } + } +} + +struct Conflict { + refname: String, + reason: String, +} + +fn first_conflict( + snapshot: &RefSnapshot, + commands: &[ReceiveCommand], + messages: &RejectMessages, +) -> Option { + commands.iter().find_map(|command| { + snapshot.conflict(command, messages).map(|reason| Conflict { + refname: command.refname().to_string(), + reason, + }) + }) +} + +fn objects_present( + repo: &Repo, + wants: &WantOids, + haves: &HaveOids, + closure: Option<&objects::FreshClosure>, +) -> bool { + if let Some(closure) = closure { + return closure.self_contained + && wants + .as_slice() + .iter() + .all(|want| closure.present.contains(want) || repo.contains(*want)); + } + matches!( + repo.select_pack_objects_filtered( + wants.wants(), + haves.haves(), + Filter::None, + crate::upload::selection_budget(), + ), + Ok(selection) if selection.send.iter().all(|oid| repo.contains(*oid)) + ) +} + +fn connectivity_reasons( + repo: &Repo, + commands: &[ReceiveCommand], + haves: &HaveOids, + closure: Option<&objects::FreshClosure>, + messages: &RejectMessages, +) -> Vec> { + let news: WantOids = commands + .iter() + .map(|command| command.new) + .filter(|new| !new.is_null()) + .collect(); + let batched_ok = news.is_empty() || objects_present(repo, &news, haves, closure); + commands + .iter() + .map(|command| { + if command.new.is_null() + || batched_ok + || objects_present(repo, &WantOids::new(vec![command.new]), haves, closure) + { + None + } else { + Some(messages.missing_objects.text()) + } + }) + .collect() +} + +pub(crate) fn fuzz(body: &[u8]) { + if let Ok(parsed) = pkt::split_receive(body) { + parsed + .commands + .iter() + .enumerate() + .for_each(|(index, line)| { + if let Some(command) = ReceiveCommand::parse(line, index == 0) { + let _ = command.to_update(); + } + }); + } +} + +pub(crate) fn is_empty(repo: &Repo) -> bool { + repo.references() + .map(|refs| refs.is_empty()) + .unwrap_or(false) +} + +pub(crate) fn ingest( + objects_dir: &Path, + pack: Option<&gix_pack::data::File>, + limits: &PackLimits, + kind: gix::hash::Kind, + live_empty: bool, +) -> (Result<(), PackError>, Option) { + let Some(pack) = pack else { + return (Ok(()), None); + }; + if let Err(error) = objects::admit_ingest(pack, kind) { + return (Err(error), None); + } + if live_empty { + match objects::ingest_and_close( + objects_dir, + pack, + limits, + kind, + knot_resource::ingest_base_budget(), + false, + ) { + Ok(Some(closure)) => return (Ok(()), Some(closure)), + Ok(None) => {} + Err(error) => return (Err(error), None), + } + } + ( + objects::index_pack_bounded(objects_dir, pack, limits, kind), + None, + ) +} + +pub fn handle( + repo: &Repo, + body: &[u8], + pack: Result, PackError>, + limits: &PackLimits, + messages: &RejectMessages, +) -> Result, PackError> { + let parsed = pkt::split_receive(body)?; + let atomic = parsed.caps.atomic; + let commands = parse_commands(&parsed); + let kind = repo.object_format().kind(); + + let (unpack, closure) = match pack { + Ok(pack) => ingest(&repo.objects_dir(), pack, limits, kind, is_empty(repo)), + Err(error) => (Err(error), None), + }; + let results: Vec = match &unpack { + Err(_) => all_failed(&commands, &messages.unpacker_error.text()), + Ok(()) => match RefSnapshot::capture(repo) { + Err(_) => all_failed(&commands, &messages.ref_snapshot_unavailable.text()), + Ok(snapshot) => { + let haves = snapshot.tips(); + if atomic { + apply_atomic( + repo, + &snapshot, + &commands, + &haves, + closure.as_ref(), + messages, + ) + } else { + commands + .iter() + .zip(connectivity_reasons( + repo, + &commands, + &haves, + closure.as_ref(), + messages, + )) + .map(|(command, connectivity)| { + RefResult::of(command, apply_one(repo, command, connectivity, messages)) + }) + .collect() + } + } + }, + }; + report(&unpack, &results) + .map(|report| pkt::frame_report(&report, &[], parsed.caps.side_band_64k)) +} + +fn apply_one( + repo: &Repo, + command: &ReceiveCommand, + connectivity: Option, + messages: &RejectMessages, +) -> Option { + if let Some(reason) = forbidden_ref(command, messages) { + return Some(reason); + } + if let Some(reason) = connectivity { + return Some(reason); + } + command + .to_update() + .and_then(|update| repo.update_ref(&update).map_err(PackError::from)) + .err() + .map(|error| error.to_string().replace('\n', " ")) +} + +fn atomic_failure( + snapshot: &RefSnapshot, + commands: &[ReceiveCommand], + messages: &RejectMessages, +) -> Vec { + let conflict = first_conflict(snapshot, commands, messages); + commands + .iter() + .map(|command| match &conflict { + Some(conflict) if conflict.refname == command.refname() => { + RefResult::of(command, Some(conflict.reason.clone())) + } + _ => RefResult::of(command, Some(messages.atomic_failed.text())), + }) + .collect() +} + +fn apply_atomic( + repo: &Repo, + snapshot: &RefSnapshot, + commands: &[ReceiveCommand], + haves: &HaveOids, + closure: Option<&objects::FreshClosure>, + messages: &RejectMessages, +) -> Vec { + let fail = |reason: String| all_failed(commands, &reason); + if commands + .iter() + .any(|command| forbidden_ref(command, messages).is_some()) + { + return commands + .iter() + .map(|command| { + let reason = forbidden_ref(command, messages) + .unwrap_or_else(|| messages.atomic_aborted.text()); + RefResult::of(command, Some(reason)) + }) + .collect(); + } + if let Some(command) = commands + .iter() + .zip(connectivity_reasons( + repo, commands, haves, closure, messages, + )) + .find_map(|(command, reason)| reason.map(|_| command)) + { + return fail( + messages + .missing_objects_for + .line(|RefKey::Ref| command.refname().to_string()), + ); + } + let updates = match commands + .iter() + .map(ReceiveCommand::to_update) + .collect::, _>>() + { + Ok(updates) => updates, + Err(error) => return fail(error.to_string().replace('\n', " ")), + }; + match repo.update_refs(&updates) { + Ok(()) => commands + .iter() + .map(|command| RefResult::of(command, None)) + .collect(), + Err(_) => atomic_failure(snapshot, commands, messages), + } +} + +fn report(unpack: &Result<(), PackError>, results: &[RefResult]) -> Result, PackError> { + let mut buf = Vec::new(); + match unpack { + Ok(()) => pkt::write_data(&mut buf, b"unpack ok\n")?, + Err(error) => pkt::write_data( + &mut buf, + format!("unpack {}\n", error.to_string().replace('\n', " ")).as_bytes(), + )?, + } + results.iter().try_fold(&mut buf, |buf, result| { + let line = match &result.failure { + None => format!("ok {}\n", result.refname), + Some(reason) => format!("ng {} {reason}\n", result.refname), + }; + pkt::write_data(buf, line.as_bytes())?; + Ok::<_, PackError>(buf) + })?; + pkt::write_flush(&mut buf)?; + Ok(buf) +} + +fn parse_commands(parsed: &pkt::Receive) -> Vec { + parsed + .commands + .iter() + .enumerate() + .filter_map(|(index, line)| ReceiveCommand::parse(line, index == 0)) + .collect() +} + +pub struct Preflight { + pub creates_branch: bool, +} + +pub(crate) fn preflight(body: &[u8]) -> Preflight { + pkt::split_receive(body) + .map(|parsed| { + let commands = parse_commands(&parsed); + Preflight { + creates_branch: commands.iter().any(|command| { + command.is_create() && command.name().is_some_and(knot_git::is_branch) + }), + } + }) + .unwrap_or(Preflight { + creates_branch: false, + }) +} + +fn all_failed(commands: &[ReceiveCommand], reason: &str) -> Vec { + commands + .iter() + .map(|command| RefResult::of(command, Some(reason.to_string()))) + .collect() +} + +fn stage_to_quarantine(staged: &Repo, commands: &[ReceiveCommand]) { + commands + .iter() + .filter(|command| !command.is_delete()) + .for_each(|command| { + if let Some(name) = command.name() { + let _ = staged.update_ref(&RefUpdate::Create { + name: name.clone(), + new: command.new, + }); + } + }); +} + +fn apply_guarded_atomic( + txn: &RefTxn<'_>, + snapshot: &RefSnapshot, + commands: &[ReceiveCommand], + seal: &dyn Fn(&[RefUpdate]), + messages: &RejectMessages, +) -> Vec { + let updates = match commands + .iter() + .map(ReceiveCommand::to_update) + .collect::, _>>() + { + Ok(updates) => updates, + Err(error) => return all_failed(commands, &error.to_string().replace('\n', " ")), + }; + match txn.update_refs(&updates) { + Ok(()) => { + seal(&updates); + commands + .iter() + .map(|command| RefResult::of(command, None)) + .collect() + } + Err(_) => atomic_failure(snapshot, commands, messages), + } +} + +fn apply_guarded_update( + txn: &RefTxn<'_>, + command: &ReceiveCommand, + seal: &dyn Fn(&[RefUpdate]), +) -> Option { + let update = match command.to_update() { + Ok(update) => update, + Err(error) => return Some(error.to_string().replace('\n', " ")), + }; + match txn.update_ref(&update) { + Ok(()) => { + seal(std::slice::from_ref(&update)); + None + } + Err(error) => Some(PackError::from(error).to_string().replace('\n', " ")), + } +} + +fn parse_push_options(parsed: &pkt::Receive) -> PushOptions { + PushOptions::new(parsed.options.iter().filter_map(|option| { + PushOption::new( + String::from_utf8_lossy(option).trim_matches(|byte: char| byte == '\n' || byte == '\r'), + ) + .ok() + })) +} + +#[allow(clippy::too_many_arguments)] +pub fn handle_guarded( + live: &Repo, + body: &[u8], + pack: Result, PackError>, + limits: &PackLimits, + guard: &dyn ReceiveGuard, + seal: &dyn Fn(&[RefUpdate]), + messages: &RejectMessages, +) -> Result { + let parsed = pkt::split_receive(body)?; + let atomic = parsed.caps.atomic; + let side_band = parsed.caps.side_band_64k; + let push_options = parse_push_options(&parsed); + let build = |report: Vec| ReceiveOutcome { + report, + side_band, + push_options: push_options.clone(), + }; + let commands = parse_commands(&parsed); + if commands.is_empty() { + return report(&Ok(()), &[]).map(build); + } + + let pack = match pack { + Ok(pack) => pack, + Err(error) => { + let results = all_failed(&commands, &messages.unpacker_error.text()); + return report(&Err(error), &results).map(build); + } + }; + let (quarantine, closure) = match Quarantine::stage( + live, + pack, + limits, + live.object_format().kind(), + is_empty(live), + ) { + Ok(staged) => staged, + Err(error) => { + let results = all_failed(&commands, &messages.unpacker_error.text()); + return report(&Err(error), &results).map(build); + } + }; + let staged = quarantine.repo(); + stage_to_quarantine(staged, &commands); + let snapshot = match RefSnapshot::capture(live) { + Ok(snapshot) => snapshot, + Err(_) => { + let results = all_failed(&commands, &messages.ref_snapshot_unavailable.text()); + return report(&Ok(()), &results).map(build); + } + }; + let haves = snapshot.tips(); + + let verdicts = guard.authorize(staged, &commands); + let reasons: Vec> = if verdicts.len() == commands.len() { + commands + .iter() + .zip(verdicts) + .zip(connectivity_reasons( + staged, + &commands, + &haves, + closure.as_ref(), + messages, + )) + .map(|((command, verdict), connectivity)| match verdict { + RefDecision::Reject(reason) => Some(reason), + RefDecision::Allow => reserved_create_only(command, messages) + .or(connectivity) + .or_else(|| snapshot.conflict(command, messages)), + }) + .collect() + } else { + commands + .iter() + .map(|_| Some(messages.authorization_unavailable.text())) + .collect() + }; + + let any_reject = reasons.iter().any(Option::is_some); + if atomic && any_reject { + let results = commands + .iter() + .zip(reasons) + .map(|(command, reason)| { + RefResult::of( + command, + Some(reason.unwrap_or_else(|| messages.atomic_aborted.text())), + ) + }) + .collect::>(); + return report(&Ok(()), &results).map(build); + } + + let applied = live.with_ref_txn(|txn| { + if reasons.iter().any(Option::is_none) { + quarantine.migrate_into(live)?; + } + let results = if atomic { + apply_guarded_atomic(txn, &snapshot, &commands, seal, messages) + } else { + commands + .iter() + .zip(reasons) + .map(|(command, reason)| match reason { + Some(reason) => RefResult::of(command, Some(reason)), + None => RefResult::of(command, apply_guarded_update(txn, command, seal)), + }) + .collect() + }; + Ok::<_, PackError>(results) + }); + match applied { + Ok(results) => report(&Ok(()), &results).map(build), + Err(error) => { + let results = all_failed(&commands, &messages.object_migration_failed.text()); + report(&Err(error), &results).map(build) + } + } +} + +#[cfg(test)] +mod tests { + use super::parse_push_options; + use crate::pkt; + use knot_types::{PushOption, PushOptions}; + + fn options(raw: &[&[u8]]) -> PushOptions { + parse_push_options(&pkt::Receive { + commands: Vec::new(), + options: raw.to_vec(), + pack: &[], + caps: pkt::Caps::default(), + }) + } + + #[test] + fn a_push_option_the_lexicon_rejects_never_reaches_the_event() { + let long = vec![b'x'; 1025]; + let parsed = options(&[b"verbose-ci\n", b"", &long, b"has\nnewline", b"ci-skip\r"]); + assert_eq!( + parsed + .as_slice() + .iter() + .map(PushOption::as_str) + .collect::>(), + vec!["verbose-ci", "ci-skip"], + "parsing trims trailing end-of-line and rejects empty, oversized, and multi-line options" + ); + + let raw: Vec> = (0..PushOptions::MAX + 10) + .map(|index| format!("option-{index}").into_bytes()) + .collect(); + let borrowed: Vec<&[u8]> = raw.iter().map(Vec::as_slice).collect(); + assert_eq!( + options(&borrowed).as_slice().len(), + PushOptions::MAX, + "directives parse from the same truncated list the event reports" + ); + } +} diff --git a/knot2/crates/knot-pack/src/receiver.rs b/knot2/crates/knot-pack/src/receiver.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/src/receiver.rs @@ -0,0 +1,162 @@ +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; + +use tempfile::NamedTempFile; + +use crate::error::PackError; +use crate::frame::ReceiveFramer; +use crate::ids::MaxWireBytes; +use crate::meter::PackLimits; + +#[derive(Debug)] +pub enum ReceiveReadError { + Io(std::io::Error), + Pack(PackError), + TooLarge, + Truncated, +} + +pub struct ReceivedPack { + preamble: Vec, + pack: Option, + kind: gix::hash::Kind, + total_len: usize, +} + +impl ReceivedPack { + pub fn preamble(&self) -> &[u8] { + &self.preamble + } + + pub fn len(&self) -> usize { + self.total_len + } + + pub fn is_empty(&self) -> bool { + self.total_len == 0 + } + + pub fn open_pack(&self) -> Result, PackError> { + match &self.pack { + Some(tmp) => gix_pack::data::File::at(tmp.path(), self.kind) + .map(Some) + .map_err(|error| PackError::Pack(error.to_string())), + None => Ok(None), + } + } +} + +pub struct PackReceiver { + dir: PathBuf, + file: NamedTempFile, + framer: ReceiveFramer, + written: u64, + limit: MaxWireBytes, + complete: Option, + kind: gix::hash::Kind, +} + +impl PackReceiver { + pub fn new( + dir: &Path, + limit: MaxWireBytes, + limits: PackLimits, + kind: gix::hash::Kind, + ) -> std::io::Result { + Ok(Self { + dir: dir.to_path_buf(), + file: NamedTempFile::new_in(dir)?, + framer: ReceiveFramer::new(limits, kind), + written: 0, + limit, + complete: None, + kind, + }) + } + + pub fn write(&mut self, chunk: &[u8]) -> Result { + if self.complete.is_some() { + return Ok(true); + } + if self.written as usize + chunk.len() > self.limit.get() { + return Err(ReceiveReadError::TooLarge); + } + self.file.write_all(chunk).map_err(ReceiveReadError::Io)?; + self.written += chunk.len() as u64; + self.scan() + } + + fn scan(&mut self) -> Result { + if self.written == 0 { + return Ok(false); + } + self.file.flush().map_err(ReceiveReadError::Io)?; + match self + .framer + .advance_file(self.file.as_file(), self.written) + .map_err(ReceiveReadError::Pack)? + { + Some(total) => { + self.complete = Some(total); + Ok(true) + } + None => Ok(false), + } + } + + pub fn finish(mut self) -> Result { + let total = match self.complete { + Some(total) => total, + None => { + if self.written == 0 { + 0 + } else if self.scan()? { + self.complete.expect("scan recorded completion") + } else { + return Err(ReceiveReadError::Truncated); + } + } + }; + let pack_start = self.framer.pack_start().unwrap_or(total); + let preamble = + read_range(self.file.as_file(), 0..pack_start).map_err(ReceiveReadError::Io)?; + let pack = match total > pack_start { + true => { + let mut tmp = NamedTempFile::new_in(&self.dir).map_err(ReceiveReadError::Io)?; + copy_range( + self.file.as_file(), + pack_start as u64..total as u64, + tmp.as_file_mut(), + ) + .map_err(ReceiveReadError::Io)?; + tmp.flush().map_err(ReceiveReadError::Io)?; + Some(tmp) + } + false => None, + }; + Ok(ReceivedPack { + preamble, + pack, + kind: self.kind, + total_len: total, + }) + } +} + +fn read_range(file: &std::fs::File, range: std::ops::Range) -> std::io::Result> { + use std::os::unix::fs::FileExt; + let mut buf = vec![0u8; range.end.saturating_sub(range.start)]; + file.read_exact_at(&mut buf, range.start as u64)?; + Ok(buf) +} + +fn copy_range( + src: &std::fs::File, + range: std::ops::Range, + dst: &mut std::fs::File, +) -> std::io::Result<()> { + let mut reader = src.try_clone()?; + reader.seek(SeekFrom::Start(range.start))?; + std::io::copy(&mut reader.take(range.end.saturating_sub(range.start)), dst)?; + Ok(()) +} diff --git a/knot2/crates/knot-pack/src/resolve.rs b/knot2/crates/knot-pack/src/resolve.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/src/resolve.rs @@ -0,0 +1,336 @@ +use std::collections::HashMap; +use std::path::Path; + +use gix::ObjectId; +use gix::object::Kind; +use gix::objs::{Find, Write}; +use gix_pack::data::{Entry, entry::Header}; + +use crate::error::{PackError, PackLimit}; +use crate::ids::{DeltaDepth, PackOffset, Rounds}; +use crate::meter::{PackLimits, inflate_into, pack_object_count}; + +struct Raw { + offset: PackOffset, + header: Header, + data: Vec, +} + +#[derive(Clone, Copy)] +struct Resolved { + oid: ObjectId, + depth: DeltaDepth, +} + +fn malformed(message: &str) -> PackError { + PackError::Pack(message.to_string()) +} + +pub fn resolve( + objects_dir: &Path, + pack: &[u8], + limits: &PackLimits, + kind: gix::hash::Kind, +) -> Result<(), PackError> { + let mut raws = parse_entries(pack, kind)?; + let odb = gix::odb::at_opts( + objects_dir, + std::iter::empty(), + gix::odb::store::init::Options { + object_hash: kind, + ..Default::default() + }, + ) + .map_err(|error| PackError::Pack(error.to_string()))?; + let mut done: HashMap = HashMap::new(); + let mut by_oid: HashMap = HashMap::new(); + resolve_rounds( + &mut raws, + &odb, + &mut done, + &mut by_oid, + limits.max_delta_depth, + Rounds::new(limits.max_delta_depth.get() + 2), + ) +} + +fn parse_entries(pack: &[u8], kind: gix::hash::Kind) -> Result, PackError> { + let hash_len = kind.len_in_bytes(); + let trailer = pack + .len() + .checked_sub(hash_len) + .ok_or_else(|| malformed("packfile is truncated"))?; + let num_objects = pack_object_count(pack)?; + + (0..num_objects.get()) + .try_fold((Vec::new(), PackOffset::new(12)), |(mut acc, offset), _| { + let mut reader: &[u8] = pack + .get(offset.get() as usize..trailer) + .ok_or_else(|| malformed("entry offset past pack end"))?; + let entry = Entry::from_read(&mut reader, offset.get(), hash_len) + .map_err(|error| PackError::Pack(error.to_string()))?; + let data_start = entry.data_offset as usize; + let mut data = Vec::with_capacity(entry.decompressed_size as usize); + let consumed = inflate_into( + pack.get(data_start..) + .ok_or_else(|| malformed("entry body past pack end"))?, + entry.decompressed_size, + &mut data, + )?; + acc.push(Raw { + offset, + header: entry.header, + data, + }); + Ok::<_, PackError>((acc, PackOffset::new(entry.data_offset + consumed))) + }) + .map(|(acc, _)| acc) +} + +fn resolve_rounds( + raws: &mut [Raw], + odb: &gix::odb::Handle, + done: &mut HashMap, + by_oid: &mut HashMap, + max_depth: DeltaDepth, + rounds_left: Rounds, +) -> Result<(), PackError> { + let pending: Vec = (0..raws.len()) + .filter(|index| !done.contains_key(&raws[*index].offset)) + .collect(); + if pending.is_empty() { + return Ok(()); + } + let Some(remaining) = rounds_left.next() else { + return Err(PackError::LimitExceeded(PackLimit::DeltaDepth)); + }; + let progressed = pending.iter().try_fold(false, |progressed, &index| { + match resolve_one(&raws[index], odb, done, by_oid, max_depth)? { + Some((kind, data, depth)) => { + let oid = odb + .write_buf(kind, &data) + .map_err(|error| PackError::Pack(error.to_string()))?; + by_oid.insert(oid, depth); + done.insert(raws[index].offset, Resolved { oid, depth }); + raws[index].data = Vec::new(); + Ok::<_, PackError>(true) + } + None => Ok(progressed), + } + })?; + if !progressed { + return Err(malformed("pack contains unresolvable delta")); + } + resolve_rounds(raws, odb, done, by_oid, max_depth, remaining) +} + +fn apply_delta_checked( + base_kind: Kind, + base_data: &[u8], + base_depth: DeltaDepth, + delta: &[u8], + max_depth: DeltaDepth, +) -> Result<(Kind, Vec, DeltaDepth), PackError> { + let depth = base_depth.deeper(); + if depth.exceeds(max_depth) { + return Err(PackError::LimitExceeded(PackLimit::DeltaDepth)); + } + Ok((base_kind, apply_delta(base_data, delta)?, depth)) +} + +fn resolve_one( + raw: &Raw, + odb: &gix::odb::Handle, + done: &HashMap, + by_oid: &HashMap, + max_depth: DeltaDepth, +) -> Result, DeltaDepth)>, PackError> { + match &raw.header { + Header::Commit | Header::Tree | Header::Blob | Header::Tag => { + let kind = raw + .header + .as_kind() + .ok_or_else(|| malformed("base entry has no object kind"))?; + Ok(Some((kind, raw.data.clone(), DeltaDepth::ZERO))) + } + Header::OfsDelta { base_distance } => { + let base_offset = raw + .offset + .checked_sub_distance(*base_distance) + .filter(|_| *base_distance != 0) + .ok_or_else(|| malformed("ofs-delta base out of range"))?; + match done.get(&base_offset) { + Some(base) => { + let mut buf = Vec::new(); + let found = odb + .try_find(&base.oid, &mut buf) + .map_err(|error| PackError::Pack(error.to_string()))? + .ok_or_else(|| malformed("ofs-delta base missing from odb"))?; + apply_delta_checked(found.kind, found.data, base.depth, &raw.data, max_depth) + .map(Some) + } + None => Ok(None), + } + } + Header::RefDelta { base_id } => { + let base_depth = by_oid.get(base_id).copied(); + let mut buf = Vec::new(); + match odb + .try_find(base_id, &mut buf) + .map_err(|error| PackError::Pack(error.to_string()))? + { + Some(base) => apply_delta_checked( + base.kind, + base.data, + base_depth.unwrap_or(DeltaDepth::ZERO), + &raw.data, + max_depth, + ) + .map(Some), + None => Ok(None), + } + } + } +} + +enum Op<'a> { + Copy { start: usize, len: usize }, + Insert(&'a [u8]), +} + +fn read_varint(delta: &[u8], pos: &mut usize) -> Result { + let mut shift = 0u32; + let mut value = 0u64; + loop { + let byte = *delta + .get(*pos) + .ok_or_else(|| malformed("delta size header truncated"))?; + *pos += 1; + value |= u64::from(byte & 0x7f) << shift; + shift += 7; + if byte & 0x80 == 0 { + return Ok(value); + } + if shift >= u64::BITS { + return Err(malformed("delta size header overflows")); + } + } +} + +fn assemble( + cmd: u8, + bit_base: u32, + count: u32, + delta: &[u8], + pos: &mut usize, +) -> Result { + (0..count).try_fold(0u64, |acc, index| { + if cmd & (1 << (bit_base + index)) == 0 { + return Ok(acc); + } + let byte = *delta + .get(*pos) + .ok_or_else(|| malformed("delta copy operand truncated"))?; + *pos += 1; + Ok(acc | (u64::from(byte) << (8 * index))) + }) +} + +fn ops<'a>(delta: &'a [u8]) -> impl Iterator, PackError>> { + let mut pos = 0usize; + std::iter::from_fn(move || { + (pos < delta.len()).then(|| { + let cmd = delta[pos]; + pos += 1; + if cmd & 0x80 != 0 { + let offset = assemble(cmd, 0, 4, delta, &mut pos)?; + let raw_size = assemble(cmd, 4, 3, delta, &mut pos)?; + let size = if raw_size == 0 { 0x10000 } else { raw_size }; + Ok(Op::Copy { + start: offset as usize, + len: size as usize, + }) + } else if cmd != 0 { + let len = cmd as usize; + let bytes = delta + .get(pos..pos + len) + .ok_or_else(|| malformed("delta insert truncated"))?; + pos += len; + Ok(Op::Insert(bytes)) + } else { + Err(malformed("delta uses reserved opcode 0")) + } + }) + }) +} + +fn apply_delta(base: &[u8], delta: &[u8]) -> Result, PackError> { + let mut pos = 0usize; + let base_size = read_varint(delta, &mut pos)?; + if base_size as usize != base.len() { + return Err(malformed("delta base size doesn't match its base object")); + } + let target_size = read_varint(delta, &mut pos)?; + let out = + ops(&delta[pos..]).try_fold(Vec::with_capacity(target_size as usize), |mut out, op| { + match op? { + Op::Copy { start, len } => { + let end = start + .checked_add(len) + .ok_or_else(|| malformed("delta copy range overflows"))?; + out.extend_from_slice( + base.get(start..end) + .ok_or_else(|| malformed("delta copy reads past base"))?, + ); + } + Op::Insert(bytes) => out.extend_from_slice(bytes), + } + Ok::<_, PackError>(out) + })?; + if out.len() as u64 != target_size { + return Err(malformed("delta produced wrong target size")); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn apply_delta_reconstructs_copy_and_insert() { + let base = b"hello world"; + let delta = [0x0b, 0x06, 0x90, 0x05, 0x01, b'!']; + assert_eq!(apply_delta(base, &delta).unwrap(), b"hello!"); + } + + #[test] + fn apply_delta_rejects_a_base_size_mismatch() { + let delta = [0x05, 0x00]; + assert!(apply_delta(b"hi", &delta).is_err()); + } + + #[test] + fn apply_delta_rejects_a_copy_past_the_base() { + let delta = [0x02, 0x10, 0x90, 0xff]; + assert!(apply_delta(b"hi", &delta).is_err()); + } + + #[test] + fn apply_delta_rejects_a_truncated_insert() { + let delta = [0x02, 0x05, 0x05, b'a', b'b']; + assert!(apply_delta(b"hi", &delta).is_err()); + } + + #[test] + fn apply_delta_rejects_the_reserved_opcode() { + let delta = [0x02, 0x00, 0x00]; + assert!(apply_delta(b"hi", &delta).is_err()); + } + + #[test] + fn apply_delta_rejects_a_truncated_size_header() { + let delta = [0x80]; + assert!(apply_delta(b"", &delta).is_err()); + } +} diff --git a/knot2/crates/knot-pack/src/upload.rs b/knot2/crates/knot-pack/src/upload.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/src/upload.rs @@ -0,0 +1,979 @@ +use std::collections::HashSet; +use std::io::{self, Read, Write}; +use std::sync::OnceLock; +use std::time::Duration; + +use knot_git::{ + CommitDepth, Deepen, Filter, Haves, PackBudget, PackfileUri, Repo, ShallowCommits, Wants, +}; +use knot_messages::{CountKey, FetchMessages, KnotKey}; +use knot_types::{KnotHostname, ObjectCount, ObjectFormat, Oid, UnixSeconds}; + +use crate::error::PackError; +use crate::objects; +use crate::pkt; +use crate::{HaveOids, WantOids}; + +const AGENT: &[u8] = b"agent=knot/0\n"; +const SELECTION_MAX_OBJECTS: ObjectCount = ObjectCount::new(16_000_000); +const SELECTION_TIME_BUDGET: Duration = Duration::from_secs(600); + +#[derive(Debug, Clone, Copy)] +pub struct SelectionLimits { + pub max_objects: ObjectCount, + pub time_budget: Duration, +} + +impl Default for SelectionLimits { + fn default() -> Self { + Self { + max_objects: SELECTION_MAX_OBJECTS, + time_budget: SELECTION_TIME_BUDGET, + } + } +} + +static SELECTION_LIMITS: OnceLock = OnceLock::new(); + +pub fn init_selection_limits(limits: SelectionLimits) { + if SELECTION_LIMITS.set(limits).is_err() { + debug_assert!(false, "selection limits initialized more than once"); + } +} + +fn selection_limits() -> SelectionLimits { + SELECTION_LIMITS.get().copied().unwrap_or_default() +} +const V0_CAPS_BASE: &str = "multi_ack_detailed no-done side-band-64k ofs-delta shallow deepen-since deepen-not filter agent=knot/0"; + +fn v0_caps(format: ObjectFormat) -> String { + format!("{V0_CAPS_BASE} object-format={}", format.capability()) +} + +pub struct StreamOpts { + pub side_band: bool, + pub sideband_all: bool, + pub no_progress: bool, + pub thin: bool, + pub filter: Filter, + pub shallow_commits: Option>, + pub packfile_uris: Vec, + pub emit_packfile_header: bool, +} + +pub enum UploadOutcome { + Buffered(Vec), + Streaming { + preamble: Vec, + wants: WantOids, + haves: HaveOids, + opts: StreamOpts, + }, +} + +fn write_v2_caps(buf: &mut Vec, format: ObjectFormat) -> Result<(), PackError> { + pkt::write_data(buf, b"version 2\n")?; + pkt::write_data(buf, AGENT)?; + pkt::write_data(buf, b"ls-refs\n")?; + pkt::write_data( + buf, + b"fetch=shallow filter wait-for-done packfile-uris sideband-all\n", + )?; + pkt::write_data(buf, b"server-option\n")?; + pkt::write_data( + buf, + format!("object-format={}\n", format.capability()).as_bytes(), + )?; + pkt::write_flush(buf)?; + Ok(()) +} + +pub fn advertise(repo: &Repo) -> Result, PackError> { + let mut buf = Vec::new(); + pkt::write_data(&mut buf, b"# service=git-upload-pack\n")?; + pkt::write_flush(&mut buf)?; + write_v2_caps(&mut buf, repo.object_format())?; + Ok(buf) +} + +pub fn advertise_ssh(repo: &Repo) -> Result, PackError> { + let mut buf = Vec::new(); + write_v2_caps(&mut buf, repo.object_format())?; + Ok(buf) +} + +pub fn advertise_v0(repo: &Repo) -> Result, PackError> { + let mut buf = Vec::new(); + pkt::write_data(&mut buf, b"# service=git-upload-pack\n")?; + pkt::write_flush(&mut buf)?; + write_v0_advert(&mut buf, repo)?; + Ok(buf) +} + +pub fn advertise_v0_ssh(repo: &Repo) -> Result, PackError> { + let mut buf = Vec::new(); + write_v0_advert(&mut buf, repo)?; + Ok(buf) +} + +fn write_v0_advert(buf: &mut Vec, repo: &Repo) -> Result<(), PackError> { + let format = repo.object_format(); + let caps = v0_caps(format); + let refs = repo.advertised_refs_for(knot_git::AdvertScope::Upload)?; + match repo.head() { + Some(head) => { + pkt::write_data( + buf, + format!("{} HEAD\0{caps} symref=HEAD:{}\n", head.target, head.name).as_bytes(), + )?; + write_plain_refs(buf, &refs)?; + } + None => match refs.split_first() { + Some((first, rest)) => { + pkt::write_data( + buf, + format!("{} {}\0{caps}\n", first.target, first.name).as_bytes(), + )?; + write_plain_refs(buf, rest)?; + } + None => { + pkt::write_data( + buf, + format!("{} capabilities^{{}}\0{caps}\n", format.null_oid()).as_bytes(), + )?; + } + }, + } + pkt::write_flush(buf)?; + Ok(()) +} + +fn write_plain_refs(buf: &mut Vec, refs: &[knot_git::RefRecord]) -> Result<(), PackError> { + refs.iter().try_for_each(|record| { + pkt::write_data( + buf, + format!("{} {}\n", record.target, record.name).as_bytes(), + ) + .map_err(PackError::from) + }) +} + +pub(crate) fn fuzz(body: &[u8]) { + if let Ok(lines) = pkt::data_payloads_all(body) { + let _ = parse_wants(&lines); + let _ = parse_oids(&lines, b"want "); + let _ = parse_oids(&lines, b"have "); + let _ = first_caps(&lines); + let _ = parse_ls_refs_args(&lines); + } +} + +pub fn plan(repo: &Repo, body: &[u8]) -> Result { + let peek = pkt::data_payloads(body)?; + match peek.first() { + Some(line) if line.starts_with(b"command=") => plan_v2(repo, &peek), + _ => plan_v0(repo, body), + } +} + +pub fn buffered( + repo: &Repo, + body: &[u8], + messages: &FetchMessages, + knot: &KnotHostname, +) -> Result, PackError> { + let mut out = Vec::new(); + streamed(repo, body, messages, knot, &mut |chunk| { + out.extend_from_slice(chunk); + Ok(()) + })?; + Ok(out) +} + +pub fn streamed( + repo: &Repo, + body: &[u8], + messages: &FetchMessages, + knot: &KnotHostname, + sink: &mut dyn FnMut(&[u8]) -> io::Result<()>, +) -> Result<(), PackError> { + match plan(repo, body)? { + UploadOutcome::Buffered(bytes) => sink(&bytes).map_err(PackError::from), + UploadOutcome::Streaming { + preamble, + wants, + haves, + opts, + } => { + sink(&preamble)?; + stream_pack(repo, &wants, &haves, &opts, messages, knot, sink)?; + if opts.side_band { + let mut flush = Vec::new(); + pkt::write_flush(&mut flush)?; + sink(&flush)?; + } + Ok(()) + } + } +} + +fn plan_v2(repo: &Repo, lines: &[&[u8]]) -> Result { + match lines.first().copied().unwrap_or_default() { + command if command.starts_with(b"command=ls-refs") => { + Ok(UploadOutcome::Buffered(ls_refs(repo, lines)?)) + } + command if command.starts_with(b"command=fetch") => plan_v2_fetch(repo, lines), + _ => Err(PackError::Protocol( + "unsupported protocol v2 command".to_string(), + )), + } +} + +struct LsRefsArgs { + symrefs: bool, + peel: bool, + prefixes: Vec, +} + +fn parse_ls_refs_args(lines: &[&[u8]]) -> LsRefsArgs { + LsRefsArgs { + symrefs: lines.iter().any(|line| line.starts_with(b"symrefs")), + peel: lines.iter().any(|line| line.starts_with(b"peel")), + prefixes: lines + .iter() + .filter_map(|line| { + std::str::from_utf8(line) + .ok()? + .trim_end() + .strip_prefix("ref-prefix ") + .map(str::to_string) + }) + .collect(), + } +} + +pub(crate) fn matches_prefix>(name: &str, prefixes: &[S]) -> bool { + prefixes.is_empty() + || prefixes + .iter() + .any(|prefix| name.starts_with(prefix.as_ref())) +} + +fn ls_refs(repo: &Repo, lines: &[&[u8]]) -> Result, PackError> { + let args = parse_ls_refs_args(lines); + let mut buf = Vec::new(); + if let Some(head) = repo.head() + && matches_prefix("HEAD", &args.prefixes) + { + let mut line = format!("{} HEAD", head.target); + if args.symrefs { + line.push_str(&format!(" symref-target:{}", head.name)); + } + line.push('\n'); + pkt::write_data(&mut buf, line.as_bytes())?; + } + repo.advertised_refs_for(knot_git::AdvertScope::Upload)? + .iter() + .filter(|record| matches_prefix(record.name.as_str(), &args.prefixes)) + .try_fold(&mut buf, |buf, record| { + let mut line = format!("{} {}", record.target, record.name); + if args.peel + && let Some(peeled) = repo.peeled_target(record.target)? + { + line.push_str(&format!(" peeled:{peeled}")); + } + line.push('\n'); + pkt::write_data(buf, line.as_bytes())?; + Ok::<_, PackError>(buf) + })?; + pkt::write_flush(&mut buf)?; + Ok(buf) +} + +fn plan_v2_fetch(repo: &Repo, lines: &[&[u8]]) -> Result { + let wants = parse_wants(lines)?; + ensure_wanted(repo, &wants)?; + let haves = HaveOids::new(parse_oids(lines, b"have ")); + let done = lines.iter().any(|line| line.starts_with(b"done")); + let wait_for_done = lines.iter().any(|line| line.starts_with(b"wait-for-done")); + let sideband_all = lines.iter().any(|line| line.starts_with(b"sideband-all")); + let no_progress = lines.iter().any(|line| line.starts_with(b"no-progress")); + let thin = lines.iter().any(|line| line.starts_with(b"thin-pack")); + let filter = parse_filter(lines)?; + let deepen = parse_deepen(repo, lines)?; + let client_shallow = parse_oids(lines, b"shallow "); + let common: HaveOids = haves + .iter() + .copied() + .filter(|oid| repo.contains(*oid)) + .collect(); + + let mut preamble = Vec::new(); + if !haves.is_empty() && !done { + seg(&mut preamble, sideband_all, b"acknowledgments\n")?; + if common.is_empty() { + seg(&mut preamble, sideband_all, b"NAK\n")?; + pkt::write_flush(&mut preamble)?; + return Ok(UploadOutcome::Buffered(preamble)); + } + common.iter().try_for_each(|oid| { + seg( + &mut preamble, + sideband_all, + format!("ACK {oid}\n").as_bytes(), + ) + })?; + if wait_for_done { + pkt::write_flush(&mut preamble)?; + return Ok(UploadOutcome::Buffered(preamble)); + } + seg(&mut preamble, sideband_all, b"ready\n")?; + pkt::write_delim(&mut preamble)?; + } + + let shallow_commits = if deepen.is_shallow_request() || repo.is_shallow() { + let plan = + repo.shallow_walk(wants.wants(), &deepen, ShallowCommits::new(&client_shallow))?; + seg(&mut preamble, sideband_all, b"shallow-info\n")?; + plan.shallow.iter().try_for_each(|oid| { + seg( + &mut preamble, + sideband_all, + format!("shallow {oid}\n").as_bytes(), + ) + })?; + plan.unshallow.iter().try_for_each(|oid| { + seg( + &mut preamble, + sideband_all, + format!("unshallow {oid}\n").as_bytes(), + ) + })?; + pkt::write_delim(&mut preamble)?; + Some(plan.commits) + } else { + None + }; + + Ok(UploadOutcome::Streaming { + preamble, + wants, + haves: common, + opts: StreamOpts { + side_band: true, + sideband_all, + no_progress, + thin, + filter, + shallow_commits, + packfile_uris: packfile_uri_candidates(repo, lines), + emit_packfile_header: true, + }, + }) +} + +fn seg(buf: &mut Vec, sideband_all: bool, content: &[u8]) -> io::Result<()> { + if sideband_all { + pkt::write_band(buf, content) + } else { + pkt::write_data(buf, content) + } +} + +fn packfile_uri_candidates(repo: &Repo, lines: &[&[u8]]) -> Vec { + let Some(protocols) = lines.iter().find_map(|line| { + std::str::from_utf8(line) + .ok()? + .trim_end() + .strip_prefix("packfile-uris ") + .map(str::to_string) + }) else { + return Vec::new(); + }; + let allowed: Vec<&str> = protocols.split(',').map(str::trim).collect(); + repo.blob_packfile_uris() + .into_iter() + .filter(|candidate| { + candidate + .uri + .as_str() + .split_once("://") + .is_some_and(|(scheme, _)| allowed.contains(&scheme)) + }) + .collect() +} + +fn parse_filter(lines: &[&[u8]]) -> Result { + let spec = lines.iter().find_map(|line| { + std::str::from_utf8(line) + .ok()? + .trim_end() + .strip_prefix("filter ") + }); + match spec { + None => Ok(Filter::None), + Some("blob:none") => Ok(Filter::BlobNone), + Some(rest) if rest.starts_with("blob:limit=") => parse_size(&rest["blob:limit=".len()..]) + .map(Filter::BlobLimit) + .ok_or_else(|| PackError::Protocol(format!("bad blob:limit filter: {rest}"))), + Some(rest) if rest.starts_with("tree:") => rest["tree:".len()..] + .parse::() + .map(|depth| Filter::TreeDepth(knot_git::TreeDepth::new(depth))) + .map_err(|_| PackError::Protocol(format!("bad tree filter: {rest}"))), + Some(other) => Err(PackError::Protocol(format!("unsupported filter: {other}"))), + } +} + +fn parse_size(text: &str) -> Option { + let (digits, scale) = match text.chars().last() { + Some('k') | Some('K') => (&text[..text.len() - 1], 1024), + Some('m') | Some('M') => (&text[..text.len() - 1], 1024 * 1024), + Some('g') | Some('G') => (&text[..text.len() - 1], 1024 * 1024 * 1024), + _ => (text, 1), + }; + digits + .parse::() + .ok() + .and_then(|value| value.checked_mul(scale)) +} + +fn parse_deepen(repo: &Repo, lines: &[&[u8]]) -> Result { + let value = |prefix: &str| -> Option<&str> { + lines.iter().find_map(|line| { + std::str::from_utf8(line) + .ok()? + .trim_end() + .strip_prefix(prefix) + }) + }; + let depth = value("deepen ") + .and_then(|text| text.parse::().ok()) + .map(CommitDepth::new); + let since = value("deepen-since ") + .and_then(|text| text.parse::().ok()) + .map(UnixSeconds::new); + let relative = lines + .iter() + .any(|line| line.starts_with(b"deepen-relative")); + let not = lines + .iter() + .filter_map(|line| { + std::str::from_utf8(line) + .ok()? + .trim_end() + .strip_prefix("deepen-not ") + }) + .map(|spec| resolve_commitish(repo, spec)) + .collect::, _>>()?; + Ok(Deepen { + depth, + since, + not, + relative, + }) +} + +fn resolve_commitish(repo: &Repo, spec: &str) -> Result { + if let Ok(oid) = Oid::from_hex(spec) + && repo.contains(oid) + { + return Ok(oid); + } + let candidates = [ + spec.to_string(), + format!("refs/{spec}"), + format!("refs/tags/{spec}"), + format!("refs/heads/{spec}"), + ]; + repo.advertised_refs_for(knot_git::AdvertScope::Upload)? + .iter() + .find(|record| candidates.iter().any(|name| record.name.as_str() == name)) + .map(|record| record.target) + .ok_or_else(|| PackError::Protocol(format!("deepen-not {spec}: unknown ref"))) +} + +fn plan_v0(repo: &Repo, body: &[u8]) -> Result { + let lines = pkt::data_payloads_all(body)?; + let wants = parse_wants(&lines)?; + ensure_wanted(repo, &wants)?; + let haves = HaveOids::new(parse_oids(&lines, b"have ")); + let done = lines.iter().any(|line| line.starts_with(b"done")); + let caps = first_caps(&lines); + let side_band = caps + .map(|caps| { + caps.split(' ') + .any(|cap| cap == "side-band-64k" || cap == "side-band") + }) + .unwrap_or(false); + let no_progress = caps + .map(|caps| caps.split(' ').any(|cap| cap == "no-progress")) + .unwrap_or(false); + let thin = caps + .map(|caps| caps.split(' ').any(|cap| cap == "thin-pack")) + .unwrap_or(false); + let multi_ack_detailed = caps + .map(|caps| caps.split(' ').any(|cap| cap == "multi_ack_detailed")) + .unwrap_or(false); + let no_done = caps + .map(|caps| caps.split(' ').any(|cap| cap == "no-done")) + .unwrap_or(false); + let filter = parse_filter(&lines)?; + let deepen = parse_deepen(repo, &lines)?; + let client_shallow = parse_oids(&lines, b"shallow "); + let common: HaveOids = haves + .iter() + .copied() + .filter(|oid| repo.contains(*oid)) + .collect(); + + let mut preamble = Vec::new(); + let shallow_commits = if deepen.is_shallow_request() || repo.is_shallow() { + let plan = + repo.shallow_walk(wants.wants(), &deepen, ShallowCommits::new(&client_shallow))?; + plan.shallow.iter().try_for_each(|oid| { + pkt::write_data(&mut preamble, format!("shallow {oid}\n").as_bytes()) + })?; + plan.unshallow.iter().try_for_each(|oid| { + pkt::write_data(&mut preamble, format!("unshallow {oid}\n").as_bytes()) + })?; + pkt::write_flush(&mut preamble)?; + Some(plan.commits) + } else { + None + }; + + if haves.is_empty() && !done { + if !(deepen.is_shallow_request() || repo.is_shallow()) { + pkt::write_data(&mut preamble, b"NAK\n")?; + } + return Ok(UploadOutcome::Buffered(preamble)); + } + + let ready = multi_ack_detailed + && !done + && !common.is_empty() + && common.len() == haves.len() + && repo.wants_satisfied_by(wants.wants(), common.haves())?; + + let stream = move |preamble: Vec, common: HaveOids| UploadOutcome::Streaming { + preamble, + wants, + haves: common, + opts: StreamOpts { + side_band, + sideband_all: false, + no_progress, + thin, + filter, + shallow_commits, + packfile_uris: Vec::new(), + emit_packfile_header: false, + }, + }; + + if multi_ack_detailed { + common.iter().try_for_each(|oid| { + pkt::write_data(&mut preamble, format!("ACK {oid} common\n").as_bytes()) + })?; + let last = common.as_slice().last().copied(); + if done { + match last { + Some(oid) => { + pkt::write_data(&mut preamble, format!("ACK {oid}\n").as_bytes())?; + } + None => pkt::write_data(&mut preamble, b"NAK\n")?, + } + return Ok(stream(preamble, common)); + } + match (ready, last) { + (true, Some(oid)) => { + pkt::write_data(&mut preamble, format!("ACK {oid} ready\n").as_bytes())?; + pkt::write_data(&mut preamble, b"NAK\n")?; + if no_done { + pkt::write_data(&mut preamble, format!("ACK {oid}\n").as_bytes())?; + return Ok(stream(preamble, common)); + } + } + _ => pkt::write_data(&mut preamble, b"NAK\n")?, + } + return Ok(UploadOutcome::Buffered(preamble)); + } + + match common.as_slice().first() { + Some(oid) => pkt::write_data(&mut preamble, format!("ACK {oid}\n").as_bytes())?, + None => pkt::write_data(&mut preamble, b"NAK\n")?, + } + match done { + true => Ok(stream(preamble, common)), + false => Ok(UploadOutcome::Buffered(preamble)), + } +} + +pub fn stream_pack( + repo: &Repo, + wants: &WantOids, + haves: &HaveOids, + opts: &StreamOpts, + messages: &FetchMessages, + knot: &KnotHostname, + sink: &mut dyn FnMut(&[u8]) -> io::Result<()>, +) -> Result<(), PackError> { + let progress = opts.side_band && !opts.no_progress; + if opts.shallow_commits.is_none() + && haves.is_empty() + && opts.filter == Filter::None + && opts.packfile_uris.is_empty() + { + if let Ok(Some(pack)) = knot_git::verbatim_clone_pack(repo, wants.wants()) { + write_packfile_header(opts, sink)?; + return stream_verbatim_pack(pack, opts, progress, messages, knot, sink); + } + if let Ok(Some(oids)) = knot_git::reachable_via_bitmap(repo, wants.wants(), Haves::new(&[])) + { + write_packfile_header(opts, sink)?; + return stream_object_set(repo, oids, opts, progress, messages, knot, sink); + } + write_packfile_header(opts, sink)?; + return stream_full_clone(repo, wants.as_slice(), opts, progress, messages, knot, sink); + } + let budget = selection_budget(); + let mut selection = match &opts.shallow_commits { + Some(commits) => repo.select_shallow_objects( + wants.wants(), + ShallowCommits::new(commits), + haves.haves(), + opts.filter, + budget, + )?, + None => { + repo.select_pack_objects_filtered(wants.wants(), haves.haves(), opts.filter, budget)? + } + }; + if !opts.packfile_uris.is_empty() { + offload_packfile_uris( + &mut selection.send, + &opts.packfile_uris, + opts.sideband_all, + sink, + )?; + } + write_packfile_header(opts, sink)?; + let count = ObjectCount::new(selection.send.len()); + emit_preamble(progress, messages, knot, count, sink)?; + { + let mut pack_sink = PackSink { + side_band: opts.side_band, + sink: &mut *sink, + }; + let thin_bases = opts.thin.then_some(&selection.client_has); + objects::write_pack( + &repo.objects_dir(), + selection.send, + thin_bases, + &mut pack_sink, + repo.object_format().kind(), + )?; + } + emit_total(progress, messages, count, sink) +} + +fn emit_progress( + progress: bool, + lines: impl FnOnce() -> Vec, + sink: &mut dyn FnMut(&[u8]) -> io::Result<()>, +) -> Result<(), PackError> { + if !progress { + return Ok(()); + } + let rendered = lines(); + if rendered.is_empty() { + return Ok(()); + } + let mut buf = Vec::new(); + rendered.iter().try_for_each(|line| { + format!("{line}\n") + .into_bytes() + .chunks(pkt::MAX_BAND) + .try_for_each(|chunk| pkt::write_band_progress(&mut buf, chunk)) + })?; + sink(&buf).map_err(PackError::from) +} + +fn emit_preamble( + progress: bool, + messages: &FetchMessages, + knot: &KnotHostname, + count: ObjectCount, + sink: &mut dyn FnMut(&[u8]) -> io::Result<()>, +) -> Result<(), PackError> { + emit_progress( + progress, + || { + messages + .motd + .lines(|KnotKey::Knot| knot.as_str().to_string()) + .into_iter() + .chain( + messages + .enumerating + .lines(|CountKey::Count| count.to_string()), + ) + .collect() + }, + sink, + ) +} + +fn emit_total( + progress: bool, + messages: &FetchMessages, + count: ObjectCount, + sink: &mut dyn FnMut(&[u8]) -> io::Result<()>, +) -> Result<(), PackError> { + emit_progress( + progress, + || messages.total.lines(|CountKey::Count| count.to_string()), + sink, + ) +} + +fn write_packfile_header( + opts: &StreamOpts, + sink: &mut dyn FnMut(&[u8]) -> io::Result<()>, +) -> Result<(), PackError> { + if !opts.emit_packfile_header { + return Ok(()); + } + let mut buf = Vec::new(); + seg(&mut buf, opts.sideband_all, b"packfile\n")?; + sink(&buf)?; + Ok(()) +} + +fn offload_packfile_uris( + send: &mut Vec, + candidates: &[PackfileUri], + sideband_all: bool, + sink: &mut dyn FnMut(&[u8]) -> io::Result<()>, +) -> Result<(), PackError> { + let present: std::collections::HashSet = send.iter().copied().collect(); + let kept: Vec<&PackfileUri> = candidates + .iter() + .filter(|candidate| present.contains(&candidate.oid)) + .collect(); + if kept.is_empty() { + return Ok(()); + } + let excluded: std::collections::HashSet = + kept.iter().map(|candidate| candidate.oid).collect(); + send.retain(|oid| !excluded.contains(oid)); + let mut buf = Vec::new(); + seg(&mut buf, sideband_all, b"packfile-uris\n")?; + kept.iter().try_for_each(|candidate| { + seg( + &mut buf, + sideband_all, + format!( + "{} {}\n", + candidate.pack_hash.as_str(), + candidate.uri.as_str() + ) + .as_bytes(), + ) + })?; + pkt::write_delim(&mut buf)?; + sink(&buf)?; + Ok(()) +} + +fn stream_verbatim_pack( + mut file: std::fs::File, + opts: &StreamOpts, + progress: bool, + messages: &FetchMessages, + knot: &KnotHostname, + sink: &mut dyn FnMut(&[u8]) -> io::Result<()>, +) -> Result<(), PackError> { + let mut header = [0u8; 12]; + file.read_exact(&mut header)?; + if &header[..4] != b"PACK" { + return Err(PackError::Pack( + "reused pack is missing its PACK signature".to_string(), + )); + } + let count = ObjectCount::from(u32::from_be_bytes([ + header[8], header[9], header[10], header[11], + ])); + emit_preamble(progress, messages, knot, count, sink)?; + { + let mut pack_sink = PackSink { + side_band: opts.side_band, + sink: &mut *sink, + }; + pack_sink.write_all(&header)?; + io::copy(&mut file, &mut pack_sink)?; + } + emit_total(progress, messages, count, sink) +} + +fn stream_object_set( + repo: &Repo, + oids: Vec, + opts: &StreamOpts, + progress: bool, + messages: &FetchMessages, + knot: &KnotHostname, + sink: &mut dyn FnMut(&[u8]) -> io::Result<()>, +) -> Result<(), PackError> { + let count = ObjectCount::new(oids.len()); + emit_preamble(progress, messages, knot, count, sink)?; + { + let mut pack_sink = PackSink { + side_band: opts.side_band, + sink: &mut *sink, + }; + objects::write_pack( + &repo.objects_dir(), + oids, + None, + &mut pack_sink, + repo.object_format().kind(), + )?; + } + emit_total(progress, messages, count, sink) +} + +fn stream_full_clone( + repo: &Repo, + wants: &[Oid], + opts: &StreamOpts, + progress: bool, + messages: &FetchMessages, + knot: &KnotHostname, + sink: &mut dyn FnMut(&[u8]) -> io::Result<()>, +) -> Result<(), PackError> { + let limits = selection_limits(); + let stall = limits.time_budget; + let roots = repo.clone_roots(wants, PackBudget::new(limits.max_objects, stall))?; + let pack = objects::count_expanded( + &repo.objects_dir(), + roots, + limits.max_objects, + stall, + repo.object_format().kind(), + )?; + let count = ObjectCount::new(pack.len()); + emit_preamble(progress, messages, knot, count, sink)?; + { + let mut pack_sink = PackSink { + side_band: opts.side_band, + sink: &mut *sink, + }; + objects::write_expanded(pack, &mut pack_sink)?; + } + emit_total(progress, messages, count, sink) +} + +struct PackSink<'a> { + side_band: bool, + sink: &'a mut dyn FnMut(&[u8]) -> io::Result<()>, +} + +impl Write for PackSink<'_> { + fn write(&mut self, data: &[u8]) -> io::Result { + if self.side_band { + data.chunks(pkt::MAX_BAND).try_for_each(|chunk| { + let mut framed = Vec::with_capacity(chunk.len() + 5); + pkt::write_band(&mut framed, chunk)?; + (self.sink)(&framed) + })?; + } else { + (self.sink)(data)?; + } + Ok(data.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +pub fn selection_budget() -> PackBudget { + let limits = selection_limits(); + PackBudget::new(limits.max_objects, limits.time_budget) +} + +fn ensure_wanted(repo: &Repo, wants: &WantOids) -> Result<(), PackError> { + let tips: Vec = repo + .advertised_refs_for(knot_git::AdvertScope::Upload)? + .iter() + .map(|record| record.target) + .collect(); + let advertised: HashSet = tips.iter().copied().collect(); + if wants.iter().all(|want| advertised.contains(want)) { + return Ok(()); + } + let commit_closure = repo.reachable_commits(&tips, selection_budget())?; + let unresolved: Vec = wants + .iter() + .copied() + .filter(|want| !advertised.contains(want) && !commit_closure.contains(want)) + .collect(); + if unresolved.is_empty() { + return Ok(()); + } + let reachable: HashSet = repo + .select_pack_objects_filtered( + Wants::new(&tips), + Haves::new(&[]), + Filter::None, + selection_budget(), + )? + .send + .into_iter() + .collect(); + match unresolved.iter().find(|want| !reachable.contains(want)) { + Some(hidden) => Err(PackError::Protocol(format!( + "want {hidden} isn't reachable from public ref" + ))), + None => Ok(()), + } +} + +fn first_caps<'a>(lines: &[&'a [u8]]) -> Option<&'a str> { + let line = lines.iter().find(|line| line.starts_with(b"want "))?; + let text = std::str::from_utf8(line).ok()?.trim_end(); + text.strip_prefix("want ")? + .split_once(' ') + .map(|(_oid, caps)| caps) +} + +fn parse_wants(lines: &[&[u8]]) -> Result { + lines + .iter() + .filter_map(|line| line.strip_prefix(b"want ")) + .map(|rest| { + let hex = rest + .split(|byte| *byte == b' ' || *byte == b'\n') + .next() + .unwrap_or_default(); + std::str::from_utf8(hex) + .ok() + .and_then(|text| Oid::from_hex(text).ok()) + .ok_or_else(|| PackError::Protocol("malformed want line".to_string())) + }) + .collect::, _>>() + .map(WantOids::new) +} + +fn parse_oids(lines: &[&[u8]], prefix: &[u8]) -> Vec { + lines + .iter() + .filter_map(|line| line.strip_prefix(prefix)) + .filter_map(|rest| { + let hex = rest.split(|byte| *byte == b' ' || *byte == b'\n').next()?; + let hex = std::str::from_utf8(hex).ok()?; + Oid::from_hex(hex).ok() + }) + .collect() +} diff --git a/knot2/crates/knot-pack/tests/chaos.rs b/knot2/crates/knot-pack/tests/chaos.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/tests/chaos.rs @@ -0,0 +1,175 @@ +use std::path::Path; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +use knot_git::Layout; +use knot_types::{Oid, RefName, RepoDid}; + +mod common; +use common::{must, pack_objects, receive_request}; + +const DID: &str = "did:plc:squid"; +const BLOB_BYTES: usize = 32 * 1024 * 1024; + +fn incompressible(len: usize) -> Vec { + let mut state = 0x2545_f491_4f6c_dd1du64; + (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + (state & 0xff) as u8 + }) + .collect() +} + +struct Seed { + c1: String, + c2: String, + request: std::path::PathBuf, +} + +fn build_seed(scratch: &Path) -> Seed { + let work = scratch.join("work"); + std::fs::create_dir_all(&work).unwrap(); + must(&work, &["init", "-q", "-b", "main"]); + std::fs::write(work.join("base.txt"), "baseline\n").unwrap(); + must(&work, &["add", "-A"]); + must(&work, &["commit", "-q", "-m", "c1"]); + let c1 = must(&work, &["rev-parse", "HEAD"]); + std::fs::write(work.join("big.bin"), incompressible(BLOB_BYTES)).unwrap(); + must(&work, &["add", "-A"]); + must(&work, &["commit", "-q", "-m", "c2"]); + let c2 = must(&work, &["rev-parse", "HEAD"]); + + let oids: Vec = must(&work, &["rev-list", "--objects", &c2, "--not", &c1]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + let pack = pack_objects(&work, &oids); + let request = scratch.join("c2.request"); + std::fs::write( + &request, + receive_request("refs/heads/main", &c1, &c2, &pack), + ) + .unwrap(); + Seed { c1, c2, request } +} + +fn fresh_repo(scratch: &Path, trial: usize, seed: &Seed) -> std::path::PathBuf { + let scan = scratch.join(format!("scan-{trial}")); + let layout = Layout::new(&scan); + let did = RepoDid::new(DID).unwrap(); + layout.create(&did).unwrap(); + let bare = layout.repo_path(&did).unwrap(); + must( + scratch.join("work").as_path(), + &[ + "push", + "-q", + bare.to_str().unwrap(), + &format!("{}:refs/heads/main", seed.c1), + ], + ); + scan +} + +fn spawn_worker(scan: &Path, request: &Path) -> std::process::Child { + Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "chaos_receive_worker", "--nocapture"]) + .env("KNOT_CHAOS_ROLE", "worker") + .env("KNOT_CHAOS_SCAN", scan) + .env("KNOT_CHAOS_REQUEST", request) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn chaos worker") +} + +fn fsck_clean(bare: &Path) -> Result<(), String> { + knot_fixtures::fsck(bare) +} + +fn main_tip(scan: &Path) -> Option { + let layout = Layout::new(scan); + let repo = layout + .open(&RepoDid::new(DID).unwrap()) + .expect("repo must reopen cleanly after a kill"); + repo.find_ref(&RefName::new("refs/heads/main").unwrap()) + .expect("references must be readable after a kill") +} + +#[test] +fn chaos_receive_worker() { + if std::env::var("KNOT_CHAOS_ROLE").as_deref() != Ok("worker") { + return; + } + let scan = std::env::var("KNOT_CHAOS_SCAN").unwrap(); + let request = std::env::var("KNOT_CHAOS_REQUEST").unwrap(); + let layout = Layout::new(&scan); + let repo = layout.open(&RepoDid::new(DID).unwrap()).unwrap(); + let body = std::fs::read(&request).unwrap(); + let _ = knot_pack::receive_pack(&repo, &body); +} + +#[test] +fn kill9_during_receive_pack_leaves_a_consistent_repo() { + let scratch = tempfile::tempdir().unwrap(); + let seed = build_seed(scratch.path()); + let c1 = Oid::from_hex(&seed.c1).unwrap(); + let c2 = Oid::from_hex(&seed.c2).unwrap(); + let did = RepoDid::new(DID).unwrap(); + + let warm_scan = fresh_repo(scratch.path(), 9000, &seed); + let started = Instant::now(); + let mut warm = spawn_worker(&warm_scan, &seed.request); + warm.wait().unwrap(); + let full = started.elapsed(); + assert_eq!( + main_tip(&warm_scan), + Some(c2), + "uninterrupted receive must fast-forward main to new tip" + ); + + let fractions = [0.30, 0.45, 0.55, 0.62, 0.70, 0.78, 0.85, 0.92, 1.05, 1.25]; + let delays: Vec = std::iter::once(Duration::from_millis(2)) + .chain(std::iter::once(Duration::from_millis(5))) + .chain(fractions.iter().map(|fraction| full.mul_f64(*fraction))) + .chain(std::iter::once(full.mul_f64(2.0))) + .chain(std::iter::once(full.mul_f64(2.0))) + .collect(); + + let outcomes: Vec = delays + .iter() + .enumerate() + .map(|(trial, delay)| { + let scan = fresh_repo(scratch.path(), trial, &seed); + let mut child = spawn_worker(&scan, &seed.request); + std::thread::sleep(*delay); + let _ = child.kill(); + child.wait().unwrap(); + + let bare = Layout::new(&scan).repo_path(&did).unwrap(); + fsck_clean(&bare).unwrap_or_else(|errors| { + panic!("trial {trial}: killed receive left a corrupt repo:\n{errors}") + }); + let tip = main_tip(&scan).unwrap_or_else(|| { + panic!("trial {trial}: main vanished after a kill, acknowledged ref was lost") + }); + assert!( + tip == c1 || tip == c2, + "trial {trial}: main must hold either acknowledged baseline or completed tip, never a torn value, got {tip}" + ); + tip + }) + .collect(); + + assert!( + outcomes.contains(&c1), + "no trial was interrupted before ref update; chaos window never opened" + ); + assert!( + outcomes.contains(&c2), + "no trial ran to completion; receive never finished under chosen delays" + ); +} diff --git a/knot2/crates/knot-pack/tests/differential.rs b/knot2/crates/knot-pack/tests/differential.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/tests/differential.rs @@ -0,0 +1,697 @@ +use std::collections::{BTreeSet, HashMap}; +use std::path::Path; + +use axum::Router; +use axum::body::{Body, Bytes}; +use axum::extract::{DefaultBodyLimit, Path as UrlPath, Query, State}; +use axum::http::header; +use axum::response::Response; +use axum::routing::{get, post}; +use knot_git::Layout; +use knot_pack::PackError; +use knot_types::{ObjectFormat, RepoDid, RepoRkey}; + +mod common; +use common::{ + advance_via_receive, git, must, object_set, seed_branches_and_tag, serve_dids, spawn, +}; + +#[derive(Clone)] +struct Receive { + layout: Layout, +} + +fn open(layout: &Layout, did: &str, name: &str) -> Result { + let did = RepoDid::new(did).map_err(|e| PackError::BadPath(e.to_string()))?; + RepoRkey::new(name).map_err(|e| PackError::BadPath(e.to_string()))?; + Ok(layout.open(&did)?) +} + +fn raw_response(content_type: &'static str, body: Vec) -> Response { + Response::builder() + .header(header::CONTENT_TYPE, content_type) + .body(Body::from(body)) + .unwrap() +} + +async fn receive_info_refs( + State(state): State, + UrlPath((did, name)): UrlPath<(String, String)>, + Query(query): Query>, +) -> Result { + let repo = open(&state.layout, &did, &name)?; + match query.get("service").map(String::as_str) { + Some("git-upload-pack") => Ok(raw_response( + "application/x-git-upload-pack-advertisement", + knot_pack::advertise_upload(&repo)?, + )), + Some("git-receive-pack") => Ok(raw_response( + "application/x-git-receive-pack-advertisement", + knot_pack::advertise_receive(&repo)?, + )), + _ => Err(PackError::UnsupportedService), + } +} + +async fn receive_upload( + State(state): State, + UrlPath((did, name)): UrlPath<(String, String)>, + body: Bytes, +) -> Result { + let repo = open(&state.layout, &did, &name)?; + let result = knot_pack::upload_pack(&repo, &body)?; + Ok(raw_response("application/x-git-upload-pack-result", result)) +} + +async fn receive_receive( + State(state): State, + UrlPath((did, name)): UrlPath<(String, String)>, + body: Bytes, +) -> Result { + let repo = open(&state.layout, &did, &name)?; + let result = knot_pack::receive_pack(&repo, &body)?; + Ok(raw_response( + "application/x-git-receive-pack-result", + result, + )) +} + +fn knot_receive_router(layout: Layout) -> Router { + Router::new() + .route("/{did}/{name}/info/refs", get(receive_info_refs)) + .route("/{did}/{name}/git-upload-pack", post(receive_upload)) + .route("/{did}/{name}/git-receive-pack", post(receive_receive)) + .layer(DefaultBodyLimit::disable()) + .with_state(Receive { layout }) +} + +fn clone_to(scratch: &Path, url: &str, dest: &Path) { + must(scratch, &["clone", "-q", url, dest.to_str().unwrap()]); +} + +fn publish(work: &Path, bares: [&Path; 2], refs: &[&str], head: &str) { + bares.into_iter().for_each(|bare| { + let push = [&["push", "-q", bare.to_str().unwrap()], refs].concat(); + must(work, &push); + must(bare, &["symbolic-ref", "HEAD", head]); + }); +} + +fn seed_cross_branch_deltas(work: &Path, knot_bare: &Path, canon_bare: &Path) { + std::fs::create_dir_all(work).unwrap(); + must(work, &["init", "-q", "-b", "base"]); + std::fs::write(work.join("README.md"), "thin\n").unwrap(); + must(work, &["add", "-A"]); + must(work, &["commit", "-q", "-m", "base"]); + let base = must(work, &["rev-parse", "HEAD"]); + + let bulk = |stem: usize, extra: &str| { + let body: String = (0..200) + .map(|line| format!("file {stem} line {line} with some shared payload\n")) + .collect(); + format!("{body}{extra}") + }; + let branch = |name: &str, extra: &str| { + must(work, &["checkout", "-q", "-b", name, &base]); + (0..8).for_each(|stem| { + std::fs::write(work.join(format!("bulk-{stem}.txt")), bulk(stem, extra)).unwrap(); + }); + must(work, &["add", "-A"]); + must(work, &["commit", "-q", "-m", name]); + }; + branch("side", ""); + branch("other", "one divergent trailing line\n"); + must(work, &["checkout", "-q", "base"]); + + publish( + work, + [knot_bare, canon_bare], + &["base", "side", "other"], + "refs/heads/base", + ); + [knot_bare, canon_bare].into_iter().for_each(|bare| { + must(bare, &["repack", "-adf", "--window=50", "--depth=50"]); + }); +} + +fn seed_nested_bares(work: &Path, knot_bare: &Path, canon_bare: &Path, format: ObjectFormat) { + let fmt = format!("--object-format={}", format.capability()); + std::fs::create_dir_all(work).unwrap(); + must(work, &["init", &fmt, "-q", "-b", "main"]); + std::fs::create_dir_all(work.join("a/c")).unwrap(); + std::fs::create_dir_all(work.join("b")).unwrap(); + std::fs::create_dir_all(work.join("deep/mid/bottom")).unwrap(); + std::fs::write(work.join("root.txt"), b"01234567").unwrap(); + std::fs::write(work.join("a/c/leaf"), b"hi\n").unwrap(); + std::fs::write(work.join("b/leaf"), b"hi\n").unwrap(); + std::fs::write(work.join("deep/mid/bottom/far.txt"), "x".repeat(20)).unwrap(); + must(work, &["add", "-A"]); + must(work, &["commit", "-q", "-m", "c1"]); + std::fs::write(work.join("deep/mid/bottom/far.txt"), "y".repeat(40)).unwrap(); + must(work, &["add", "-A"]); + must(work, &["commit", "-q", "-m", "c2"]); + publish(work, [knot_bare, canon_bare], &["main"], "refs/heads/main"); +} + +#[derive(Clone, Copy, PartialEq)] +enum Mode { + Serve, + Receive, +} + +struct Servers { + canon_bare: std::path::PathBuf, + knot_bare: std::path::PathBuf, + canon_url: String, + knot_url: String, +} + +async fn stand_up( + scan: &Path, + canon_root: &Path, + did: &RepoDid, + name: &RepoRkey, + format: ObjectFormat, + mode: Mode, +) -> Servers { + let layout = Layout::new(scan).with_object_format(format); + layout.create(did).unwrap(); + let knot_bare = layout.repo_path(did).unwrap(); + + let canon_bare = canon_root.join(format!("{}.git", name.as_str())); + let fmt = format!("--object-format={}", format.capability()); + must( + Path::new("/tmp"), + &["init", "--bare", &fmt, "-q", canon_bare.to_str().unwrap()], + ); + + let knot = match mode { + Mode::Serve => { + spawn( + knot_pack::router( + layout, + serve_dids(), + std::sync::Arc::new(knot_runtime::SystemClock), + ), + "[::1]:0", + ) + .await + } + Mode::Receive => spawn(knot_receive_router(layout), "[::1]:0").await, + }; + let knot_url = match mode { + Mode::Serve => format!("http://{knot}/{}", did.as_str()), + Mode::Receive => format!("http://{knot}/{}/{}", did.as_str(), name.as_str()), + }; + Servers { + canon_url: format!("file://{}", canon_bare.to_str().unwrap()), + canon_bare, + knot_bare, + knot_url, + } +} + +fn single_pack_idx(bare: &Path) -> std::path::PathBuf { + std::fs::read_dir(bare.join("objects/pack")) + .unwrap() + .filter_map(Result::ok) + .map(|entry| entry.path()) + .find(|path| path.extension().is_some_and(|ext| ext == "idx")) + .expect("a single pack index after repack") +} + +async fn clone_lifecycle(format: ObjectFormat, did: &str, name: &str) { + let scan = tempfile::tempdir().unwrap(); + let canon_root = tempfile::tempdir().unwrap(); + let did = RepoDid::new(did).unwrap(); + let name = RepoRkey::new(name).unwrap(); + let s = stand_up( + scan.path(), + canon_root.path(), + &did, + &name, + format, + Mode::Serve, + ) + .await; + + let scratch = tempfile::tempdir().unwrap(); + let work = scratch.path().join("work"); + seed_branches_and_tag(&work, [&s.knot_bare, &s.canon_bare], format); + + let canon = scratch.path().join("clone-canon"); + let knot = scratch.path().join("clone-knot"); + clone_to(scratch.path(), &s.canon_url, &canon); + clone_to(scratch.path(), &s.knot_url, &knot); + let parse = |dir: &Path, rev: &str| must(dir, &["rev-parse", rev]); + assert_eq!( + parse(&knot, "--show-object-format"), + format.capability(), + "{format:?} clone inherits object format" + ); + assert_eq!( + parse(&canon, "HEAD^{tree}"), + parse(&knot, "HEAD^{tree}"), + "{format:?} same checked-out tree" + ); + assert_eq!( + must(&canon, &["ls-files", "-s"]), + must(&knot, &["ls-files", "-s"]), + "{format:?} identical working tree" + ); + assert_eq!( + object_set(&canon), + object_set(&knot), + "{format:?} clone transfers canonical object set" + ); + assert_eq!( + must(&canon, &["branch", "-r"]), + must(&knot, &["branch", "-r"]), + "{format:?} same remote branches" + ); + + must( + &s.knot_bare, + &["-c", "repack.writeBitmaps=false", "repack", "-adq"], + ); + let repo = knot_git::Repo::open(&s.knot_bare).unwrap(); + assert!( + knot_git::write_bitmap(&repo, &single_pack_idx(&s.knot_bare)).unwrap(), + "{format:?} single-pack repo gets a bitmap" + ); + must(&s.knot_bare, &["rev-list", "--test-bitmap", "HEAD"]); + + let bm_canon = scratch.path().join("reclone-canon"); + let bm_knot = scratch.path().join("reclone-knot"); + clone_to(scratch.path(), &s.canon_url, &bm_canon); + clone_to(scratch.path(), &s.knot_url, &bm_knot); + assert_eq!( + object_set(&bm_canon), + object_set(&bm_knot), + "{format:?} bitmap fast path serves canonical object set" + ); + must(&bm_knot, &["fsck", "--strict"]); + + std::fs::write(work.join("incremental.txt"), "fetch me\n").unwrap(); + must(&work, &["add", "-A"]); + must(&work, &["commit", "-q", "-m", "c4"]); + let tip = must(&work, &["rev-parse", "HEAD"]); + let old = must(&work, &["rev-parse", "HEAD~1"]); + advance_via_receive(&s.knot_bare, &work, &old, &tip); + must( + &work, + &["push", "-q", s.canon_bare.to_str().unwrap(), "main"], + ); + must(&canon, &["fetch", "-q", "origin"]); + must(&knot, &["fetch", "-q", "origin"]); + assert_eq!( + must(&knot, &["rev-parse", "origin/main"]), + tip, + "{format:?} fetch advances origin/main" + ); + assert_eq!( + object_set(&canon), + object_set(&knot), + "{format:?} incremental fetch transfers canonical object set" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn differential_clone_fetch_and_bitmap_serving_match_canonical() { + clone_lifecycle(ObjectFormat::SHA1, "did:plc:squid", "scallop").await; + clone_lifecycle(ObjectFormat::SHA256, "did:plc:nautilus", "whelk").await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_shallow_server_repo_is_served_and_the_clone_becomes_shallow() { + let scan = tempfile::tempdir().unwrap(); + let canon_root = tempfile::tempdir().unwrap(); + let did = RepoDid::new("did:plc:periwinkle").unwrap(); + let name = RepoRkey::new("conch").unwrap(); + let s = stand_up( + scan.path(), + canon_root.path(), + &did, + &name, + ObjectFormat::SHA1, + Mode::Serve, + ) + .await; + + let scratch = tempfile::tempdir().unwrap(); + let work = scratch.path().join("work"); + std::fs::create_dir_all(&work).unwrap(); + must(&work, &["init", "-q", "-b", "main"]); + (1..=5).for_each(|n| { + std::fs::write(work.join("history.txt"), format!("revision {n}\n")).unwrap(); + must(&work, &["add", "-A"]); + must(&work, &["commit", "-q", "-m", &format!("c{n}")]); + }); + let work_url = format!("file://{}", work.to_str().unwrap()); + [&s.knot_bare, &s.canon_bare].into_iter().for_each(|bare| { + must( + bare, + &[ + "fetch", + "-q", + "--depth=2", + &work_url, + "main:refs/heads/main", + ], + ); + must(bare, &["symbolic-ref", "HEAD", "refs/heads/main"]); + assert!(bare.join("shallow").exists(), "server repo is shallow"); + }); + + let canon = scratch.path().join("clone-canon"); + let knot = scratch.path().join("clone-knot"); + clone_to(scratch.path(), &s.canon_url, &canon); + clone_to(scratch.path(), &s.knot_url, &knot); + assert!( + knot.join(".git/shallow").exists(), + "clone of a shallow server is itself shallow" + ); + must(&knot, &["fsck"]); + assert_eq!( + must(&canon, &["rev-list", "--count", "HEAD"]), + must(&knot, &["rev-list", "--count", "HEAD"]), + "both clones see the same clamped depth" + ); + assert_eq!( + object_set(&canon), + object_set(&knot), + "shallow clone transfers canonical object set" + ); +} + +fn fetch_branch(scratch: &Path, url: &str, label: &str, branch: &str) -> BTreeSet { + let clone = scratch.join(format!("clone-{label}-{branch}")); + must( + scratch, + &[ + "clone", + "-q", + "--single-branch", + "--branch", + "base", + url, + clone.to_str().unwrap(), + ], + ); + must(&clone, &["fetch", "-q", "origin", branch]); + must(&clone, &["fsck", "--strict"]); + object_set(&clone) +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_thin_fetch_never_deltas_against_objects_the_client_lacks() { + let scan = tempfile::tempdir().unwrap(); + let canon_root = tempfile::tempdir().unwrap(); + let did = RepoDid::new("did:plc:limpet").unwrap(); + let name = RepoRkey::new("whelk").unwrap(); + let s = stand_up( + scan.path(), + canon_root.path(), + &did, + &name, + ObjectFormat::SHA1, + Mode::Serve, + ) + .await; + + let scratch = tempfile::tempdir().unwrap(); + seed_cross_branch_deltas(&scratch.path().join("work"), &s.knot_bare, &s.canon_bare); + + ["side", "other"].into_iter().for_each(|branch| { + assert_eq!( + fetch_branch(scratch.path(), &s.canon_url, "canon", branch), + fetch_branch(scratch.path(), &s.knot_url, "knot", branch), + "thin fetch of {branch} onto a base-only clone transfers canonical object set" + ); + }); +} + +fn push_sequence(remote: &str, scratch: &Path, label: &str) -> Vec<(String, bool)> { + let clone = scratch.join(format!("push-{label}")); + must(scratch, &["clone", "-q", remote, clone.to_str().unwrap()]); + let commit = |file: &str, body: &str, msg: &str| { + std::fs::write(clone.join(file), body).unwrap(); + must(&clone, &["add", "-A"]); + must(&clone, &["commit", "-q", "-m", msg]); + }; + let push = |args: &[&str]| git(&clone, &[&["push", "-q", "origin"], args].concat()).0; + + commit("ff.txt", "fast forward\n", "fast forward"); + let ff = push(&["main"]); + must(&clone, &["checkout", "-q", "-b", "feature"]); + commit("feature.txt", "new branch\n", "feature"); + let new_branch = push(&["feature"]); + let delete = push(&["--delete", "feature"]); + must(&clone, &["checkout", "-q", "main"]); + must(&clone, &["reset", "-q", "--hard", "HEAD~1"]); + commit("diverge.txt", "non fast forward\n", "diverge"); + let non_ff = push(&["main"]); + + vec![ + ("fast-forward".to_string(), ff), + ("new-branch".to_string(), new_branch), + ("delete-branch".to_string(), delete), + ("non-fast-forward".to_string(), non_ff), + ] +} + +async fn push_diff(format: ObjectFormat, did: &str, name: &str) { + let scan = tempfile::tempdir().unwrap(); + let canon_root = tempfile::tempdir().unwrap(); + let did = RepoDid::new(did).unwrap(); + let name = RepoRkey::new(name).unwrap(); + let s = stand_up( + scan.path(), + canon_root.path(), + &did, + &name, + format, + Mode::Receive, + ) + .await; + + let scratch = tempfile::tempdir().unwrap(); + seed_branches_and_tag( + &scratch.path().join("work"), + [&s.knot_bare, &s.canon_bare], + format, + ); + + let advertised = |bare: &Path| { + must( + bare, + &[ + "for-each-ref", + "--format=%(refname) %(objectname)", + "refs/heads/", + "refs/tags/", + ], + ) + }; + assert_eq!( + push_sequence(&s.canon_url, scratch.path(), "canon"), + push_sequence(&s.knot_url, scratch.path(), "knot"), + "{format:?} knot accepts and rejects the same pushes as canonical git" + ); + assert_eq!( + advertised(&s.canon_bare), + advertised(&s.knot_bare), + "{format:?} both servers hold the same refs after an identical push sequence" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn differential_push_verdicts_match_canonical() { + push_diff(ObjectFormat::SHA1, "did:plc:squid", "barnacle").await; + push_diff(ObjectFormat::SHA256, "did:plc:cuttle", "scallop").await; +} + +fn enable_filter(bare: &Path) { + must(bare, &["config", "uploadpack.allowFilter", "true"]); + must(bare, &["config", "uploadpack.allowAnySHA1InWant", "true"]); +} + +fn filtered_clone(scratch: &Path, url: &str, label: &str, filter: &str) -> BTreeSet { + let clone = scratch.join(format!("clone-{label}")); + must( + scratch, + &[ + "clone", + "-q", + "--no-checkout", + &format!("--filter={filter}"), + url, + clone.to_str().unwrap(), + ], + ); + object_set(&clone) +} + +fn fetched_set( + scratch: &Path, + url: &str, + label: &str, + oid: &str, + filter: &str, + format: ObjectFormat, +) -> BTreeSet { + let dest = scratch.join(format!("fetch-{label}")); + let fmt = format!("--object-format={}", format.capability()); + must(scratch, &["init", &fmt, "-q", dest.to_str().unwrap()]); + must( + &dest, + &["fetch", "-q", &format!("--filter={filter}"), url, oid], + ); + object_set(&dest) +} + +fn blobless_checkout(scratch: &Path, url: &str, label: &str) -> BTreeSet { + let clone = scratch.join(format!("clone-{label}")); + must( + scratch, + &[ + "clone", + "-q", + "--filter=blob:none", + url, + clone.to_str().unwrap(), + ], + ); + must(&clone, &["fsck", "--strict"]); + object_set(&clone) +} + +async fn partial_clone(format: ObjectFormat, did: &str, name: &str) { + let scan = tempfile::tempdir().unwrap(); + let canon_root = tempfile::tempdir().unwrap(); + let did = RepoDid::new(did).unwrap(); + let name = RepoRkey::new(name).unwrap(); + let s = stand_up( + scan.path(), + canon_root.path(), + &did, + &name, + format, + Mode::Serve, + ) + .await; + enable_filter(&s.canon_bare); + + let scratch = tempfile::tempdir().unwrap(); + let work = scratch.path().join("work"); + seed_nested_bares(&work, &s.knot_bare, &s.canon_bare, format); + + [ + "tree:0", + "tree:1", + "tree:2", + "tree:3", + "tree:4", + "tree:5", + "blob:none", + "blob:limit=3", + "blob:limit=4", + "blob:limit=8", + "blob:limit=20", + ] + .into_iter() + .for_each(|filter| { + assert_eq!( + filtered_clone( + scratch.path(), + &s.canon_url, + &format!("canon-{filter}"), + filter + ), + filtered_clone( + scratch.path(), + &s.knot_url, + &format!("knot-{filter}"), + filter + ), + "{format:?} a --filter={filter} clone transfers canonical object set" + ); + }); + + let deep = must(&work, &["rev-parse", "HEAD:deep"]); + let far = must(&work, &["rev-parse", "HEAD:deep/mid/bottom/far.txt"]); + [ + (&deep, "tree:0"), + (&deep, "tree:1"), + (&deep, "tree:2"), + (&deep, "tree:3"), + (&deep, "blob:none"), + (&deep, "blob:limit=10"), + (&far, "blob:none"), + (&far, "blob:limit=10"), + ] + .into_iter() + .enumerate() + .for_each(|(case, (oid, filter))| { + assert_eq!( + fetched_set( + scratch.path(), + &s.canon_url, + &format!("canon-{case}"), + oid, + filter, + format + ), + fetched_set( + scratch.path(), + &s.knot_url, + &format!("knot-{case}"), + oid, + filter, + format + ), + "{format:?} explicit want {oid} --filter={filter} transfers canonical object set" + ); + }); + + assert_eq!( + blobless_checkout(scratch.path(), &s.canon_url, "canon"), + blobless_checkout(scratch.path(), &s.knot_url, "knot"), + "{format:?} a blobless clone faults its checkout blobs in like canonical git" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn differential_partial_clone_matches_canonical() { + partial_clone(ObjectFormat::SHA1, "did:plc:anemone", "barnacle").await; + partial_clone(ObjectFormat::SHA256, "did:plc:cuttle", "uni").await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn an_empty_sha256_repo_clones_to_sha256_over_v2_and_v0() { + let scan = tempfile::tempdir().unwrap(); + let did = RepoDid::new("did:plc:limpet").unwrap(); + let layout = Layout::new(scan.path()).with_object_format(ObjectFormat::SHA256); + layout.create(&did).unwrap(); + let knot = spawn( + knot_pack::router( + layout, + serve_dids(), + std::sync::Arc::new(knot_runtime::SystemClock), + ), + "[::1]:0", + ) + .await; + let url = format!("http://{knot}/{}", did.as_str()); + + let scratch = tempfile::tempdir().unwrap(); + [("v2", "2"), ("v0", "0")].into_iter().for_each(|(label, version)| { + let dst = scratch.path().join(format!("clone-{label}")); + must(scratch.path(), &["-c", &format!("protocol.version={version}"), "clone", "-q", &url, dst.to_str().unwrap()]); + assert_eq!( + must(&dst, &["rev-parse", "--show-object-format"]), + "sha256", + "{label}: an empty knot sha256 repo clones to sha256 from its advertised capabilities" + ); + }); +} diff --git a/knot2/crates/knot-pack/tests/fetch.rs b/knot2/crates/knot-pack/tests/fetch.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/tests/fetch.rs @@ -0,0 +1,334 @@ +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::sync::Arc; + +use axum::http; +use knot_git::{Layout, Repo, Staging}; +use knot_pack::{FetchError, HaveOids, PackLimits, WantOids, ingest_pack, local_pack, local_refs}; +use knot_runtime::{FakeHttp, HttpRequest, HttpResponse, HttpTransport}; +use knot_types::{Oid, RefName, RepoDid}; +use url::Url; + +mod common; +use common::{commit, must, pack_objects}; + +fn seed_source(dir: &Path) -> PathBuf { + let work = dir.join("work"); + std::fs::create_dir_all(&work).unwrap(); + must(&work, &["init", "-q", "-b", "main"]); + commit(&work, "reef.txt", "kelp forest\n", "first"); + commit(&work, "tide.txt", "rock pool\n", "second"); + must(&work, &["tag", "v1"]); + must(&work, &["branch", "anemone"]); + let bare = dir.join("source.git"); + must( + dir, + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + ); + bare +} + +fn stock_git_server(repo: PathBuf) -> Arc { + Arc::new(FakeHttp::new(move |request: &HttpRequest| { + let response = |body: Vec| { + Ok(HttpResponse { + status: http::StatusCode::OK, + headers: http::HeaderMap::new(), + body: body.into(), + }) + }; + if request.url.path().ends_with("/info/refs") { + let out = knot_fixtures::command(&repo) + .args([ + "upload-pack", + "--stateless-rpc", + "--http-backend-info-refs", + ".", + ]) + .env("GIT_PROTOCOL", "version=2") + .output() + .expect("git upload-pack advertises"); + assert!(out.status.success()); + let mut body = b"001e# service=git-upload-pack\n0000".to_vec(); + body.extend_from_slice(&out.stdout); + return response(body); + } + let mut child = knot_fixtures::command(&repo) + .args(["upload-pack", "--stateless-rpc", "."]) + .env("GIT_PROTOCOL", "version=2") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("git upload-pack serves"); + child + .stdin + .take() + .unwrap() + .write_all(request.body.as_deref().unwrap_or_default()) + .unwrap(); + let out = child.wait_with_output().expect("git upload-pack finishes"); + assert!( + out.status.success(), + "upload-pack: {}", + String::from_utf8_lossy(&out.stderr) + ); + response(out.stdout) + })) +} + +fn knot_server(repo_path: PathBuf) -> Arc { + Arc::new(FakeHttp::new(move |request: &HttpRequest| { + let repo = Repo::open(&repo_path).unwrap(); + let body = if request.url.path().ends_with("/info/refs") { + knot_pack::advertise_upload(&repo).unwrap() + } else { + knot_pack::upload_pack(&repo, request.body.as_deref().unwrap_or_default()).unwrap() + }; + Ok(HttpResponse { + status: http::StatusCode::OK, + headers: http::HeaderMap::new(), + body: body.into(), + }) + })) +} + +fn base_url() -> Url { + Url::parse("https://kelp.oyster.cafe/did:plc:squid/uni").unwrap() +} + +const CAP: u64 = 64 * 1024 * 1024; + +fn refnames(records: &[knot_git::RefRecord]) -> Vec<&str> { + records.iter().map(|record| record.name.as_str()).collect() +} + +async fn clone_through(http: &dyn HttpTransport, source: &Repo, target: &Repo) { + let refs = knot_pack::remote_refs(http, &base_url(), &["HEAD", "refs/heads/", "refs/tags/"]) + .await + .unwrap(); + assert_eq!( + refs.head_symref.as_ref().map(RefName::as_str), + Some("refs/heads/main") + ); + let mut expected = source.advertised_refs().unwrap().to_vec(); + expected.sort_by(|a, b| a.name.as_str().cmp(b.name.as_str())); + let mut got = refs.refs.clone(); + got.sort_by(|a, b| a.name.as_str().cmp(b.name.as_str())); + assert_eq!(refnames(&got), refnames(&expected)); + assert_eq!(got, expected); + + let pack = knot_pack::remote_pack( + http, + &base_url(), + &WantOids::new(refs.tips()), + &HaveOids::default(), + CAP, + ) + .await + .unwrap(); + ingest_pack( + &target.objects_dir(), + &pack, + &PackLimits::default(), + target.object_format().kind(), + ) + .unwrap(); + let closure = target + .select_pack_objects( + knot_git::Wants::new(&refs.tips()), + knot_git::Haves::new(&[]), + ) + .unwrap(); + assert!(closure.iter().all(|oid| target.contains(*oid))); +} + +#[tokio::test] +async fn the_client_clones_from_both_stock_git_and_native_knot_servers() { + let dir = tempfile::tempdir().unwrap(); + let source_path = seed_source(dir.path()); + let source = Repo::open(&source_path).unwrap(); + + let stock = Repo::create(dir.path().join("fork-stock.git")).unwrap(); + clone_through( + stock_git_server(source_path.clone()).as_ref(), + &source, + &stock, + ) + .await; + + let knot = Repo::create(dir.path().join("fork-knot.git")).unwrap(); + clone_through(knot_server(source_path.clone()).as_ref(), &source, &knot).await; +} + +#[tokio::test] +async fn an_incremental_pull_completes_through_staging() { + let dir = tempfile::tempdir().unwrap(); + let source_path = seed_source(dir.path()); + let source = Repo::open(&source_path).unwrap(); + let clone = Repo::create(dir.path().join("fork.git")).unwrap(); + let http = knot_server(source_path.clone()); + clone_through(http.as_ref(), &source, &clone).await; + let main = RefName::new("refs/heads/main").unwrap(); + let old_tip = source.find_ref(&main).unwrap().unwrap(); + clone + .update_ref(&knot_git::RefUpdate::Create { + name: main.clone(), + new: old_tip, + }) + .unwrap(); + + let work = dir.path().join("work"); + commit(&work, "spray.txt", "salt\n", "third"); + let new_hex = must(&work, &["rev-parse", "HEAD"]); + let new_tip = Oid::from_hex(&new_hex).unwrap(); + let oids: Vec = must( + &work, + &[ + "rev-list", + "--objects", + &new_hex, + "--not", + &old_tip.to_hex(), + ], + ) + .lines() + .filter_map(|line| line.split_whitespace().next()) + .map(str::to_string) + .collect(); + let pack = pack_objects(&work, &oids); + ingest_pack( + &source.objects_dir(), + &pack, + &PackLimits::default(), + source.object_format().kind(), + ) + .unwrap(); + source + .update_ref(&knot_git::RefUpdate::Update { + name: main.clone(), + old: old_tip, + new: new_tip, + }) + .unwrap(); + assert_ne!(old_tip, new_tip); + + let pack = knot_pack::remote_pack( + http.as_ref(), + &base_url(), + &WantOids::new(vec![new_tip]), + &HaveOids::new(vec![old_tip]), + CAP, + ) + .await + .unwrap(); + let staging = Staging::new(&clone).unwrap(); + ingest_pack( + &staging.repo().objects_dir(), + &pack, + &PackLimits::default(), + clone.object_format().kind(), + ) + .unwrap(); + let closure = staging + .repo() + .select_pack_objects( + knot_git::Wants::new(&[new_tip]), + knot_git::Haves::new(&[old_tip]), + ) + .unwrap(); + assert!(closure.iter().all(|oid| staging.repo().contains(*oid))); + staging.migrate_into(&clone).unwrap(); + assert!(clone.contains(new_tip)); +} + +#[tokio::test] +async fn a_tiny_pack_limit_refuses_both_remote_and_local_transfers() { + let dir = tempfile::tempdir().unwrap(); + let source_path = seed_source(dir.path()); + let source = Repo::open(&source_path).unwrap(); + let tips: Vec = source + .advertised_refs() + .unwrap() + .iter() + .map(|record| record.target) + .collect(); + + let http = knot_server(source_path); + let remote = knot_pack::remote_pack( + http.as_ref(), + &base_url(), + &WantOids::new(tips.clone()), + &HaveOids::default(), + 16, + ) + .await; + assert!(matches!( + remote, + Err(FetchError::PackTooLarge { limit: 16 }) + )); + + let local = local_pack(&source, &WantOids::new(tips), &HaveOids::default(), 16); + assert!(matches!(local, Err(FetchError::PackTooLarge { limit: 16 }))); +} + +#[test] +fn local_refs_and_pack_mirror_a_same_knot_source() { + let dir = tempfile::tempdir().unwrap(); + let source_path = seed_source(dir.path()); + let source = Repo::open(&source_path).unwrap(); + let hidden = RefName::new("refs/hidden/feature/main").unwrap(); + let main = RefName::new("refs/heads/main").unwrap(); + let tip = source.find_ref(&main).unwrap().unwrap(); + source + .update_ref(&knot_git::RefUpdate::Create { + name: hidden, + new: tip, + }) + .unwrap(); + + let refs = local_refs(&source, &["HEAD", "refs/heads/", "refs/tags/"]).unwrap(); + assert_eq!( + refs.head_symref.as_ref().map(RefName::as_str), + Some("refs/heads/main") + ); + assert!( + refs.refs + .iter() + .all(|record| !record.name.as_str().starts_with("refs/hidden/")), + "hidden ref must never leave source repo through a fork" + ); + + let layout = Layout::new(dir.path().join("scan")); + let fork = layout + .create(&RepoDid::new("did:plc:limpet").unwrap()) + .unwrap(); + let pack = local_pack( + &source, + &WantOids::new(refs.tips()), + &HaveOids::default(), + CAP, + ) + .unwrap(); + ingest_pack( + &fork.objects_dir(), + &pack, + &PackLimits::default(), + fork.object_format().kind(), + ) + .unwrap(); + let closure = fork + .select_pack_objects( + knot_git::Wants::new(&refs.tips()), + knot_git::Haves::new(&[]), + ) + .unwrap(); + assert!(closure.iter().all(|oid| fork.contains(*oid))); +} diff --git a/knot2/crates/knot-pack/tests/fuzz_smoke.rs b/knot2/crates/knot-pack/tests/fuzz_smoke.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/tests/fuzz_smoke.rs @@ -0,0 +1,76 @@ +use knot_git::Layout; +use knot_types::RepoDid; +use proptest::prelude::*; +use proptest::test_runner::TestRunner; + +fn empty_repo() -> (knot_git::Repo, tempfile::TempDir) { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let repo = layout.create(&did).unwrap(); + (repo, scan) +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(1024))] + + #[test] + fn parsers_never_panic(data in proptest::collection::vec(any::(), 0..4096)) { + knot_pack::fuzz::pkt(&data); + knot_pack::fuzz::pack(&data); + knot_pack::fuzz::receive_commands(&data); + knot_pack::fuzz::upload_args(&data); + } +} + +#[test] +fn a_version3_pack_header_is_a_typed_rejection_not_a_panic() { + let header = b"PACK\x00\x00\x00\x03\x00\x00\x00\x00"; + assert!( + knot_pack::meter_pack( + header, + &knot_pack::PackLimits::default(), + gix_hash::Kind::Sha1 + ) + .is_err(), + "a v3 pack header must be declined with a typed error" + ); + knot_pack::fuzz::pack(header); +} + +#[test] +fn a_lying_decompressed_size_never_preallocates_the_declared_amount() { + let bomb = [ + 0x50, 0x41, 0x43, 0x4b, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x20, 0xff, 0xa0, 0xa8, + 0xa8, 0xed, 0xff, 0xff, 0x54, 0x41, 0x43, 0xff, 0xf4, 0x38, 0x06, 0x3e, 0xff, 0xff, 0xff, + 0xc7, 0x00, 0xc7, 0xff, 0xff, 0xff, 0xff, 0x3e, 0xff, 0x3e, + ]; + assert!( + knot_pack::meter_pack( + &bomb, + &knot_pack::PackLimits::default(), + gix_hash::Kind::Sha1 + ) + .is_err(), + "an entry declaring a petabyte object must be a typed error, never an allocation" + ); + knot_pack::fuzz::pack(&bomb); +} + +#[test] +fn repo_entry_points_never_panic() { + let (repo, _scan) = empty_repo(); + let mut runner = TestRunner::default(); + runner + .run(&proptest::collection::vec(any::(), 0..8192), |data| { + let _ = knot_pack::upload_pack(&repo, &data); + let _ = knot_pack::receive_pack(&repo, &data); + let _ = knot_pack::meter_pack( + &data, + &knot_pack::PackLimits::default(), + repo.object_format().kind(), + ); + Ok(()) + }) + .unwrap(); +} diff --git a/knot2/crates/knot-pack/tests/git_client.rs b/knot2/crates/knot-pack/tests/git_client.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/tests/git_client.rs @@ -0,0 +1,931 @@ +use std::net::SocketAddr; +use std::path::Path; +use std::sync::Arc; + +use axum::body::Body; +use axum::http::header; +use knot_git::{Layout, RefUpdate}; +use knot_pack::{RepoLookup, RepoResolver, RepoTarget}; +use knot_types::{OwnerDid, RefName, RepoDid, RepoRkey}; + +mod common; +use common::{commit, git, must, pkt, serve_dids, spawn, unsideband}; + +fn seed_repo(work: &Path, bare: &str, file: &str, contents: &str) { + std::fs::create_dir_all(work).unwrap(); + must(work, &["init", "-q", "-b", "main"]); + commit(work, file, contents, "initial"); + must(work, &["push", "-q", bare, "main"]); + must( + Path::new(bare), + &["symbolic-ref", "HEAD", "refs/heads/main"], + ); +} + +fn url(addr: SocketAddr, did: &RepoDid, _name: &RepoRkey) -> String { + format!("http://{addr}/{}", did.as_str()) +} + +fn pack_object_oids(pack: &[u8]) -> std::collections::BTreeSet { + use std::io::Write; + use std::process::Stdio; + + let bare = tempfile::tempdir().unwrap(); + let path = bare.path().to_str().unwrap(); + assert!( + knot_fixtures::command(bare.path()) + .args(["init", "--bare", "-q", path]) + .output() + .unwrap() + .status + .success() + ); + let mut child = knot_fixtures::command(bare.path()) + .args(["index-pack", "--stdin"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + child.stdin.take().unwrap().write_all(pack).unwrap(); + let indexed = child.wait_with_output().unwrap(); + assert!( + indexed.status.success(), + "index-pack of our pack failed:\n{}", + String::from_utf8_lossy(&indexed.stderr) + ); + let listed = knot_fixtures::command(bare.path()) + .args([ + "cat-file", + "--batch-all-objects", + "--batch-check=%(objectname)", + ]) + .output() + .unwrap(); + String::from_utf8_lossy(&listed.stdout) + .lines() + .map(|line| line.trim().to_string()) + .filter(|line| !line.is_empty()) + .collect() +} + +fn canonical_pack(work: &Path, revs: &[String]) -> Vec { + use std::io::Write; + use std::process::Stdio; + + let mut child = knot_fixtures::command(work) + .args(["pack-objects", "--revs", "--stdout", "-q"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + child + .stdin + .take() + .unwrap() + .write_all(revs.join("\n").as_bytes()) + .unwrap(); + let out = child.wait_with_output().unwrap(); + assert!(out.status.success(), "canonical pack-objects failed"); + out.stdout +} + +fn v2_fetch_body(wants: &[String], haves: &[String]) -> Vec { + let mut body = pkt(b"command=fetch\n"); + body.extend_from_slice(b"0001"); + wants + .iter() + .for_each(|want| body.extend(pkt(format!("want {want}\n").as_bytes()))); + haves + .iter() + .for_each(|have| body.extend(pkt(format!("have {have}\n").as_bytes()))); + body.extend(pkt(b"done\n")); + body.extend_from_slice(b"0000"); + body +} + +#[tokio::test(flavor = "multi_thread")] +async fn http_routing_resolves_owner_rkey_and_dot_git_and_404s_the_unhosted() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let owner = OwnerDid::new("did:plc:nel").unwrap(); + let plain_did = RepoDid::new("did:plc:squid").unwrap(); + let literal_did = RepoDid::new("did:plc:whelk").unwrap(); + layout.create(&plain_did).unwrap(); + layout.create(&literal_did).unwrap(); + + let resolver: Arc = { + let owner = owner.clone(); + let plain_did = plain_did.clone(); + let literal_did = literal_did.clone(); + Arc::new(move |target: &RepoTarget| match target { + RepoTarget::OwnerRkey(o, n) if *o == owner && n.as_str() == "anemone" => { + RepoLookup::Hosted(plain_did.clone()) + } + RepoTarget::OwnerRkey(o, n) if *o == owner && n.as_str() == "barnacle.git" => { + RepoLookup::Hosted(literal_did.clone()) + } + _ => RepoLookup::Unhosted, + }) + }; + let addr = spawn( + knot_pack::router( + layout.clone(), + resolver, + std::sync::Arc::new(knot_runtime::SystemClock), + ), + "[::1]:0", + ) + .await; + + let scratch = tempfile::tempdir().unwrap(); + seed_repo( + &scratch.path().join("work-plain"), + layout.repo_path(&plain_did).unwrap().to_str().unwrap(), + "README.md", + "plain\n", + ); + seed_repo( + &scratch.path().join("work-literal"), + layout.repo_path(&literal_did).unwrap().to_str().unwrap(), + "README.md", + "literal\n", + ); + + let clone_ok = |name: &str, label: &str, expect: &str| { + let dest = scratch.path().join(label); + let remote = format!("http://{addr}/{}/{name}", owner.as_str()); + let (ok, out) = git( + scratch.path(), + &["clone", "-q", &remote, dest.to_str().unwrap()], + ); + assert!( + ok, + "clone of {name} must resolve through the registry:\n{out}" + ); + assert_eq!( + std::fs::read_to_string(dest.join("README.md")).unwrap(), + expect + ); + }; + clone_ok("anemone", "clone-plain", "plain\n"); + clone_ok("anemone.git", "clone-suffixed", "plain\n"); + clone_ok("barnacle.git", "clone-literal", "literal\n"); + + let clone_404 = |remote: String, why: &str| { + let dest = scratch.path().join("clone-404"); + let (ok, _out) = git( + scratch.path(), + &["clone", "-q", &remote, dest.to_str().unwrap()], + ); + assert!(!ok, "{why}"); + let _ = std::fs::remove_dir_all(&dest); + }; + clone_404( + format!("http://{addr}/{}/conch", owner.as_str()), + "rkey with no registry entry must 404, not route to the wrong repo", + ); + clone_404( + format!("http://{addr}/{}", plain_did.as_str()), + "a direct-DID path the resolver doesn't host must 404, even though the repo exists on disk", + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn http_clone_while_the_index_is_warming_is_unavailable_not_404() { + use tower::ServiceExt; + + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let resolver: Arc = Arc::new(|_target: &RepoTarget| RepoLookup::Unavailable); + + let by_name = axum::http::Request::builder() + .uri("/did:plc:nel/anemone/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + let response = knot_pack::router( + layout.clone(), + Arc::clone(&resolver), + std::sync::Arc::new(knot_runtime::SystemClock), + ) + .oneshot(by_name) + .await + .unwrap(); + assert_eq!( + response.status(), + axum::http::StatusCode::SERVICE_UNAVAILABLE, + "warming registry is a retryable 503 on owner/rkey route, never a 404" + ); + + let by_did = axum::http::Request::builder() + .uri("/did:plc:squid/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + let response = knot_pack::router( + layout, + resolver, + std::sync::Arc::new(knot_runtime::SystemClock), + ) + .oneshot(by_did) + .await + .unwrap(); + assert_eq!( + response.status(), + axum::http::StatusCode::SERVICE_UNAVAILABLE, + "warming registry is a retryable 503 on direct-DID route, never a 404" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn shallow_clone_over_protocol_v0() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let name = RepoRkey::new("scallop").unwrap(); + layout.create(&did).unwrap(); + let bare = layout.repo_path(&did).unwrap(); + let addr = spawn( + knot_pack::router( + layout.clone(), + serve_dids(), + std::sync::Arc::new(knot_runtime::SystemClock), + ), + "[::1]:0", + ) + .await; + + let scratch = tempfile::tempdir().unwrap(); + let work = scratch.path().join("work"); + seed_three_commits(&work, bare.to_str().unwrap()); + commit(&work, "d.txt", "c4\n", "c4"); + must(&work, &["push", "-q", bare.to_str().unwrap(), "main"]); + let remote = url(addr, &did, &name); + + let clone = scratch.path().join("clone"); + let (ok, out) = git( + scratch.path(), + &[ + "-c", + "protocol.version=0", + "clone", + "--depth=1", + "-q", + &remote, + clone.to_str().unwrap(), + ], + ); + assert!(ok, "v0 shallow clone failed:\n{out}"); + assert!( + clone.join(".git/shallow").exists(), + "depth-limited v0 clone must be marked shallow" + ); + assert_eq!( + must(&clone, &["rev-list", "--count", "HEAD"]).trim(), + "1", + "depth=1 over v0 must yield exactly one commit" + ); + + let (ok, out) = git( + &clone, + &[ + "-c", + "protocol.version=0", + "fetch", + "--depth=2", + "-q", + "origin", + ], + ); + assert!(ok, "v0 deepening fetch failed:\n{out}"); + assert_eq!( + must(&clone, &["rev-list", "--count", "origin/main"]).trim(), + "2", + "deepen to depth=2 over v0 must reveal second commit" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn shallow_fetch_of_an_annotated_tag() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let name = RepoRkey::new("whelk").unwrap(); + layout.create(&did).unwrap(); + let bare = layout.repo_path(&did).unwrap(); + let addr = spawn( + knot_pack::router( + layout.clone(), + serve_dids(), + std::sync::Arc::new(knot_runtime::SystemClock), + ), + "[::1]:0", + ) + .await; + + let scratch = tempfile::tempdir().unwrap(); + let work = scratch.path().join("work"); + seed_repo(&work, bare.to_str().unwrap(), "a.txt", "c1\n"); + commit(&work, "b.txt", "c2\n", "c2"); + must(&work, &["tag", "-a", "release", "-m", "release"]); + must( + &work, + &["push", "-q", bare.to_str().unwrap(), "main", "release"], + ); + let remote = url(addr, &did, &name); + + ["2", "0"].iter().enumerate().for_each(|(index, version)| { + let dest = scratch.path().join(format!("tagfetch-{index}")); + std::fs::create_dir_all(&dest).unwrap(); + must(&dest, &["init", "-q"]); + let (ok, out) = git( + &dest, + &[ + "-c", + &format!("protocol.version={version}"), + "fetch", + "--depth=1", + "-q", + &remote, + "refs/tags/release:refs/tags/release", + ], + ); + assert!(ok, "shallow tag fetch over v{version} failed:\n{out}"); + assert_eq!( + must(&dest, &["cat-file", "-t", "release"]).trim(), + "tag", + "annotated tag object itself must be transferred over v{version}" + ); + assert_eq!( + must(&dest, &["rev-list", "--count", "release^{commit}"]).trim(), + "1", + "depth-1 tag fetch over v{version} must contain exactly the tagged commit" + ); + }); +} + +#[tokio::test(flavor = "multi_thread")] +async fn pack_slot_limit_serializes_concurrent_clones_without_breaking_them() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let name = RepoRkey::new("cuttle").unwrap(); + layout.create(&did).unwrap(); + let bare = layout.repo_path(&did).unwrap(); + + let scratch = tempfile::tempdir().unwrap(); + let work = scratch.path().join("work"); + seed_repo(&work, bare.to_str().unwrap(), "README.md", "kelp\n"); + let tip = must(&work, &["rev-parse", "HEAD"]).trim().to_string(); + + let addr = spawn( + knot_pack::router_with_pack_slots( + layout, + serve_dids(), + knot_resource::PackSlots::new(1), + std::sync::Arc::new(knot_runtime::SystemClock), + ), + "[::1]:0", + ) + .await; + let remote = url(addr, &did, &name); + + let children: Vec<(usize, std::path::PathBuf, std::process::Child)> = (0..6) + .map(|index| { + let dest = scratch.path().join(format!("clone-{index}")); + let child = knot_fixtures::command(scratch.path()) + .args(["clone", "-q", &remote, dest.to_str().unwrap()]) + .spawn() + .expect("git clone spawns"); + (index, dest, child) + }) + .collect(); + + children.into_iter().for_each(|(index, dest, mut child)| { + assert!( + child.wait().unwrap().success(), + "single pack slot must still let concurrent clone {index} complete" + ); + assert_eq!( + must(&dest, &["rev-parse", "HEAD"]).trim(), + tip, + "clone served under a one-slot limit must still check out the right tip" + ); + }); +} + +#[tokio::test(flavor = "multi_thread")] +async fn http_upload_archive_serves_a_framed_tar_and_guards_refuse_cob_raw_oids_and_traversal() { + use tower::ServiceExt as _; + + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + layout.create(&did).unwrap(); + let bare = layout.repo_path(&did).unwrap(); + + let scratch = tempfile::tempdir().unwrap(); + let work = scratch.path().join("work"); + seed_repo(&work, bare.to_str().unwrap(), "README.md", "archive me\n"); + + let mut framed = Vec::new(); + framed.extend(pkt(b"argument --format=tar\n")); + framed.extend(pkt(b"argument HEAD\n")); + framed.extend_from_slice(b"0000"); + let response = knot_pack::router( + layout.clone(), + serve_dids(), + std::sync::Arc::new(knot_runtime::SystemClock), + ) + .oneshot( + axum::http::Request::builder() + .method("POST") + .uri(format!("/{}/git-upload-archive", did.as_str())) + .header( + header::CONTENT_TYPE, + "application/x-git-upload-archive-request", + ) + .body(Body::from(framed)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), axum::http::StatusCode::OK); + assert_eq!( + response + .headers() + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()), + Some("application/x-git-upload-archive-result"), + ); + let body = http_body_util::BodyExt::collect(response.into_body()) + .await + .unwrap() + .to_bytes(); + assert!( + body.starts_with(b"0008ACK\n"), + "archive response opens with the ACK pkt-line" + ); + assert!( + body.windows("README.md".len()) + .any(|window| window == b"README.md"), + "framed archive contains README.md entry" + ); + + let repo = layout.open(&did).unwrap(); + let head = repo.head().expect("seeded head").target; + let tree = repo.find_commit(head).unwrap().tree; + let archive = |args: &[&str]| { + let mut request = Vec::new(); + args.iter() + .for_each(|arg| request.extend(pkt(arg.as_bytes()))); + request.extend_from_slice(b"0000"); + knot_pack::upload_archive(&repo, &request).unwrap() + }; + + let raw_arg = format!("argument {}\n", tree.to_hex()); + let raw_oid = archive(&["argument --format=tar\n", raw_arg.as_str()]); + assert!( + String::from_utf8_lossy(&raw_oid).contains("NACK"), + "raw tree oid must be declined like uploadArchive.allowUnreachable=false" + ); + + let traversal = archive(&[ + "argument --format=tar\n", + "argument --prefix=../evil/\n", + "argument HEAD\n", + ]); + assert!( + String::from_utf8_lossy(&traversal).contains("NACK"), + "traversal prefix must be declined" + ); + + repo.update_ref(&RefUpdate::Create { + name: RefName::new("refs/cobs/sh.tangled.repo.collaborator/secret").unwrap(), + new: head, + }) + .unwrap(); + repo.update_ref(&RefUpdate::Delete { + name: RefName::new("refs/heads/main").unwrap(), + old: head, + }) + .unwrap(); + let cob = archive(&[ + "argument --format=tar\n", + "argument refs/cobs/sh.tangled.repo.collaborator/secret\n", + ]); + assert!( + String::from_utf8_lossy(&cob).contains("NACK"), + "archiving cob-only tree must be refused" + ); + assert!( + !cob.windows("README.md".len()) + .any(|window| window == b"README.md"), + "refused archive mustn't leak the hidden tree's contents" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn push_over_http_is_refused() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let name = RepoRkey::new("conch").unwrap(); + layout.create(&did).unwrap(); + let addr = spawn( + knot_pack::router( + layout.clone(), + serve_dids(), + std::sync::Arc::new(knot_runtime::SystemClock), + ), + "[::1]:0", + ) + .await; + + let scratch = tempfile::tempdir().unwrap(); + let work = scratch.path().join("work"); + std::fs::create_dir_all(&work).unwrap(); + must(&work, &["init", "-q", "-b", "main"]); + commit(&work, "a.txt", "one\n", "one"); + let (ok, out) = git(&work, &["push", &url(addr, &did, &name), "main"]); + assert!(!ok, "push over HTTP must be refused, got success:\n{out}"); +} + +fn seed_three_commits(work: &Path, bare: &str) { + seed_repo(work, bare, "a.txt", "c1\n"); + commit(work, "b.txt", "c2\n", "c2"); + must(work, &["push", "-q", bare, "main"]); + commit(work, "c.txt", "c3\n", "c3"); + must(work, &["push", "-q", bare, "main"]); +} + +fn commit_dated(work: &Path, file: &str, contents: &str, message: &str, iso_date: &str) { + std::fs::write(work.join(file), contents).unwrap(); + must(work, &["add", "-A"]); + let out = knot_fixtures::command_at(work, iso_date) + .args(["commit", "-q", "-m", message]) + .output() + .unwrap(); + assert!(out.status.success(), "dated commit failed"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn shallow_clone_depth_exclude_since() { + let did = RepoDid::new("did:plc:squid").unwrap(); + let s = common::stand(&did).await; + let bare = &s.bare; + let scratch = s.scratch.path(); + + let work = scratch.join("work"); + std::fs::create_dir_all(&work).unwrap(); + must(&work, &["init", "-q", "-b", "main"]); + commit_dated(&work, "a.txt", "c1\n", "c1", "2020-01-01T00:00:00 +0000"); + must(&work, &["tag", "base"]); + commit_dated(&work, "b.txt", "c2\n", "c2", "2021-01-01T00:00:00 +0000"); + commit_dated(&work, "c.txt", "c3\n", "c3", "2022-01-01T00:00:00 +0000"); + commit_dated(&work, "d.txt", "c4\n", "c4", "2024-01-01T00:00:00 +0000"); + must( + &work, + &["push", "-q", bare.to_str().unwrap(), "main", "base"], + ); + must(bare, &["symbolic-ref", "HEAD", "refs/heads/main"]); + let remote = format!("http://{}/{}", s.addr, did.as_str()); + + let depth = scratch.join("clone-depth"); + must( + scratch, + &["clone", "--depth=1", "-q", &remote, depth.to_str().unwrap()], + ); + assert!( + depth.join(".git/shallow").exists(), + "depth-limited clone must be marked shallow" + ); + assert_eq!( + must(&depth, &["rev-list", "--count", "HEAD"]).trim(), + "1", + "depth=1 must yield exactly one commit" + ); + let (ok, out) = git(&depth, &["fetch", "--depth=2", "-q", "origin"]); + assert!(ok, "deepening fetch failed:\n{out}"); + assert_eq!( + must(&depth, &["rev-list", "--count", "origin/main"]).trim(), + "2", + "deepen to depth=2 must reveal second commit" + ); + let (ok, out) = git(&depth, &["fetch", "--deepen=1", "-q", "origin"]); + assert!(ok, "relative deepen fetch failed:\n{out}"); + assert_eq!( + must(&depth, &["rev-list", "--count", "origin/main"]).trim(), + "3", + "--deepen=1 from depth 2 must reveal third commit" + ); + let (ok, out) = git(&depth, &["fetch", "--unshallow", "-q", "origin"]); + assert!(ok, "unshallow fetch failed:\n{out}"); + assert!( + !depth.join(".git/shallow").exists(), + "unshallow fetch must drop the shallow marker" + ); + assert_eq!( + must(&depth, &["rev-list", "--count", "origin/main"]).trim(), + "4", + "unshallow must restore full history" + ); + + let exclude = scratch.join("clone-exclude"); + let (ok, out) = git( + scratch, + &[ + "clone", + "--shallow-exclude=base", + "-q", + &remote, + exclude.to_str().unwrap(), + ], + ); + assert!(ok, "shallow-exclude clone failed:\n{out}"); + assert_eq!( + must(&exclude, &["rev-list", "--count", "HEAD"]).trim(), + "3", + "shallow-exclude=base must drop excluded commit and its ancestors" + ); + + let since = scratch.join("clone-since"); + let (ok, out) = git( + scratch, + &[ + "clone", + "--shallow-since=2023-01-01", + "-q", + &remote, + since.to_str().unwrap(), + ], + ); + assert!(ok, "shallow-since clone failed:\n{out}"); + assert_eq!( + must(&since, &["rev-list", "--count", "HEAD"]).trim(), + "1", + "shallow-since must keep only commits at or after cutoff" + ); +} + +#[test] +fn upload_pack_object_set_matches_canonical_git() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + layout.create(&did).unwrap(); + let bare = layout.repo_path(&did).unwrap(); + + let scratch = tempfile::tempdir().unwrap(); + let work = scratch.path().join("work"); + std::fs::create_dir_all(&work).unwrap(); + must(&work, &["init", "-q", "-b", "main"]); + commit(&work, "a.txt", "one\n", "c1"); + let c1 = must(&work, &["rev-parse", "HEAD"]).trim().to_string(); + commit(&work, "a.txt", "two\n", "c2"); + let c2 = must(&work, &["rev-parse", "HEAD"]).trim().to_string(); + must(&work, &["checkout", "-q", "-b", "dev", &c1]); + commit(&work, "b.txt", "three\n", "c3"); + must(&work, &["checkout", "-q", "main"]); + must(&work, &["tag", "-a", "v1", "-m", "release", &c2]); + must( + &work, + &["push", "-q", bare.to_str().unwrap(), "main", "dev", "v1"], + ); + + let repo = layout.open(&did).unwrap(); + let tips: Vec = repo + .advertised_refs() + .unwrap() + .iter() + .map(|record| record.target.to_hex().to_string()) + .collect(); + + let clone = unsideband(&knot_pack::upload_pack(&repo, &v2_fetch_body(&tips, &[])).unwrap()); + assert_eq!( + pack_object_oids(&clone), + pack_object_oids(&canonical_pack(&work, &tips)), + "full clone must transfer exactly the object set canonical git packs" + ); + + let incremental = unsideband( + &knot_pack::upload_pack( + &repo, + &v2_fetch_body(std::slice::from_ref(&c2), std::slice::from_ref(&c1)), + ) + .unwrap(), + ); + assert_eq!( + pack_object_oids(&incremental), + pack_object_oids(&canonical_pack(&work, &[c2, format!("^{c1}")])), + "incremental fetch must transfer only the objects missing from the client" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn http_boundary_encoding_and_streaming() { + use flate2::Compression; + use flate2::write::GzEncoder; + use http_body_util::BodyExt; + use std::io::Write; + use tower::ServiceExt; + + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + layout.create(&did).unwrap(); + let bare = layout.repo_path(&did).unwrap(); + let scratch = tempfile::tempdir().unwrap(); + let work = scratch.path().join("work"); + seed_repo(&work, bare.to_str().unwrap(), "README.md", "boundary\n"); + let tip = must(&work, &["rev-parse", "HEAD"]).trim().to_string(); + + let router = knot_pack::router( + layout, + serve_dids(), + std::sync::Arc::new(knot_runtime::SystemClock), + ); + let upload_uri = format!("/{}/git-upload-pack", did.as_str()); + + let oversized = router + .clone() + .oneshot( + axum::http::Request::builder() + .method("POST") + .uri(upload_uri.clone()) + .body(Body::from(vec![0u8; 17 * 1024 * 1024])) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + oversized.status(), + axum::http::StatusCode::PAYLOAD_TOO_LARGE, + "oversized request body must be refused at the HTTP boundary before buffering" + ); + + let unsupported = router + .clone() + .oneshot( + axum::http::Request::builder() + .method("POST") + .uri(upload_uri.clone()) + .header("content-encoding", "br") + .body(Body::from(vec![0u8; 16])) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + unsupported.status(), + axum::http::StatusCode::UNSUPPORTED_MEDIA_TYPE, + "a body the knot cannot decode is rejected before parsing, never mis-read as identity" + ); + + let mut plain = pkt(b"command=ls-refs\n"); + plain.extend_from_slice(b"0001"); + plain.extend(pkt(b"ref-prefix refs/heads/\n")); + plain.extend_from_slice(b"0000"); + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(&plain).unwrap(); + let gzipped = encoder.finish().unwrap(); + let decoded = router + .clone() + .oneshot( + axum::http::Request::builder() + .method("POST") + .uri(upload_uri.clone()) + .header("content-encoding", "gzip") + .body(Body::from(gzipped)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(decoded.status(), axum::http::StatusCode::OK); + let body = decoded.into_body().collect().await.unwrap().to_bytes(); + assert!( + String::from_utf8_lossy(&body).contains("refs/heads/main"), + "gzip-encoded ls-refs request must be transparently decoded and answered" + ); + + let mut fetch = pkt(b"command=fetch\n"); + fetch.extend_from_slice(b"0001"); + fetch.extend(pkt(format!("want {tip}\n").as_bytes())); + fetch.extend(pkt(b"done\n")); + fetch.extend_from_slice(b"0000"); + let streamed = router + .oneshot( + axum::http::Request::builder() + .method("POST") + .uri(upload_uri) + .body(Body::from(fetch)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(streamed.status(), axum::http::StatusCode::OK); + assert!( + streamed.headers().get(header::CONTENT_LENGTH).is_none(), + "streamed pack response mustn't be buffered into a length-delimited body" + ); + let collected = streamed.into_body().collect().await.unwrap().to_bytes(); + assert!( + collected.windows(4).any(|window| window == b"PACK"), + "streamed response must contain a real PACK" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn the_knot_meta_repo_is_never_served_over_http() { + use tower::ServiceExt; + + let scan = tempfile::tempdir().unwrap(); + let knot = knot_types::KnotId::new("did:web:oyster.cafe").unwrap(); + let layout = Layout::new(scan.path()).reserving_meta(&knot).unwrap(); + layout.bootstrap_meta(&knot).unwrap(); + assert!( + layout.meta_path(&knot).unwrap().exists(), + "meta-repo must exist on disk so this tests the guard, not mere absence" + ); + + let visible = RepoDid::new("did:plc:squid").unwrap(); + layout.create(&visible).unwrap(); + + let router = knot_pack::router( + layout, + serve_dids(), + std::sync::Arc::new(knot_runtime::SystemClock), + ); + + let status = |method: &'static str, uri: &'static str| { + let router = router.clone(); + async move { + router + .oneshot( + axum::http::Request::builder() + .method(method) + .uri(uri) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap() + .status() + } + }; + + let not_found = axum::http::StatusCode::NOT_FOUND; + assert_eq!( + status( + "GET", + "/did:web:oyster.cafe/info/refs?service=git-upload-pack" + ) + .await, + not_found, + "knot DID is refused on GET info/refs" + ); + assert_eq!( + status("POST", "/did:web:oyster.cafe/git-upload-pack").await, + not_found, + "knot DID is refused on POST upload-pack" + ); + assert_eq!( + status( + "GET", + "/did:web:oyster.cafe/anemone/info/refs?service=git-upload-pack" + ) + .await, + not_found, + "knot DID is refused on named info/refs route" + ); + assert_eq!( + status("POST", "/did:web:oyster.cafe/anemone/git-upload-pack").await, + not_found, + "knot DID is refused on named upload-pack route" + ); + assert_eq!( + status("POST", "/did:web:oyster.cafe/git-upload-archive").await, + not_found, + "knot DID is refused on upload-archive route" + ); + assert_eq!( + status("POST", "/did:web:oyster.cafe/anemone/git-upload-archive").await, + not_found, + "knot DID is refused on named upload-archive route" + ); + assert_eq!( + status( + "GET", + "/did:web:OYSTER.cafe/info/refs?service=git-upload-pack" + ) + .await, + not_found, + "case-variant of the knot DID canonicalizes to the same reserved repo" + ); + + assert_eq!( + status("GET", "/did:plc:squid/info/refs?service=git-upload-pack").await, + axum::http::StatusCode::OK, + "ordinary repo path still serves its advertisement" + ); +} diff --git a/knot2/crates/knot-pack/tests/h3_conformance.rs b/knot2/crates/knot-pack/tests/h3_conformance.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/tests/h3_conformance.rs @@ -0,0 +1,579 @@ +use std::collections::BTreeSet; +use std::net::SocketAddr; +use std::num::{NonZeroU32, NonZeroU64}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use bytes::{Buf, Bytes}; +use http::{HeaderMap, Method, Uri}; +use knot_edge::{ + BodyInactivityTimeout, BurstSize, CertSource, EdgeConfig, EdgeGuards, HeaderTimeout, + IdleTimeout, ListenLimits, MaxInflightRequests, RequestTimeout, RequestsPerSecond, + RequiresFullHandshake, StaticCertPaths, TlsSetup, WriteRequestTimeout, +}; +use knot_git::Layout; +use knot_pack::{CacheConfig, RepoLookup, RepoResolver, RepoTarget}; +use knot_types::{ObjectFormat, RepoDid}; +use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}; +use rustls::crypto::aws_lc_rs; +use rustls::pki_types::{CertificateDer, ServerName, UnixTime}; +use rustls::{DigitallySignedStruct, SignatureScheme}; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; + +mod common; +use common::{must, pack_objects, receive_request}; + +type Captured = (Method, Uri, HeaderMap, Bytes); + +fn nz32(value: u32) -> NonZeroU32 { + NonZeroU32::new(value).unwrap() +} + +fn nz64(value: u64) -> NonZeroU64 { + NonZeroU64::new(value).unwrap() +} + +fn object_set(dir: &Path) -> BTreeSet { + must( + dir, + &[ + "cat-file", + "--batch-all-objects", + "--batch-check=%(objectname)", + ], + ) + .lines() + .map(|line| line.trim().to_string()) + .filter(|line| !line.is_empty()) + .collect() +} + +fn serve_dids() -> Arc { + Arc::new(|target: &RepoTarget| match target { + RepoTarget::Did(did) => RepoLookup::Hosted(did.clone()), + RepoTarget::OwnerRkey(_, _) => RepoLookup::Unhosted, + }) +} + +fn free_port() -> u16 { + std::net::TcpListener::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap() + .port() +} + +fn write_self_signed(dir: &Path) -> (PathBuf, PathBuf, Vec) { + let generated = + rcgen::generate_simple_self_signed(vec!["localhost".to_string(), "127.0.0.1".to_string()]) + .unwrap(); + let cert_path = dir.join("cert.pem"); + let key_path = dir.join("key.pem"); + std::fs::write(&cert_path, generated.cert.pem()).unwrap(); + std::fs::write(&key_path, generated.signing_key.serialize_pem()).unwrap(); + let der = generated.cert.der().as_ref().to_vec(); + (cert_path, key_path, der) +} + +fn edge_config(addr: SocketAddr, cert: PathBuf, key: PathBuf) -> EdgeConfig { + EdgeConfig { + http_addr: knot_edge::PublicBind::new(addr), + limits: ListenLimits::new( + HeaderTimeout::from_millis(nz64(30_000)), + IdleTimeout::from_millis(nz64(120_000)), + nz32(1024), + ), + guards: EdgeGuards::new( + RequestsPerSecond::new(nz32(1_000_000)), + BurstSize::new(nz32(1_000_000)), + MaxInflightRequests::new(nz32(10_000)), + RequestTimeout::from_millis(nz64(120_000)), + BodyInactivityTimeout::from_millis(nz64(120_000)), + WriteRequestTimeout::from_millis(nz64(1_800_000)), + None, + ), + tls: Some(TlsSetup { + source: CertSource::Static(StaticCertPaths { + cert_path: knot_edge::CertChainPath::new(cert), + key_path: knot_edge::PrivateKeyPath::new(key), + }), + http3: true, + internal: None, + }), + } +} + +struct Edge { + addr: SocketAddr, + shutdown: CancellationToken, + log: Arc>>, + task: JoinHandle>, + client: quinn::Endpoint, +} + +async fn probe_identity( + endpoint: &quinn::Endpoint, + addr: SocketAddr, + expected_cert: &[u8], +) -> Option { + let connecting = endpoint.connect(addr, "localhost").ok()?; + let connection = tokio::time::timeout(Duration::from_millis(250), connecting) + .await + .ok()? + .ok()?; + let ours = connection + .peer_identity() + .and_then(|identity| identity.downcast::>>().ok()) + .map(|certs| { + certs + .first() + .is_some_and(|cert| cert.as_ref() == expected_cert) + }) + .unwrap_or(false); + connection.close(0u32.into(), b"probe done"); + Some(ours) +} + +async fn await_ready( + endpoint: &quinn::Endpoint, + addr: SocketAddr, + task: &mut JoinHandle>, + expected_cert: &[u8], +) -> bool { + for _ in 0..200 { + if task.is_finished() { + return false; + } + if let Some(ours) = probe_identity(endpoint, addr, expected_cert).await { + return ours; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + false +} + +async fn stand_up(layout: Layout, certdir: &Path) -> Edge { + static TRACE: std::sync::Once = std::sync::Once::new(); + TRACE.call_once(|| { + let _ = tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .try_init(); + }); + for _ in 0..8 { + let addr: SocketAddr = format!("127.0.0.1:{}", free_port()).parse().unwrap(); + let (cert, key, cert_der) = write_self_signed(certdir); + let (write_routes, advertisement) = knot_pack::edge_routes( + layout.clone(), + serve_dids(), + None, + None, + knot_resource::PackSlots::new(4), + CacheConfig::default(), + Arc::new(knot_messages::Catalog::defaults()), + knot_pack::default_hostname().clone(), + Arc::new(knot_runtime::SystemClock), + ); + let log: Arc>> = Arc::new(Mutex::new(Vec::new())); + let sink = log.clone(); + let recorded = write_routes.layer(axum::middleware::from_fn( + move |request: axum::extract::Request, next: axum::middleware::Next| { + let sink = sink.clone(); + async move { + let (parts, body) = request.into_parts(); + let bytes = axum::body::to_bytes(body, usize::MAX) + .await + .unwrap_or_default(); + sink.lock().unwrap().push(( + parts.method.clone(), + parts.uri.clone(), + parts.headers.clone(), + bytes.clone(), + )); + next.run(axum::extract::Request::from_parts( + parts, + axum::body::Body::from(bytes), + )) + .await + } + }, + )); + let app = RequiresFullHandshake::new(recorded); + let shutdown = CancellationToken::new(); + let mut task = tokio::spawn(knot_edge::serve( + edge_config(addr, cert, key), + app, + advertisement, + shutdown.clone(), + )); + let client = h3_client(); + if await_ready(&client, addr, &mut task, &cert_der).await { + return Edge { + addr, + shutdown, + log, + task, + client, + }; + } + client.close(0u32.into(), b"stand up retry"); + shutdown.cancel(); + let _ = task.await; + } + panic!("couldn't bind a free TCP+UDP port for the edge after several attempts"); +} + +#[derive(Debug)] +struct AcceptAnyServerCert; + +impl ServerCertVerifier for AcceptAnyServerCert { + fn verify_server_cert( + &self, + _end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + _server_name: &ServerName<'_>, + _ocsp_response: &[u8], + _now: UnixTime, + ) -> Result { + Ok(ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature( + message, + cert, + dss, + &aws_lc_rs::default_provider().signature_verification_algorithms, + ) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature( + message, + cert, + dss, + &aws_lc_rs::default_provider().signature_verification_algorithms, + ) + } + + fn supported_verify_schemes(&self) -> Vec { + aws_lc_rs::default_provider() + .signature_verification_algorithms + .supported_schemes() + } +} + +fn h3_client() -> quinn::Endpoint { + let mut crypto = + rustls::ClientConfig::builder_with_provider(Arc::new(aws_lc_rs::default_provider())) + .with_protocol_versions(&[&rustls::version::TLS13]) + .unwrap() + .dangerous() + .with_custom_certificate_verifier(Arc::new(AcceptAnyServerCert)) + .with_no_client_auth(); + crypto.alpn_protocols = vec![b"h3".to_vec()]; + let quic = quinn::crypto::rustls::QuicClientConfig::try_from(crypto).unwrap(); + let mut endpoint = quinn::Endpoint::client("127.0.0.1:0".parse().unwrap()).unwrap(); + endpoint.set_default_client_config(quinn::ClientConfig::new(Arc::new(quic))); + endpoint +} + +async fn drain( + stream: &mut h3::client::RequestStream, Bytes>, +) -> Vec { + let mut out = Vec::new(); + while let Some(mut chunk) = stream.recv_data().await.unwrap() { + out.extend_from_slice(&chunk.copy_to_bytes(chunk.remaining())); + } + out +} + +async fn finish_request( + stream: &mut h3::client::RequestStream, Bytes>, +) { + match stream.finish().await { + Ok(()) => (), + Err(h3::error::StreamError::RemoteTerminate { code, .. }) + if code == h3::error::Code::H3_NO_ERROR => {} + Err(error) => panic!("finishing the request stream failed: {error}"), + } +} + +async fn replay_over_h3(edge: &Edge, did: &str, request: &Captured) -> Vec { + let (_, uri, headers, body) = request; + let connection = edge + .client + .connect(edge.addr, "localhost") + .unwrap() + .await + .unwrap(); + let quic = connection.clone(); + let (mut driver, mut sender) = h3::client::new(h3_quinn::Connection::new(connection)) + .await + .unwrap(); + let drive = tokio::spawn(async move { + let _ = std::future::poll_fn(|cx| driver.poll_close(cx)).await; + }); + + let warmup = http::Request::get(format!( + "https://localhost/{did}/info/refs?service=git-upload-pack" + )) + .header("git-protocol", "version=2") + .body(()) + .unwrap(); + let mut warm = sender.send_request(warmup).await.unwrap(); + finish_request(&mut warm).await; + assert!( + warm.recv_response().await.unwrap().status().is_success(), + "h3 info/refs advertisement must serve over QUIC" + ); + drain(&mut warm).await; + + let mut builder = http::Request::builder() + .method(Method::POST) + .uri(format!("https://localhost{}", uri.path())); + for name in ["content-type", "content-encoding", "git-protocol", "accept"] { + if let Some(value) = headers.get(name) { + builder = builder.header(name, value); + } + } + let mut stream = sender + .send_request(builder.body(()).unwrap()) + .await + .unwrap(); + stream.send_data(body.clone()).await.unwrap(); + finish_request(&mut stream).await; + let response = stream.recv_response().await.unwrap(); + assert!( + response.status().is_success(), + "h3 upload-pack returned {}", + response.status() + ); + let out = drain(&mut stream).await; + quic.close(0u32.into(), b"done"); + drive.abort(); + out +} + +fn fetch_request(log: &Arc>>) -> Captured { + let captured = log.lock().unwrap(); + captured + .iter() + .find(|(method, uri, _, body)| { + method == Method::POST + && uri.path().ends_with("/git-upload-pack") + && body + .windows(b"command=fetch".len()) + .any(|window| window == b"command=fetch") + }) + .cloned() + .unwrap_or_else(|| { + let summary: Vec = captured + .iter() + .map(|(method, uri, headers, body)| { + format!( + "{method} {uri} git-protocol={:?} body[..64]={:?}", + headers.get("git-protocol"), + String::from_utf8_lossy(&body[..body.len().min(64)]) + ) + }) + .collect(); + panic!("git issued no protocol-v2 fetch over the TLS edge, captured: {summary:#?}") + }) +} + +fn extract_pack(response: &[u8]) -> Vec { + let mut channel = Vec::new(); + let mut pos = 0usize; + while pos + 4 <= response.len() { + let len = std::str::from_utf8(&response[pos..pos + 4]) + .ok() + .and_then(|hex| usize::from_str_radix(hex, 16).ok()) + .unwrap_or(0); + pos += 4; + if len < 4 { + continue; + } + let end = (pos + len - 4).min(response.len()); + let payload = &response[pos..end]; + pos = end; + if payload.first() == Some(&1) { + channel.extend_from_slice(&payload[1..]); + } + } + match channel.windows(4).position(|window| window == b"PACK") { + Some(start) => channel.split_off(start), + None => channel, + } +} + +fn index_pack(repo: &Path, pack: &[u8]) { + let (indexed, report) = + knot_fixtures::feed(repo, &["index-pack", "--stdin", "--fix-thin"], pack); + assert!(indexed, "index-pack failed: {report}"); +} + +fn advance_knot(bare: &Path, work: &Path, old: &str, new: &str) { + let oids: Vec = must(work, &["rev-list", "--objects", new, "--not", old]) + .lines() + .filter_map(|line| line.split_whitespace().next()) + .map(str::to_string) + .collect(); + let request = receive_request("refs/heads/main", old, new, &pack_objects(work, &oids)); + let repo = knot_git::Repo::open(bare).expect("open knot bare"); + let report = knot_pack::receive_pack(&repo, &request).expect("knot receive"); + assert!( + String::from_utf8_lossy(&report).contains("ok refs/heads/main"), + "knot must accept a receive that advances main" + ); +} + +fn init(dir: &Path, format: ObjectFormat) { + std::fs::create_dir_all(dir).unwrap(); + let fmt = format!("--object-format={}", format.capability()); + must(dir, &["init", &fmt, "-q", dir.to_str().unwrap()]); +} + +fn seed(work: &Path, bares: [&Path; 2], format: ObjectFormat) { + let fmt = format!("--object-format={}", format.capability()); + std::fs::create_dir_all(work).unwrap(); + must(work, &["init", &fmt, "-q", "-b", "main"]); + std::fs::write(work.join("README.md"), "h3 conformance\n").unwrap(); + must(work, &["add", "-A"]); + must(work, &["commit", "-q", "-m", "c1"]); + let c1 = must(work, &["rev-parse", "HEAD"]); + std::fs::write(work.join("src.txt"), "more\n").unwrap(); + must(work, &["add", "-A"]); + must(work, &["commit", "-q", "-m", "c2"]); + must(work, &["checkout", "-q", "-b", "dev", &c1]); + std::fs::write(work.join("dev.txt"), "branch\n").unwrap(); + must(work, &["add", "-A"]); + must(work, &["commit", "-q", "-m", "c3"]); + must(work, &["checkout", "-q", "main"]); + must(work, &["tag", "-a", "v1", "-m", "release"]); + bares.into_iter().for_each(|bare| { + must( + work, + &["push", "-q", bare.to_str().unwrap(), "main", "dev", "v1"], + ); + must(bare, &["symbolic-ref", "HEAD", "refs/heads/main"]); + }); +} + +async fn h3_serves_the_canonical_object_set(format: ObjectFormat, did_str: &str) { + let scan = tempfile::tempdir().unwrap(); + let certdir = tempfile::tempdir().unwrap(); + let scratch = tempfile::tempdir().unwrap(); + let canon_root = tempfile::tempdir().unwrap(); + + let did = RepoDid::new(did_str).unwrap(); + let layout = Layout::new(scan.path()).with_object_format(format); + layout.create(&did).unwrap(); + let knot_bare = layout.repo_path(&did).unwrap(); + let canon_bare = canon_root.path().join("canon.git"); + let fmt = format!("--object-format={}", format.capability()); + must( + canon_root.path(), + &["init", "--bare", &fmt, "-q", canon_bare.to_str().unwrap()], + ); + + let work = scratch.path().join("work"); + seed(&work, [&knot_bare, &canon_bare], format); + + let canon_url = format!("file://{}", canon_bare.to_str().unwrap()); + let canon_clone = scratch.path().join("canon-clone"); + must( + scratch.path(), + &["clone", "-q", &canon_url, canon_clone.to_str().unwrap()], + ); + let canonical = object_set(&canon_clone); + + let edge = stand_up(layout, certdir.path()).await; + let url = format!("https://{}/{}", edge.addr, did.as_str()); + + let h1_clone = scratch.path().join("h1-clone"); + must( + scratch.path(), + &[ + "-c", + "http.sslVerify=false", + "clone", + "-q", + &url, + h1_clone.to_str().unwrap(), + ], + ); + must(&h1_clone, &["config", "http.sslVerify", "false"]); + assert_eq!( + must(&canon_clone, &["rev-parse", "HEAD^{tree}"]), + must(&h1_clone, &["rev-parse", "HEAD^{tree}"]), + "{format:?} h1/h2 TLS clone checks out the canonical tree" + ); + assert_eq!( + canonical, + object_set(&h1_clone), + "{format:?} h1/h2 TLS clone transfers the canonical object set" + ); + + let h3_clone = scratch.path().join("h3-clone"); + init(&h3_clone, format); + let pack = extract_pack(&replay_over_h3(&edge, did.as_str(), &fetch_request(&edge.log)).await); + index_pack(&h3_clone, &pack); + assert_eq!( + canonical, + object_set(&h3_clone), + "{format:?} h3 clone over QUIC transfers the canonical object set" + ); + + let old = must(&work, &["rev-parse", "HEAD"]); + std::fs::write(work.join("incremental.txt"), "fetch me\n").unwrap(); + must(&work, &["add", "-A"]); + must(&work, &["commit", "-q", "-m", "c4"]); + let new = must(&work, &["rev-parse", "HEAD"]); + advance_knot(&knot_bare, &work, &old, &new); + must(&work, &["push", "-q", canon_bare.to_str().unwrap(), "main"]); + + let canon_after = scratch.path().join("canon-after"); + must( + scratch.path(), + &["clone", "-q", &canon_url, canon_after.to_str().unwrap()], + ); + let canonical_after = object_set(&canon_after); + + edge.log.lock().unwrap().clear(); + must(&h1_clone, &["fetch", "-q", "origin"]); + assert_eq!( + canonical_after, + object_set(&h1_clone), + "{format:?} h1/h2 TLS fetch advances to the canonical object set" + ); + + let fetch_pack = + extract_pack(&replay_over_h3(&edge, did.as_str(), &fetch_request(&edge.log)).await); + index_pack(&h3_clone, &fetch_pack); + assert_eq!( + canonical_after, + object_set(&h3_clone), + "{format:?} h3 incremental fetch over QUIC advances to the canonical object set" + ); + + edge.shutdown.cancel(); + edge.task.abort(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn the_git_superset_guarantee_holds_over_h3_in_both_object_formats() { + h3_serves_the_canonical_object_set(ObjectFormat::SHA1, "did:plc:squid").await; + h3_serves_the_canonical_object_set(ObjectFormat::SHA256, "did:plc:cuttle").await; +} diff --git a/knot2/crates/knot-pack/tests/handle_owner.rs b/knot2/crates/knot-pack/tests/handle_owner.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/tests/handle_owner.rs @@ -0,0 +1,122 @@ +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use http_body_util::BodyExt; +use knot_git::Layout; +use knot_pack::{CacheConfig, HandleResolver, RepoLookup, RepoResolver, RepoTarget}; +use knot_types::{AccountDid, Handle, OwnerDid, RepoDid}; +use tower::ServiceExt; + +struct FakeHandles; + +impl HandleResolver for FakeHandles { + fn resolve( + &self, + handle: Handle, + ) -> Pin> + Send + '_>> { + Box::pin(async move { + (handle.as_str() == "nel.pet").then(|| AccountDid::new("did:plc:nel").unwrap()) + }) + } +} + +fn repo_resolver() -> Arc { + let owner = OwnerDid::new("did:plc:nel").unwrap(); + let repo = RepoDid::new("did:plc:whelk").unwrap(); + Arc::new(move |target: &RepoTarget| match target { + RepoTarget::OwnerRkey(o, n) if *o == owner && n.as_str() == "squid" => { + RepoLookup::Hosted(repo.clone()) + } + RepoTarget::Did(d) if *d == repo => RepoLookup::Hosted(d.clone()), + _ => RepoLookup::Unhosted, + }) +} + +fn build(layout: &Layout, handle_resolver: Option>) -> axum::Router { + let (_write, advertisement) = knot_pack::edge_routes( + layout.clone(), + repo_resolver(), + None, + handle_resolver, + knot_resource::PackSlots::new(4), + CacheConfig::default(), + Arc::new(knot_messages::Catalog::defaults()), + knot_pack::default_hostname().clone(), + Arc::new(knot_runtime::SystemClock), + ); + advertisement.into_router() +} + +async fn get(router: axum::Router, uri: &str) -> (StatusCode, Vec) { + let response = router + .oneshot(Request::get(uri).body(Body::empty()).unwrap()) + .await + .unwrap(); + let status = response.status(); + let body = response + .into_body() + .collect() + .await + .unwrap() + .to_bytes() + .to_vec(); + (status, body) +} + +fn hosted_layout() -> (tempfile::TempDir, Layout) { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + layout + .create(&RepoDid::new("did:plc:whelk").unwrap()) + .unwrap(); + (scan, layout) +} + +#[tokio::test] +async fn a_handle_owner_serves_the_same_repo_as_its_did() { + let (_scan, layout) = hosted_layout(); + let (handle_status, handle_body) = get( + build(&layout, Some(Arc::new(FakeHandles))), + "/nel.pet/squid/info/refs?service=git-upload-pack", + ) + .await; + let (did_status, did_body) = get( + build(&layout, Some(Arc::new(FakeHandles))), + "/did:plc:nel/squid/info/refs?service=git-upload-pack", + ) + .await; + assert_eq!(handle_status, StatusCode::OK); + assert_eq!(did_status, StatusCode::OK); + assert_eq!( + handle_body, did_body, + "handle owner and DID owner must serve the same repository" + ); +} + +#[tokio::test] +async fn a_handle_owner_is_not_found_when_unknown_or_unresolvable() { + let (_scan, layout) = hosted_layout(); + let (unknown, _) = get( + build(&layout, Some(Arc::new(FakeHandles))), + "/olaren.dev/squid/info/refs?service=git-upload-pack", + ) + .await; + assert_eq!( + unknown, + StatusCode::NOT_FOUND, + "a handle the resolver rejects isn't found" + ); + let (no_resolver, _) = get( + build(&layout, None), + "/nel.pet/squid/info/refs?service=git-upload-pack", + ) + .await; + assert_eq!( + no_resolver, + StatusCode::NOT_FOUND, + "a handle owner with no resolver configured isn't found" + ); +} diff --git a/knot2/crates/knot-pack/tests/hardening.rs b/knot2/crates/knot-pack/tests/hardening.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/tests/hardening.rs @@ -0,0 +1,1484 @@ +use std::io::Write; +use std::process::Stdio; + +use knot_git::{Layout, RefUpdate}; +use knot_pack::{DeltaDepth, PackLimits}; +use knot_types::{ObjectCount, ObjectFormat, Oid, RefName, RepoDid}; + +mod common; +use common::{ + commit, delta_bomb_pack, index_into_bare, must, pack_objects, pack_objects_tuned, pkt, + receive_request, seeded, unsideband, +}; + +fn generous() -> PackLimits { + PackLimits { + max_objects: ObjectCount::new(1_000_000), + max_object_bytes: knot_pack::MaxObjectBytes::new(1 << 30), + max_total_bytes: knot_pack::MaxTotalBytes::new(1 << 31), + max_delta_depth: DeltaDepth::new(50), + } +} + +fn v2_fetch(wants: &[&str], haves: &[&str], done: bool) -> Vec { + let mut req = pkt(b"command=fetch\n"); + req.extend_from_slice(b"0001"); + wants + .iter() + .for_each(|want| req.extend(pkt(format!("want {want}\n").as_bytes()))); + haves + .iter() + .for_each(|have| req.extend(pkt(format!("have {have}\n").as_bytes()))); + if done { + req.extend(pkt(b"done\n")); + } + req.extend_from_slice(b"0000"); + req +} + +fn thin_resolves_against_base(base_pack: &[u8], thin: &[u8]) -> bool { + let bare = tempfile::tempdir().unwrap(); + knot_fixtures::must( + bare.path(), + &["init", "--bare", "-q", bare.path().to_str().unwrap()], + ); + let feed = |args: &[&str], pack: &[u8]| knot_fixtures::feed(bare.path(), args, pack).0; + feed(&["index-pack", "--stdin"], base_pack) + && feed(&["index-pack", "--stdin", "--fix-thin"], thin) +} + +fn v2_fetch_thin(want: &str, have: &str) -> Vec { + let mut req = pkt(b"command=fetch\n"); + req.extend_from_slice(b"0001"); + req.extend(pkt(b"thin-pack\n")); + req.extend(pkt(format!("want {want}\n").as_bytes())); + req.extend(pkt(format!("have {have}\n").as_bytes())); + req.extend(pkt(b"done\n")); + req.extend_from_slice(b"0000"); + req +} + +fn has_band(resp: &[u8], band: u8) -> bool { + let mut pos = 0usize; + while pos + 4 <= resp.len() { + let len = std::str::from_utf8(&resp[pos..pos + 4]) + .ok() + .and_then(|hex| usize::from_str_radix(hex, 16).ok()) + .unwrap_or(0); + pos += 4; + if len < 4 { + continue; + } + let end = (pos + len - 4).min(resp.len()); + if resp[pos..end].first() == Some(&band) { + return true; + } + pos = end; + } + false +} + +fn created_refs(bare: &knot_git::Repo) -> Vec { + bare.references() + .unwrap() + .into_iter() + .map(|record| record.name.as_str().to_string()) + .collect() +} + +#[test] +fn pack_with_missing_parent_is_now_rejected() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let bare = layout.create(&did).unwrap(); + + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + must(work, &["init", "-q", "-b", "main"]); + commit(work, "old.txt", "old\n", "c1"); + let c1 = must(work, &["rev-parse", "HEAD"]); + must(work, &["rm", "-q", "old.txt"]); + commit(work, "new.txt", "new\n", "c2"); + let c2 = must(work, &["rev-parse", "HEAD"]); + + let oids: Vec = must(work, &["rev-list", "--objects", &c2, "--not", &c1]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + let pack = pack_objects(work, &oids); + + let mut first = format!("{} {} refs/heads/dangle", Oid::null().to_hex(), c2).into_bytes(); + first.push(0); + first.extend_from_slice(b"report-status\n"); + let mut req = pkt(&first); + req.extend_from_slice(b"0000"); + req.extend_from_slice(&pack); + + let report = knot_pack::receive_pack(&bare, &req).unwrap(); + let text = String::from_utf8_lossy(&report).replace('\0', ""); + + let created = layout + .open(&did) + .unwrap() + .references() + .unwrap() + .into_iter() + .any(|r| r.name.as_str() == "refs/heads/dangle"); + + assert!( + text.contains("ng refs/heads/dangle missing necessary objects"), + "pack whose tip has a missing parent must be rejected:\n{text}" + ); + assert!(!created, "dangling ref mustn't have been created"); +} + +#[test] +fn ls_refs_prefix_and_cob_hiding() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let (bare, _work, c1, pack) = seeded(&layout, &did); + knot_pack::receive_pack( + &bare, + &receive_request("refs/heads/main", &Oid::null().to_hex(), &c1, &pack), + ) + .unwrap(); + bare.update_ref(&RefUpdate::Create { + name: RefName::new("refs/cobs/sh.tangled.repo.collaborator/secret").unwrap(), + new: Oid::from_hex(&c1).unwrap(), + }) + .unwrap(); + + let ls = |prefix: Option<&str>| { + let mut req = pkt(b"command=ls-refs\n"); + if let Some(prefix) = prefix { + req.extend_from_slice(b"0001"); + req.extend(pkt(format!("ref-prefix {prefix}\n").as_bytes())); + } + req.extend_from_slice(b"0000"); + String::from_utf8_lossy(&knot_pack::upload_pack(&bare, &req).unwrap()).into_owned() + }; + + let tags = ls(Some("refs/tags/")); + assert!( + !tags.contains("refs/heads/main"), + "tags-only ref-prefix must exclude heads:\n{tags}" + ); + + let heads = ls(Some("refs/heads/")); + assert!( + heads.contains("refs/heads/main"), + "heads ref-prefix must keep heads:\n{heads}" + ); + + let cobs = ls(Some("refs/cobs/")); + assert!( + !cobs.contains("refs/cobs"), + "explicit ref-prefix refs/cobs/ must still return nothing:\n{cobs}" + ); + + let all = ls(None); + assert!( + all.contains("refs/heads/main"), + "unfiltered ls-refs must still advertise heads:\n{all}" + ); + assert!( + !all.contains("refs/cobs"), + "unfiltered ls-refs must never advertise cob refs:\n{all}" + ); +} + +#[test] +fn push_namespace_gating_accepts_only_unreserved_refs() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let (bare, _work, c1, pack) = seeded(&layout, &did); + + let cases: [(&str, &str, bool); 3] = [ + ( + "refs/cobs/sh.tangled.repo.collaborator/evil", + "ng refs/cobs/sh.tangled.repo.collaborator/evil", + false, + ), + ( + "refs/hidden/feature/main", + "ng refs/hidden/feature/main", + false, + ), + ("refs/notes/commits", "ok refs/notes/commits", true), + ]; + cases.iter().for_each(|(refname, verdict, lands)| { + let req = receive_request(refname, &Oid::null().to_hex(), &c1, &pack); + let report = String::from_utf8_lossy(&knot_pack::receive_pack(&bare, &req).unwrap()) + .replace('\0', ""); + assert!(report.contains(*verdict), "{verdict}:\n{report}"); + assert_eq!( + created_refs(&bare) + .iter() + .any(|name| name.as_str() == *refname), + *lands, + "ref landing mismatch for {refname}", + ); + }); +} + +#[test] +fn pack_missing_a_blob_is_rejected() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let bare = layout.create(&did).unwrap(); + + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + must(work, &["init", "-q", "-b", "main"]); + commit(work, "a.txt", "secret\n", "c1"); + let c1 = must(work, &["rev-parse", "HEAD"]); + let tree = must(work, &["rev-parse", "HEAD^{tree}"]); + let pack = pack_objects(work, &[c1.clone(), tree]); + + let req = receive_request("refs/heads/main", &Oid::null().to_hex(), &c1, &pack); + let report = + String::from_utf8_lossy(&knot_pack::receive_pack(&bare, &req).unwrap()).replace('\0', ""); + assert!( + report.contains("ng refs/heads/main missing necessary objects"), + "pack whose tree references a missing blob must be rejected:\n{report}" + ); + assert!( + created_refs(&bare).is_empty(), + "ref to an object-incomplete commit mustn't be created" + ); +} + +#[test] +fn pack_with_a_submodule_gitlink_is_accepted() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let bare = layout.create(&did).unwrap(); + + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + must(work, &["init", "-q", "-b", "main"]); + std::fs::write(work.join("a.txt"), "alpha\n").unwrap(); + must(work, &["add", "a.txt"]); + let absent = "0123456789abcdef0123456789abcdef01234567"; + let cacheinfo = format!("160000,{absent},vendor"); + must( + work, + &["update-index", "--add", "--cacheinfo", cacheinfo.as_str()], + ); + let tree = must(work, &["write-tree"]); + let head = must(work, &["commit-tree", &tree, "-m", "adds a submodule"]); + let oids: Vec = must(work, &["rev-list", "--objects", &head]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + let pack = pack_objects(work, &oids); + + let req = receive_request("refs/heads/main", &Oid::null().to_hex(), &head, &pack); + let report = + String::from_utf8_lossy(&knot_pack::receive_pack(&bare, &req).unwrap()).replace('\0', ""); + assert!( + report.contains("ok refs/heads/main"), + "a gitlink names a submodule commit the host needn't hold, so the push must land:\n{report}" + ); + assert!( + created_refs(&bare) + .iter() + .any(|name| name.as_str() == "refs/heads/main"), + "the submodule-bearing ref must be created" + ); +} + +#[test] +fn empty_root_commit_over_the_virtual_tree_is_accepted() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let bare = layout.create(&did).unwrap(); + + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + must(work, &["init", "-q", "-b", "main"]); + must(work, &["commit", "-q", "--allow-empty", "-m", "empty root"]); + let head = must(work, &["rev-parse", "HEAD"]); + let pack = pack_objects(work, std::slice::from_ref(&head)); + + let req = receive_request("refs/heads/main", &Oid::null().to_hex(), &head, &pack); + let report = + String::from_utf8_lossy(&knot_pack::receive_pack(&bare, &req).unwrap()).replace('\0', ""); + assert!( + report.contains("ok refs/heads/main"), + "an empty root commit points at git's virtual empty tree, so the push must land:\n{report}" + ); + assert!( + created_refs(&bare) + .iter() + .any(|name| name.as_str() == "refs/heads/main"), + "the empty root ref must be created" + ); +} + +fn sole_idx(pack_dir: std::path::PathBuf) -> Vec { + let idx = std::fs::read_dir(&pack_dir) + .unwrap() + .filter_map(Result::ok) + .map(|entry| entry.path()) + .find(|path| path.extension().is_some_and(|ext| ext == "idx")) + .expect("exactly one idx file"); + std::fs::read(idx).unwrap() +} + +fn folded_index_matches_canonical_git(format: ObjectFormat) { + let fmt = format!("--object-format={}", format.capability()); + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + must(work, &["init", &fmt, "-q", "-b", "main"]); + std::fs::create_dir_all(work.join("dir/nested")).unwrap(); + let grow = |lines: usize| { + (0..lines) + .map(|n| format!("line {n}\n")) + .collect::() + }; + (0..40).for_each(|revision| { + std::fs::write(work.join("dir/nested/a.txt"), grow(revision * 50 + 10)).unwrap(); + std::fs::write(work.join("b.txt"), grow(revision * 30 + 5)).unwrap(); + must(work, &["add", "-A"]); + must(work, &["commit", "-q", "-m", &format!("rev {revision}")]); + }); + must(work, &["tag", "-a", "v1", "-m", "release one"]); + let head = must(work, &["rev-parse", "HEAD"]); + let oids: Vec = must(work, &["rev-list", "--objects", "--all"]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + let pack = pack_objects_tuned(work, &oids, true); + + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()).with_object_format(format); + let did = RepoDid::new("did:plc:squid").unwrap(); + let bare = layout.create(&did).unwrap(); + let report = String::from_utf8_lossy( + &knot_pack::receive_pack( + &bare, + &receive_request("refs/heads/main", &format.null_oid().to_hex(), &head, &pack), + ) + .unwrap(), + ) + .replace('\0', ""); + assert!( + report.contains("ok refs/heads/main"), + "fold push must land:\n{report}" + ); + let mine = sole_idx(bare.objects_dir().join("pack")); + + let scratch = tempfile::tempdir().unwrap(); + must(scratch.path(), &["init", "--bare", &fmt, "-q"]); + let mut child = knot_fixtures::command(scratch.path()) + .args(["index-pack", "--stdin"]) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + child.stdin.take().unwrap().write_all(&pack).unwrap(); + assert!( + child.wait().unwrap().success(), + "canonical index-pack failed" + ); + let canonical = sole_idx(scratch.path().join("objects/pack")); + + assert_eq!( + mine, + canonical, + "folded ingest index must be byte-identical to canonical git index-pack for {}", + format.capability() + ); +} + +fn bushy_delta_repo(work: &std::path::Path, fmt: &str) -> String { + must(work, &["init", fmt, "-q", "-b", "main"]); + let body: String = (0..400).map(|n| format!("shared line {n}\n")).collect(); + (0..10).for_each(|revision| { + (0..150).for_each(|file| { + let contents = format!( + "{body}unique {file} rev {revision}\ntail {}\n", + file * 7 + revision + ); + std::fs::write(work.join(format!("f{file:03}.txt")), contents).unwrap(); + }); + must(work, &["add", "-A"]); + must(work, &["commit", "-q", "-m", &format!("rev {revision}")]); + }); + must(work, &["rev-parse", "HEAD"]) +} + +#[test] +fn folded_bushy_push_matches_git_through_the_parallel_delta_path() { + let format = ObjectFormat::SHA1; + let fmt = format!("--object-format={}", format.capability()); + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + let head = bushy_delta_repo(work, &fmt); + let oids: Vec = must(work, &["rev-list", "--objects", "--all"]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + let pack = pack_objects_tuned(work, &oids, true); + + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()).with_object_format(format); + let did = RepoDid::new("did:plc:squid").unwrap(); + let bare = layout.create(&did).unwrap(); + let report = String::from_utf8_lossy( + &knot_pack::receive_pack( + &bare, + &receive_request("refs/heads/main", &format.null_oid().to_hex(), &head, &pack), + ) + .unwrap(), + ) + .replace('\0', ""); + assert!( + report.contains("ok refs/heads/main"), + "bushy push must land:\n{report}" + ); + let mine = sole_idx(bare.objects_dir().join("pack")); + + let scratch = tempfile::tempdir().unwrap(); + must(scratch.path(), &["init", "--bare", &fmt, "-q"]); + let mut child = knot_fixtures::command(scratch.path()) + .args(["index-pack", "--stdin"]) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + child.stdin.take().unwrap().write_all(&pack).unwrap(); + assert!( + child.wait().unwrap().success(), + "canonical index-pack failed" + ); + let canonical = sole_idx(scratch.path().join("objects/pack")); + assert_eq!( + mine, canonical, + "a bushy delta pack drives the work-stealing traversal; its folded index must match canonical git" + ); +} + +#[test] +fn a_forced_base_spill_folds_the_bushy_pack_to_the_canonical_index() { + let format = ObjectFormat::SHA1; + let fmt = format!("--object-format={}", format.capability()); + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + let _head = bushy_delta_repo(work, &fmt); + let oids: Vec = must(work, &["rev-list", "--objects", "--all"]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + let pack = pack_objects_tuned(work, &oids, true); + + let staged = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(staged.path(), &pack).unwrap(); + let objects = tempfile::tempdir().unwrap(); + let folded = knot_pack::bench_ingest_with_base_budget( + objects.path(), + staged.path(), + format.kind(), + Some(4096), + ) + .unwrap(); + assert!( + folded, + "the bushy pack is self-contained, so a forced spill must still fold it" + ); + let mine = sole_idx(objects.path().join("pack")); + + let scratch = tempfile::tempdir().unwrap(); + must(scratch.path(), &["init", "--bare", &fmt, "-q"]); + let mut child = knot_fixtures::command(scratch.path()) + .args(["index-pack", "--stdin"]) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + child.stdin.take().unwrap().write_all(&pack).unwrap(); + assert!( + child.wait().unwrap().success(), + "canonical index-pack failed" + ); + let canonical = sole_idx(scratch.path().join("objects/pack")); + assert_eq!( + mine, canonical, + "paging the delta-base working set to disk mustn't change the folded index" + ); +} + +#[test] +fn the_streaming_connectivity_verify_agrees_with_the_in_ram_map() { + let kind = ObjectFormat::SHA1.kind(); + let stage = |pack: &[u8]| -> Option { + let staged = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(staged.path(), pack).unwrap(); + let objects = tempfile::tempdir().unwrap(); + knot_pack::bench_ingest_external(objects.path(), staged.path(), kind).unwrap() + }; + + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + must(work, &["init", "-q", "-b", "main"]); + commit(work, "a.txt", "hello\n", "c1"); + let all: Vec = must(work, &["rev-list", "--objects", "--all"]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + assert_eq!( + stage(&pack_objects(work, &all)), + Some(true), + "the streaming probe must accept a self-contained pack" + ); + + let c1 = must(work, &["rev-parse", "HEAD"]); + let tree = must(work, &["rev-parse", "HEAD^{tree}"]); + assert_eq!( + stage(&pack_objects(work, &[c1, tree])), + Some(false), + "the streaming probe must reject a pack whose tree references a missing blob" + ); + + let sub_dir = tempfile::tempdir().unwrap(); + let sub = sub_dir.path(); + must(sub, &["init", "-q", "-b", "main"]); + std::fs::write(sub.join("x.txt"), "sub\n").unwrap(); + must(sub, &["add", "x.txt"]); + let cacheinfo = "160000,0123456789abcdef0123456789abcdef01234567,vendor"; + must(sub, &["update-index", "--add", "--cacheinfo", cacheinfo]); + let subtree = must(sub, &["write-tree"]); + let subhead = must(sub, &["commit-tree", &subtree, "-m", "with gitlink"]); + let sub_oids: Vec = must(sub, &["rev-list", "--objects", &subhead]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + assert_eq!( + stage(&pack_objects(sub, &sub_oids)), + Some(true), + "a gitlink names a submodule commit the host needn't hold, so absence is fine" + ); +} + +#[test] +fn folded_first_push_index_is_byte_identical_to_canonical_git() { + folded_index_matches_canonical_git(ObjectFormat::SHA1); +} + +#[test] +fn folded_first_push_index_is_byte_identical_under_sha256() { + folded_index_matches_canonical_git(ObjectFormat::SHA256); +} + +#[test] +fn garbage_pack_reports_unpack_error() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let bare = layout.create(&did).unwrap(); + + let fake = "1111111111111111111111111111111111111111"; + let req = receive_request( + "refs/heads/main", + &Oid::null().to_hex(), + fake, + b"not a packfile", + ); + let report = + String::from_utf8_lossy(&knot_pack::receive_pack(&bare, &req).unwrap()).replace('\0', ""); + assert!( + !report.contains("unpack ok"), + "non-PACK body mustn't report unpack ok:\n{report}" + ); + assert!( + report.contains("unpack pack:") && report.contains("PACK signature"), + "malformed pack must surface an unpack error:\n{report}" + ); + assert!( + created_refs(&bare).is_empty(), + "no ref may be created when pack is malformed" + ); +} + +#[test] +fn delete_only_push_has_no_pack() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let (bare, _work, c1, pack) = seeded(&layout, &did); + + let create = receive_request("refs/heads/main", &Oid::null().to_hex(), &c1, &pack); + let created = String::from_utf8_lossy(&knot_pack::receive_pack(&bare, &create).unwrap()) + .replace('\0', ""); + assert!( + created.contains("unpack ok") && created.contains("ok refs/heads/main"), + "setup push failed:\n{created}" + ); + + let delete = receive_request("refs/heads/main", &c1, &Oid::null().to_hex(), b""); + let report = String::from_utf8_lossy(&knot_pack::receive_pack(&bare, &delete).unwrap()) + .replace('\0', ""); + assert!( + report.contains("unpack ok"), + "delete-only push has no pack and must still report unpack ok:\n{report}" + ); + assert!( + report.contains("ok refs/heads/main"), + "deleting head must succeed:\n{report}" + ); + assert!( + created_refs(&bare).is_empty(), + "ref must be gone after a delete" + ); +} + +#[test] +fn v2_fetch_negotiation_acks_readies_waits_and_ignores_unknowns() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let (bare, _path, old, tip, _blob) = pushed_history(&layout, &did); + + let fake_have = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"; + let nak = String::from_utf8_lossy( + &knot_pack::upload_pack(&bare, &v2_fetch(&[&tip], &[fake_have], false)).unwrap(), + ) + .into_owned(); + assert!( + nak.contains("acknowledgments"), + "must open acknowledgments:\n{nak}" + ); + assert!(nak.contains("NAK"), "no common commit -> NAK:\n{nak}"); + assert!( + !nak.contains("packfile"), + "with no common commit server mustn't send a pack this round:\n{nak}" + ); + + let bytes = knot_pack::upload_pack(&bare, &v2_fetch(&[&tip], &[&old], false)).unwrap(); + let ready = String::from_utf8_lossy(&bytes).into_owned(); + assert!( + ready.contains(&format!("ACK {old}")), + "must ACK common commit:\n{ready}" + ); + assert!( + ready.contains("ready"), + "must declare ready once a common commit is found" + ); + assert!( + ready.contains("packfile"), + "must open packfile section after ready" + ); + assert!( + bytes.windows(4).any(|window| window == b"PACK"), + "side-band payload must contain a real PACK" + ); + + let waiting = String::from_utf8_lossy( + &knot_pack::upload_pack( + &bare, + &v2_fetch_with(&[&tip], &[&old], false, &["wait-for-done"]), + ) + .unwrap(), + ) + .into_owned(); + assert!( + waiting.contains(&format!("ACK {old}")), + "wait-for-done still acknowledges common commit:\n{waiting}" + ); + assert!( + !waiting.contains("ready"), + "wait-for-done mustn't declare ready; it waits for the client's done:\n{waiting}" + ); + assert!( + !waiting.contains("packfile"), + "wait-for-done mustn't open pack before done arrives:\n{waiting}" + ); + let finished = knot_pack::upload_pack( + &bare, + &v2_fetch_with(&[&tip], &[&old], true, &["wait-for-done"]), + ) + .unwrap(); + let finished_text = String::from_utf8_lossy(&finished).into_owned(); + assert!( + finished_text.contains("packfile"), + "once done arrives server opens the pack:\n{finished_text}" + ); + assert!( + finished.windows(4).any(|window| window == b"PACK"), + "follow-up round must contain a real PACK" + ); + + let mut req = pkt(b"command=fetch\n"); + req.extend_from_slice(b"0001"); + [ + "thin-pack\n", + "ofs-delta\n", + "include-tag\n", + "no-progress\n", + "some-future-capability-knot-does-not-know\n", + ] + .iter() + .for_each(|arg| req.extend(pkt(arg.as_bytes()))); + req.extend(pkt(format!("want {tip}\n").as_bytes())); + req.extend(pkt(b"done\n")); + req.extend_from_slice(b"0000"); + let resp = knot_pack::upload_pack(&bare, &req).unwrap(); + let text = String::from_utf8_lossy(&resp); + assert!( + text.contains("packfile"), + "unknown fetch arguments must be ignored and pack still produced:\n{text}" + ); + assert!( + resp.windows(4).any(|window| window == b"PACK"), + "response must still contain a real PACK despite unknown arguments" + ); + + let progress = knot_pack::upload_pack(&bare, &v2_fetch(&[&tip], &[], true)).unwrap(); + assert!( + has_band(&progress, 2), + "fetch must emit sideband band-2 progress" + ); + let suppressed = + knot_pack::upload_pack(&bare, &v2_fetch_with(&[&tip], &[], true, &["no-progress"])) + .unwrap(); + assert!( + !has_band(&suppressed, 2), + "no-progress must suppress band-2 output" + ); + assert!( + suppressed.windows(4).any(|window| window == b"PACK"), + "pack itself must still be sent when progress is suppressed" + ); +} + +fn v2_fetch_with(wants: &[&str], haves: &[&str], done: bool, extra: &[&str]) -> Vec { + let mut req = pkt(b"command=fetch\n"); + req.extend_from_slice(b"0001"); + extra + .iter() + .for_each(|line| req.extend(pkt(format!("{line}\n").as_bytes()))); + wants + .iter() + .for_each(|want| req.extend(pkt(format!("want {want}\n").as_bytes()))); + haves + .iter() + .for_each(|have| req.extend(pkt(format!("have {have}\n").as_bytes()))); + if done { + req.extend(pkt(b"done\n")); + } + req.extend_from_slice(b"0000"); + req +} + +fn pkt_payloads(resp: &[u8]) -> Vec> { + let mut out = Vec::new(); + let mut pos = 0usize; + while pos + 4 <= resp.len() { + let len = std::str::from_utf8(&resp[pos..pos + 4]) + .ok() + .and_then(|hex| usize::from_str_radix(hex, 16).ok()) + .unwrap_or(0); + pos += 4; + if len < 4 { + continue; + } + let end = (pos + len - 4).min(resp.len()); + out.push(resp[pos..end].to_vec()); + pos = end; + } + out +} + +fn banded(payloads: &[Vec], section: &[u8]) -> bool { + payloads + .iter() + .any(|payload| payload.first() == Some(&1) && payload[1..].starts_with(section)) +} + +fn plain(payloads: &[Vec], section: &[u8]) -> bool { + payloads.iter().any(|payload| payload == section) +} + +fn pushed_history( + layout: &Layout, + did: &RepoDid, +) -> (knot_git::Repo, std::path::PathBuf, String, String, String) { + let bare = layout.create(did).unwrap(); + let bare_path = layout.repo_path(did).unwrap(); + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + must(work, &["init", "-q", "-b", "main"]); + commit(work, "a.txt", "1\n", "c1"); + let old = must(work, &["rev-parse", "HEAD"]); + let blob = must(work, &["rev-parse", "HEAD:a.txt"]); + commit(work, "b.txt", "2\n", "c2"); + let tip = must(work, &["rev-parse", "HEAD"]); + must(work, &["push", "-q", bare_path.to_str().unwrap(), "main"]); + (bare, bare_path, old, tip, blob) +} + +#[test] +fn v2_sideband_all_band_frames_negotiation_and_packfile_uris() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let (bare, bare_path, old, tip, blob) = pushed_history(&layout, &did); + + let plain_resp = + knot_pack::upload_pack(&bare, &v2_fetch_with(&[&tip], &[&old], false, &[])).unwrap(); + let plain_payloads = pkt_payloads(&plain_resp); + assert!( + plain(&plain_payloads, b"acknowledgments\n"), + "without sideband-all acknowledgments header is a plain pkt-line" + ); + assert!( + !banded(&plain_payloads, b"acknowledgments\n"), + "without sideband-all negotiation mustn't be band-framed" + ); + + let banded_resp = knot_pack::upload_pack( + &bare, + &v2_fetch_with(&[&tip], &[&old], false, &["sideband-all"]), + ) + .unwrap(); + let banded_payloads = pkt_payloads(&banded_resp); + assert!( + banded(&banded_payloads, b"acknowledgments\n"), + "sideband-all moves acknowledgments header into band 1" + ); + assert!( + banded(&banded_payloads, &format!("ACK {old}\n").into_bytes()), + "sideband-all wraps ACK lines too" + ); + assert!( + !plain(&banded_payloads, b"acknowledgments\n"), + "under sideband-all nothing in negotiation is sent as a plain pkt-line" + ); + + let packhash = "0123456789abcdef0123456789abcdef01234567"; + let uri = format!("https://cdn.nel.pet/{packhash}.pack"); + must( + &bare_path, + &[ + "config", + "uploadpack.blobPackfileUri", + &format!("{blob} {packhash} {uri}"), + ], + ); + let bare = layout.open(&did).unwrap(); + + let plain_uris = knot_pack::upload_pack( + &bare, + &v2_fetch_with(&[&tip], &[], false, &["packfile-uris https"]), + ) + .unwrap(); + let plain_uri_payloads = pkt_payloads(&plain_uris); + assert!( + plain(&plain_uri_payloads, b"packfile-uris\n"), + "without sideband-all packfile-uris header is a plain pkt-line, not band 1" + ); + assert!( + plain( + &plain_uri_payloads, + format!("{packhash} {uri}\n").as_bytes() + ), + "configured blob uri must be advertised verbatim" + ); + + let banded_uris = knot_pack::upload_pack( + &bare, + &v2_fetch_with( + &[&tip], + &[], + false, + &["packfile-uris https", "sideband-all"], + ), + ) + .unwrap(); + let banded_uri_payloads = pkt_payloads(&banded_uris); + assert!( + banded(&banded_uri_payloads, b"packfile-uris\n"), + "sideband-all moves packfile-uris header into band 1" + ); + assert!( + !plain(&banded_uri_payloads, b"packfile-uris\n"), + "under sideband-all packfile-uris header is never a plain pkt-line" + ); +} + +fn big_blob_repo( + layout: &Layout, + did: &RepoDid, + size: usize, +) -> (knot_git::Repo, String, Vec, usize) { + let bare = layout.create(did).unwrap(); + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + must(work, &["init", "-q", "-b", "main"]); + let filler: String = std::iter::repeat_n('a', size).collect(); + commit(work, "big.txt", &filler, "c1"); + let c1 = must(work, &["rev-parse", "HEAD"]); + let oids: Vec = must(work, &["rev-list", "--objects", &c1]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + let count = oids.len(); + let pack = pack_objects(work, &oids); + (bare, c1, pack, count) +} + +#[test] +fn pack_limits_reject_oversized_and_overdeep_packs() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + + let (size_bare, size_tip, size_pack, count) = + big_blob_repo(&layout, &RepoDid::new("did:plc:squid").unwrap(), 256 * 1024); + assert!(count >= 2, "commit packs at least a commit and a tree"); + + let ofs_bare = layout + .create(&RepoDid::new("did:plc:whelk").unwrap()) + .unwrap(); + let ofs_work = tempfile::tempdir().unwrap(); + let ow = ofs_work.path(); + must(ow, &["init", "-q", "-b", "main"]); + let base: String = std::iter::repeat_n('a', 128 * 1024).collect(); + commit(ow, "big.txt", &base, "c1"); + let oc1 = must(ow, &["rev-parse", "HEAD"]); + let mut tweaked = base.clone(); + tweaked.push('b'); + commit(ow, "big.txt", &tweaked, "c2"); + let oc2 = must(ow, &["rev-parse", "HEAD"]); + let ofs_oids: Vec = must(ow, &["rev-list", "--objects", &oc1, &oc2]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + let ofs_pack = pack_objects_tuned(ow, &ofs_oids, true); + + let (ref_bare, ref_tip, ref_pack) = + ref_delta_chain_repo(&layout, &RepoDid::new("did:plc:limpet").unwrap(), 4); + + let cases: [(&knot_git::Repo, &[u8], &str, PackLimits, &str); 5] = [ + ( + &size_bare, + size_pack.as_slice(), + size_tip.as_str(), + PackLimits { + max_object_bytes: knot_pack::MaxObjectBytes::new(4096), + ..generous() + }, + "unpack pack exceeds per-object size limit", + ), + ( + &size_bare, + size_pack.as_slice(), + size_tip.as_str(), + PackLimits { + max_total_bytes: knot_pack::MaxTotalBytes::new(4096), + ..generous() + }, + "unpack pack exceeds total decompressed size limit", + ), + ( + &size_bare, + size_pack.as_slice(), + size_tip.as_str(), + PackLimits { + max_objects: ObjectCount::new(count - 1), + ..generous() + }, + "unpack pack exceeds object count limit", + ), + ( + &ofs_bare, + ofs_pack.as_slice(), + oc2.as_str(), + PackLimits { + max_delta_depth: DeltaDepth::new(0), + ..generous() + }, + "unpack pack exceeds delta chain depth limit", + ), + ( + &ref_bare, + ref_pack.as_slice(), + ref_tip.as_str(), + PackLimits { + max_delta_depth: DeltaDepth::new(0), + ..generous() + }, + "unpack pack exceeds delta chain depth limit", + ), + ]; + cases + .into_iter() + .for_each(|(bare, pack, tip, limits, message)| { + let req = receive_request("refs/heads/main", &Oid::null().to_hex(), tip, pack); + let report = String::from_utf8_lossy( + &knot_pack::receive_pack_with_limits(bare, &req, &limits).unwrap(), + ) + .replace('\0', ""); + assert!(report.contains(message), "{message}:\n{report}"); + assert!( + created_refs(bare).is_empty(), + "refused pack must create no ref" + ); + }); +} + +#[test] +fn a_delta_declaring_an_oversized_result_is_rejected_on_both_ingest_paths() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let limits = generous(); + let bomb = delta_bomb_pack(1 << 40); + let absent_tip = "f".repeat(40); + + let empty = layout + .create(&RepoDid::new("did:plc:cuttle").unwrap()) + .unwrap(); + let fresh_req = receive_request("refs/heads/main", &Oid::null().to_hex(), &absent_tip, &bomb); + let fresh_report = String::from_utf8_lossy( + &knot_pack::receive_pack_with_limits(&empty, &fresh_req, &limits).unwrap(), + ) + .replace('\0', ""); + assert!( + fresh_report.contains("reconstructed size"), + "the fold traversal must refuse a delta declaring an oversized result:\n{fresh_report}" + ); + assert!( + created_refs(&empty).is_empty(), + "refused bomb must create no ref" + ); + + let (seeded_bare, tip, seed_pack, _) = + big_blob_repo(&layout, &RepoDid::new("did:plc:scallop").unwrap(), 1024); + let seed_req = receive_request("refs/heads/main", &Oid::null().to_hex(), &tip, &seed_pack); + let seed_report = String::from_utf8_lossy( + &knot_pack::receive_pack_with_limits(&seeded_bare, &seed_req, &limits).unwrap(), + ) + .replace('\0', ""); + assert!( + seed_report.contains("unpack ok"), + "seeding the repo must succeed:\n{seed_report}" + ); + + let bomb_req = receive_request("refs/heads/bomb", &Oid::null().to_hex(), &absent_tip, &bomb); + let meter_report = String::from_utf8_lossy( + &knot_pack::receive_pack_with_limits(&seeded_bare, &bomb_req, &limits).unwrap(), + ) + .replace('\0', ""); + assert!( + meter_report.contains("per-object size"), + "the meter gate must refuse the bomb on the buffered path:\n{meter_report}" + ); +} + +fn ref_delta_chain_repo( + layout: &Layout, + did: &RepoDid, + revisions: usize, +) -> (knot_git::Repo, String, Vec) { + let bare = layout.create(did).unwrap(); + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + must(work, &["init", "-q", "-b", "main"]); + let tips: Vec = (0..revisions) + .map(|step| { + let body: String = std::iter::repeat_n('a', 64 * 1024).collect(); + commit( + work, + "big.txt", + &format!("{body}{step}\n"), + &format!("c{step}"), + ); + must(work, &["rev-parse", "HEAD"]) + }) + .collect(); + let tip = tips.last().unwrap().clone(); + let oids: Vec = must(work, &["rev-list", "--objects", "HEAD"]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + let pack = pack_objects_tuned(work, &oids, false); + (bare, tip, pack) +} + +#[test] +fn ref_delta_with_in_pack_base_is_resolved() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let (bare, tip, pack) = ref_delta_chain_repo(&layout, &did, 4); + + let req = receive_request("refs/heads/main", &Oid::null().to_hex(), &tip, &pack); + let report = + String::from_utf8_lossy(&knot_pack::receive_pack(&bare, &req).unwrap()).replace('\0', ""); + assert!( + report.contains("unpack ok") && report.contains("ok refs/heads/main"), + "self-contained ref-delta pack, which gix alone cannot index, must be resolved natively:\n{report}" + ); + assert_eq!( + bare.find_ref(&RefName::new("refs/heads/main").unwrap()) + .unwrap(), + Some(Oid::from_hex(&tip).unwrap()), + "ref must point at the pushed tip" + ); + let reopened = layout.open(&did).unwrap(); + assert!( + reopened.find_commit(Oid::from_hex(&tip).unwrap()).is_ok(), + "every resolved object must be readable from the odb after push" + ); +} + +#[test] +fn crafted_stale_old_oid_is_rejected_by_server_cas() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let bare = layout.create(&did).unwrap(); + + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + must(work, &["init", "-q", "-b", "main"]); + commit(work, "a.txt", "one\n", "c1"); + let c1 = must(work, &["rev-parse", "HEAD"]); + commit(work, "a.txt", "two\n", "c2"); + let c2 = must(work, &["rev-parse", "HEAD"]); + + let oids1: Vec = must(work, &["rev-list", "--objects", &c1]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + let create = receive_request( + "refs/heads/main", + &Oid::null().to_hex(), + &c1, + &pack_objects(work, &oids1), + ); + let created = String::from_utf8_lossy(&knot_pack::receive_pack(&bare, &create).unwrap()) + .replace('\0', ""); + assert!( + created.contains("ok refs/heads/main"), + "setup push failed:\n{created}" + ); + + let wrong = "1234567812345678123456781234567812345678"; + let oids2: Vec = must(work, &["rev-list", "--objects", &c2, "--not", &c1]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + let stale = receive_request("refs/heads/main", wrong, &c2, &pack_objects(work, &oids2)); + let report = + String::from_utf8_lossy(&knot_pack::receive_pack(&bare, &stale).unwrap()).replace('\0', ""); + assert!( + report.contains("unpack ok"), + "pack itself is valid and must unpack:\n{report}" + ); + assert!( + report.contains("ng refs/heads/main"), + "crafted stale old-oid must be refused by the server-side compare-and-swap:\n{report}" + ); + assert_eq!( + bare.find_ref(&RefName::new("refs/heads/main").unwrap()) + .unwrap(), + Some(Oid::from_hex(&c1).unwrap()), + "ref must stay at its original tip after a rejected stale update" + ); +} + +#[test] +fn fetch_emits_a_thin_pack_against_client_haves() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let bare = layout.create(&did).unwrap(); + + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + must(work, &["init", "-q", "-b", "main"]); + let big: String = (0..20_000).map(|line| format!("line {line}\n")).collect(); + let small: String = (0..5_000).map(|line| format!("line {line}\n")).collect(); + commit(work, "f.txt", &big, "c1"); + let c1 = must(work, &["rev-parse", "HEAD"]); + commit(work, "f.txt", &small, "c2"); + let c2 = must(work, &["rev-parse", "HEAD"]); + + let oids: Vec = must(work, &["rev-list", "--objects", &c2]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + let pack = pack_objects_tuned(work, &oids, true); + let report = String::from_utf8_lossy( + &knot_pack::receive_pack( + &bare, + &receive_request("refs/heads/main", &Oid::null().to_hex(), &c2, &pack), + ) + .unwrap(), + ) + .replace('\0', ""); + assert!( + report.contains("ok refs/heads/main"), + "setup push failed:\n{report}" + ); + + let thin = unsideband(&knot_pack::upload_pack(&bare, &v2_fetch_thin(&c2, &c1)).unwrap()); + assert!( + !index_into_bare(&[], &thin), + "thin-pack fetch must delta against the client's have and omit it, so it cannot resolve standalone" + ); + + let base_oids: Vec = must(work, &["rev-list", "--objects", &c1]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + assert!( + thin_resolves_against_base(&pack_objects(work, &base_oids), &thin), + "client that already has the base must resolve the thin pack w/ --fix-thin" + ); + + let fat = unsideband(&knot_pack::upload_pack(&bare, &v2_fetch(&[&c2], &[&c1], true)).unwrap()); + assert!( + index_into_bare(&[], &fat), + "without thin-pack same fetch must be self-contained" + ); +} + +#[test] +fn valid_pack_passes_default_limits() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let (bare, c1, pack, _) = big_blob_repo(&layout, &did, 64 * 1024); + + let req = receive_request("refs/heads/main", &Oid::null().to_hex(), &c1, &pack); + let report = + String::from_utf8_lossy(&knot_pack::receive_pack(&bare, &req).unwrap()).replace('\0', ""); + assert!( + report.contains("unpack ok") && report.contains("ok refs/heads/main"), + "valid pack within the default limits must be accepted:\n{report}" + ); +} + +#[test] +fn selection_and_full_clone_expansion_abort_past_object_limit_and_deadline() { + use std::time::Duration; + + use knot_bench::{ChurnCount, CommitCount, HistorySpec, PathCount, build_history}; + use knot_git::{Filter, PackBudget, SelectionLimit}; + + let history = build_history(HistorySpec { + commits: CommitCount::new(8), + paths: PathCount::new(16), + churn: ChurnCount::new(2), + }); + let repo = history.repo(); + let tips = history.tips(); + let stall = Duration::from_secs(60); + + let selected = repo + .select_pack_objects_filtered( + knot_git::Wants::new(&tips), + knot_git::Haves::new(&[]), + Filter::None, + PackBudget::unbounded(), + ) + .unwrap() + .send + .len(); + assert!( + selected > 4, + "fixture must contain more objects than the limit under test, has {selected}" + ); + assert!( + matches!( + repo.select_pack_objects_filtered( + knot_git::Wants::new(&tips), + knot_git::Haves::new(&[]), + Filter::None, + PackBudget::new(ObjectCount::new(4), stall) + ), + Err(knot_git::GitError::Selection(SelectionLimit::Objects)) + ), + "selection past the object-set limit must abort, freeing its core within the budget" + ); + assert!( + matches!( + repo.select_pack_objects_filtered( + knot_git::Wants::new(&tips), + knot_git::Haves::new(&[]), + Filter::None, + PackBudget::new(ObjectCount::new(usize::MAX), Duration::ZERO) + ), + Err(knot_git::GitError::Selection(SelectionLimit::Time)) + ), + "selection that makes no progress within its stall window must abort with a time limit, not a partial pack" + ); + + let dir = repo.objects_dir(); + let fmt = repo.object_format().kind(); + let roots = repo.clone_roots(&tips, PackBudget::unbounded()).unwrap(); + let expanded = knot_pack::count_expanded( + &dir, + roots.clone(), + ObjectCount::new(usize::MAX), + stall, + fmt, + ) + .unwrap() + .len(); + assert!( + expanded > 4, + "fixture must contain more objects than the limit under test, has {expanded}" + ); + assert!( + matches!( + knot_pack::count_expanded(&dir, roots.clone(), ObjectCount::new(4), stall, fmt), + Err(knot_pack::PackError::SelectionTooLarge) + ), + "full-clone expansion past the object limit must abort before the entry stream opens" + ); + assert!( + matches!( + knot_pack::count_expanded( + &dir, + roots, + ObjectCount::new(usize::MAX), + Duration::ZERO, + fmt + ), + Err(knot_pack::PackError::SelectionTimeout) + ), + "full-clone expansion that makes no progress within its stall window must abort with a time limit, \ + not a partial pack" + ); +} + +#[test] +fn parallel_selection_matches_the_oracle_and_honors_its_budget() { + use std::time::Duration; + + use knot_bench::{ChurnCount, CommitCount, HistorySpec, PathCount, build_history}; + use knot_git::{Filter, PackBudget, SelectionLimit}; + + let history = build_history(HistorySpec { + commits: CommitCount::new(5000), + paths: PathCount::new(2), + churn: ChurnCount::new(1), + }); + let repo = history.repo(); + let tips = history.tips(); + let stall = Duration::from_secs(60); + + let selected = repo + .select_pack_objects_filtered( + knot_git::Wants::new(&tips), + knot_git::Haves::new(&[]), + Filter::None, + PackBudget::unbounded(), + ) + .unwrap() + .send; + + let dir = repo.objects_dir(); + let fmt = repo.object_format().kind(); + let roots = repo.clone_roots(&tips, PackBudget::unbounded()).unwrap(); + let expanded = knot_pack::count_expanded(&dir, roots, ObjectCount::new(usize::MAX), stall, fmt) + .unwrap() + .len(); + assert_eq!( + selected.len(), + expanded, + "the parallel selection walk must reach the same object set as the expansion oracle" + ); + + assert!( + matches!( + repo.select_pack_objects_filtered( + knot_git::Wants::new(&tips), + knot_git::Haves::new(&[]), + Filter::None, + PackBudget::new(ObjectCount::new(4), stall) + ), + Err(knot_git::GitError::Selection(SelectionLimit::Objects)) + ), + "the parallel walk must honor its object limit" + ); + assert!( + matches!( + repo.select_pack_objects_filtered( + knot_git::Wants::new(&tips), + knot_git::Haves::new(&[]), + Filter::None, + PackBudget::new(ObjectCount::new(usize::MAX), Duration::ZERO) + ), + Err(knot_git::GitError::Selection(SelectionLimit::Time)) + ), + "the parallel walk with no progress budget must abort on its stall window" + ); +} + +#[test] +fn full_clone_roots_expand_a_directly_wanted_tree() { + use std::time::Duration; + + use knot_bench::{ChurnCount, CommitCount, HistorySpec, PathCount, build_history}; + use knot_git::{Filter, PackBudget}; + use knot_types::Oid; + + let history = build_history(HistorySpec { + commits: CommitCount::new(4), + paths: PathCount::new(16), + churn: ChurnCount::new(2), + }); + let repo = history.repo(); + let tip = history.tip(); + let tree = Oid::from( + repo.git() + .rev_parse_single(format!("{}^{{tree}}", tip.to_hex()).as_bytes()) + .unwrap() + .detach(), + ); + + let slow = repo + .select_pack_objects_filtered( + knot_git::Wants::new(&[tree]), + knot_git::Haves::new(&[]), + Filter::None, + PackBudget::unbounded(), + ) + .unwrap() + .send + .len(); + assert!( + slow > 1, + "directly-wanted tree must include its blobs and subtrees, the selection walk found {slow}" + ); + + let stall = Duration::from_secs(60); + let roots = repo.clone_roots(&[tree], PackBudget::unbounded()).unwrap(); + let fast = knot_pack::count_expanded( + &repo.objects_dir(), + roots, + ObjectCount::new(usize::MAX), + stall, + repo.object_format().kind(), + ) + .unwrap() + .len(); + assert_eq!( + fast, slow, + "full-clone fast path must enumerate the same object set as the selection walk \ + for a directly-wanted tree" + ); +} diff --git a/knot2/crates/knot-pack/tests/quarantine.rs b/knot2/crates/knot-pack/tests/quarantine.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/tests/quarantine.rs @@ -0,0 +1,583 @@ +use std::path::Path; +use std::time::Instant; + +use knot_cob::{CobHome, CobStore}; +use knot_cobs::{Grant, MembersChange}; +use knot_git::{Layout, Repo}; +use knot_pack::{ + MaxWireBytes, PackLimits, PackReceiver, ReceiveCommand, ReceiveFramer, ReceiveGuard, + RefDecision, receive_pack_guarded, receive_request_complete, sweep_incoming, +}; +use knot_runtime::{K256Signer, SeededEntropy, Signer}; +use knot_types::{AccountDid, ActorId, ObjectFormat, Oid, RepoDid, UnixSeconds}; + +const SHA1: ObjectFormat = ObjectFormat::SHA1; + +mod common; +use common::{commit, must, pack_objects, receive_request}; + +fn limits() -> PackLimits { + PackLimits::default() +} + +fn seeded_pack(layout: &Layout, did: &RepoDid) -> (Repo, String, Vec) { + let bare = layout.create(did).unwrap(); + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + must(work, &["init", "-q", "-b", "main"]); + commit(work, "a.txt", "x\n", "c1"); + let c1 = must(work, &["rev-parse", "HEAD"]); + let oids: Vec = must(work, &["rev-list", "--objects", &c1]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + let pack = pack_objects(work, &oids); + (bare, c1, pack) +} + +fn report_text(report: &[u8]) -> String { + String::from_utf8_lossy(report).replace('\0', "") +} + +fn dir_count(path: &Path) -> usize { + std::fs::read_dir(path) + .map(|entries| entries.filter_map(Result::ok).count()) + .unwrap_or(0) +} + +fn live_object_count(repo: &Repo) -> usize { + let objects = repo.objects_dir(); + let loose: usize = std::fs::read_dir(&objects) + .map(|entries| { + entries + .filter_map(Result::ok) + .filter(|entry| { + entry + .file_name() + .to_str() + .is_some_and(|name| name.len() == 2) + && entry.path().is_dir() + }) + .map(|shard| dir_count(&shard.path())) + .sum() + }) + .unwrap_or(0); + let packs = std::fs::read_dir(objects.join("pack")) + .map(|entries| { + entries + .filter_map(Result::ok) + .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "pack")) + .count() + }) + .unwrap_or(0); + loose + packs +} + +struct AllowPublic; + +impl ReceiveGuard for AllowPublic { + fn authorize(&self, _staged: &Repo, commands: &[ReceiveCommand]) -> Vec { + commands + .iter() + .map(|command| { + if command.name().is_some_and(knot_git::is_public_ref) { + RefDecision::Allow + } else { + RefDecision::Reject("not public ref".to_string()) + } + }) + .collect() + } +} + +struct DenyAll; + +impl ReceiveGuard for DenyAll { + fn authorize(&self, _staged: &Repo, commands: &[ReceiveCommand]) -> Vec { + commands + .iter() + .map(|_| RefDecision::Reject("unauthorized".to_string())) + .collect() + } +} + +struct AllowAll; + +impl ReceiveGuard for AllowAll { + fn authorize(&self, _staged: &Repo, commands: &[ReceiveCommand]) -> Vec { + commands.iter().map(|_| RefDecision::Allow).collect() + } +} + +struct CobVerify { + owner: ActorId, + home: CobHome, +} + +impl ReceiveGuard for CobVerify { + fn authorize(&self, staged: &Repo, commands: &[ReceiveCommand]) -> Vec { + commands + .iter() + .map(|command| { + let store = CobStore::new(staged); + let Ok(name) = knot_types::RefName::new(command.refname()) else { + return RefDecision::Reject("invalid ref name".to_string()); + }; + match knot_cobs::verify_cob_ref(&store, &self.home, &name, &self.owner) { + Ok(_) => RefDecision::Allow, + Err(error) => RefDecision::Reject(error.to_string()), + } + }) + .collect() + } +} + +#[test] +fn a_denied_push_leaves_no_objects_in_the_live_odb() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let (bare, c1, pack) = seeded_pack(&layout, &did); + + assert_eq!( + live_object_count(&bare), + 0, + "fresh bare repo has no objects" + ); + let request = receive_request("refs/heads/main", &Oid::null().to_hex(), &c1, &pack); + let report = receive_pack_guarded( + &bare, + &request, + &limits(), + &DenyAll, + &|_| {}, + &knot_pack::default_catalog().reject, + ) + .unwrap() + .report; + let report = report_text(&report); + assert!( + report.contains("ng refs/heads/main unauthorized"), + "{report}" + ); + + assert!( + bare.references().unwrap().is_empty(), + "denied push mustn't create the ref" + ); + assert_eq!( + live_object_count(&bare), + 0, + "denied push must leave no objects in the live odb" + ); + assert!(!bare.contains(Oid::from_hex(&c1).unwrap())); +} + +#[test] +fn an_authorized_push_migrates_objects_then_a_stale_push_is_rejected() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let (bare, c1, pack) = seeded_pack(&layout, &did); + + let report = receive_pack_guarded( + &bare, + &receive_request("refs/heads/main", &Oid::null().to_hex(), &c1, &pack), + &limits(), + &AllowPublic, + &|_| {}, + &knot_pack::default_catalog().reject, + ) + .unwrap() + .report; + let report = report_text(&report); + assert!(report.contains("unpack ok"), "{report}"); + assert!(report.contains("ok refs/heads/main"), "{report}"); + let refs = bare.references().unwrap(); + assert_eq!(refs.len(), 1); + assert_eq!(refs[0].name.as_str(), "refs/heads/main"); + assert_eq!(refs[0].target, Oid::from_hex(&c1).unwrap()); + assert!(bare.contains(Oid::from_hex(&c1).unwrap())); + let after_first = live_object_count(&bare); + + let wrong_old = "1".repeat(40); + let fresh = "2".repeat(40); + let report = receive_pack_guarded( + &bare, + &receive_request("refs/heads/main", &wrong_old, &fresh, b""), + &limits(), + &AllowPublic, + &|_| {}, + &knot_pack::default_catalog().reject, + ) + .unwrap() + .report; + let report = report_text(&report); + assert!(report.contains("ng refs/heads/main"), "{report}"); + assert_eq!( + bare.find_ref(&knot_types::RefName::new("refs/heads/main").unwrap()) + .unwrap(), + Some(Oid::from_hex(&c1).unwrap()), + "stale push mustn't move the ref" + ); + assert_eq!( + live_object_count(&bare), + after_first, + "stale push mustn't add objects to the live odb" + ); +} + +#[test] +fn a_cob_ref_verifies_against_the_owner_key_and_is_refused_for_a_stranger() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let signer = K256Signer::generate(&SeededEntropy::new(7)); + + let source_did = RepoDid::new("did:plc:source").unwrap(); + let home = CobHome::from(&source_did); + let source = layout.create(&source_did).unwrap(); + let store = CobStore::new(&source); + let grant = Grant { + subject: AccountDid::new("did:plc:nel").unwrap(), + added_by: AccountDid::new("did:plc:nel").unwrap(), + created_at: UnixSeconds::new(1), + }; + let created = store + .create( + &home, + &MembersChange::Add(grant), + &signer, + UnixSeconds::new(1), + ) + .unwrap(); + let tip = created.tip.oid().to_hex(); + let refname = format!( + "refs/cobs/sh.tangled.knot.member/{}", + created.object.oid().to_hex() + ); + + let reachable: Vec = must(source.path(), &["rev-list", "--objects", &tip]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + let pack = pack_objects(source.path(), &reachable); + let owner = ActorId::from_secp256k1(signer.public_key().as_bytes()); + + let dest = layout + .create(&RepoDid::new("did:plc:dest").unwrap()) + .unwrap(); + let report = receive_pack_guarded( + &dest, + &receive_request(&refname, &Oid::null().to_hex(), &tip, &pack), + &limits(), + &CobVerify { + owner: owner.clone(), + home: home.clone(), + }, + &|_| {}, + &knot_pack::default_catalog().reject, + ) + .unwrap() + .report; + assert!( + report_text(&report).contains(&format!("ok {refname}")), + "owner-signed COB ref must be accepted at the receive boundary: {}", + report_text(&report) + ); + + let stranger_key = K256Signer::generate(&SeededEntropy::new(9)); + let stranger = ActorId::from_secp256k1(stranger_key.public_key().as_bytes()); + let dest2 = layout + .create(&RepoDid::new("did:plc:dest2").unwrap()) + .unwrap(); + let report = receive_pack_guarded( + &dest2, + &receive_request(&refname, &Oid::null().to_hex(), &tip, &pack), + &limits(), + &CobVerify { + owner: stranger, + home: home.clone(), + }, + &|_| {}, + &knot_pack::default_catalog().reject, + ) + .unwrap() + .report; + assert!( + report_text(&report).contains(&format!("ng {refname}")), + "COB ref not signed by the resolved owner key must be refused: {}", + report_text(&report) + ); + assert_eq!( + live_object_count(&dest2), + 0, + "refused COB ref push leaves no objects behind" + ); +} + +#[test] +fn a_reserved_ref_update_is_refused_even_when_the_guard_allows_it() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let bare = layout.create(&did).unwrap(); + + let cob_ref = format!("refs/cobs/sh.tangled.knot.member/{}", "a".repeat(40)); + let old = "1".repeat(40); + let new = "2".repeat(40); + let report = receive_pack_guarded( + &bare, + &receive_request(&cob_ref, &old, &new, b""), + &limits(), + &AllowAll, + &|_| {}, + &knot_pack::default_catalog().reject, + ) + .unwrap() + .report; + let report = report_text(&report); + + assert!( + report.contains(&format!("ng {cob_ref}")), + "non-create update to a reserved ref must be refused even under an allow-all guard: {report}" + ); + assert!( + report.contains("cannot be modified"), + "refusal must name the create-only rule, not connectivity or compare-and-swap: {report}" + ); + assert!( + bare.references().unwrap().is_empty(), + "refused reserved-ref update must land nothing" + ); + assert_eq!( + live_object_count(&bare), + 0, + "refused reserved-ref update must migrate no objects" + ); +} + +fn pseudo_random(seed: u64, len: usize) -> Vec { + let mut state = seed.wrapping_mul(0x9E37_79B9_7F4A_7C15).wrapping_add(1); + (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + (state >> 24) as u8 + }) + .collect() +} + +fn incompressible_pack(megabytes: usize) -> Vec { + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + must(work, &["init", "-q", "-b", "main"]); + (0..megabytes).for_each(|i| { + let blob = pseudo_random(i as u64 + 1, 1024 * 1024); + std::fs::write(work.join(format!("blob-{i:04}.bin")), blob).unwrap(); + }); + must(work, &["add", "-A"]); + must(work, &["commit", "-q", "-m", "bulk"]); + let head = must(work, &["rev-parse", "HEAD"]); + let oids: Vec = must(work, &["rev-list", "--objects", &head]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + pack_objects(work, &oids) +} + +#[test] +fn the_receive_framer_scans_incrementally_in_linear_time() { + const READ_CHUNK: usize = 64 * 1024; + let pack = incompressible_pack(24); + let body = receive_request( + "refs/heads/main", + &Oid::null().to_hex(), + &"1".repeat(40), + &pack, + ); + assert!( + pack.len() > 8 * 1024 * 1024, + "pack must be large enough to expose any quadratic scaling: {} bytes", + pack.len() + ); + + let single_start = Instant::now(); + let complete = ReceiveFramer::new(limits(), SHA1.kind()) + .advance_bytes(&body) + .unwrap(); + let single = single_start.elapsed(); + assert_eq!( + complete, + Some(body.len()), + "one advance over the full body frames whole request" + ); + + let chunks = body.len().div_ceil(READ_CHUNK); + let chunked_start = Instant::now(); + let mut framer = ReceiveFramer::new(limits(), SHA1.kind()); + let mut framed_at = None; + (1..=chunks).for_each(|n| { + let end = (n * READ_CHUNK).min(body.len()); + if framed_at.is_none() + && let Some(total) = framer.advance_bytes(&body[..end]).unwrap() + { + framed_at = Some(total); + } + }); + let chunked = chunked_start.elapsed(); + assert_eq!( + framed_at, + Some(body.len()), + "feeding the same body in 64KB chunks frames it at identical length" + ); + + let ratio = chunked.as_secs_f64() / single.as_secs_f64().max(1e-6); + assert!( + ratio < 5.0, + "resumable framer never re-inflates settled objects, so the {chunks}-chunk feed \ + must stay within a small constant of one pass; got {ratio:.2}x" + ); +} + +#[test] +fn receive_request_completion_framing_and_the_per_object_limit() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let (_bare, c1, pack) = seeded_pack(&layout, &did); + let request = receive_request("refs/heads/main", &Oid::null().to_hex(), &c1, &pack); + + assert_eq!( + receive_request_complete(&request[..request.len() / 2], &limits(), SHA1.kind()).unwrap(), + None, + "half-delivered request isn't yet complete" + ); + assert_eq!( + receive_request_complete(&request, &limits(), SHA1.kind()).unwrap(), + Some(request.len()), + "whole request reports its exact length" + ); + let mut trailing = request.clone(); + trailing.extend_from_slice(b"junk-after-the-pack"); + assert_eq!( + receive_request_complete(&trailing, &limits(), SHA1.kind()).unwrap(), + Some(request.len()), + "framer stops at the pack trailer, ignoring trailing bytes" + ); + + let tight = PackLimits { + max_object_bytes: knot_pack::MaxObjectBytes::new(1), + ..PackLimits::default() + }; + assert!( + receive_request_complete(&request, &tight, SHA1.kind()).is_err(), + "object beyond the per-object limit is refused by the framer" + ); +} + +#[test] +fn sweep_incoming_removes_abandoned_staging_directories() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let bare = layout.create(&did).unwrap(); + + let staging = bare.path().join(".knot-incoming.4242.0"); + std::fs::create_dir_all(staging.join("objects")).unwrap(); + std::fs::write(staging.join("objects").join("leftover"), b"x").unwrap(); + assert!(staging.exists()); + + let swept = sweep_incoming(scan.path()); + assert_eq!(swept, 1, "exactly one staging directory is swept"); + assert!(!staging.exists(), "abandoned staging directory is gone"); + assert!( + bare.path().join("objects").exists(), + "live repository is untouched" + ); +} + +#[test] +fn streamed_receipt_reassembles_a_request_byte_for_byte() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let (_bare, c1, pack) = seeded_pack(&layout, &did); + let request = receive_request("refs/heads/main", &Oid::null().to_hex(), &c1, &pack); + + let dir = tempfile::tempdir().unwrap(); + let mut receiver = PackReceiver::new( + dir.path(), + MaxWireBytes::new(1 << 30), + limits(), + SHA1.kind(), + ) + .unwrap(); + let completed = request + .chunks(7) + .try_fold(false, |_, chunk| receiver.write(chunk)) + .unwrap(); + assert!( + completed, + "the framer must detect completion within the streamed request" + ); + + let body = receiver.finish().unwrap(); + let mut reconstructed = body.preamble().to_vec(); + if let Some(pack) = body.open_pack().unwrap() { + reconstructed.extend_from_slice(&std::fs::read(pack.path()).unwrap()); + } + assert_eq!( + &reconstructed[..], + &request[..], + "the streamed body must be byte-identical to the buffered request" + ); + assert_eq!( + receive_request_complete(&request, &limits(), SHA1.kind()).unwrap(), + Some(request.len()), + "streamed total must agree with the buffered framer" + ); +} + +#[test] +fn streamed_receipt_handles_empty_and_delete_only_requests() { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let (_bare, c1, _pack) = seeded_pack(&layout, &did); + let dir = tempfile::tempdir().unwrap(); + + let empty = PackReceiver::new( + dir.path(), + MaxWireBytes::new(1 << 30), + limits(), + SHA1.kind(), + ) + .unwrap() + .finish() + .unwrap(); + assert!( + empty.is_empty(), + "a stream with no bytes yields an empty body" + ); + + let delete = receive_request("refs/heads/main", &c1, &Oid::null().to_hex(), &[]); + let mut receiver = PackReceiver::new( + dir.path(), + MaxWireBytes::new(1 << 30), + limits(), + SHA1.kind(), + ) + .unwrap(); + let completed = delete + .chunks(5) + .try_fold(false, |_, chunk| receiver.write(chunk)) + .unwrap(); + assert!(completed, "a delete-only push completes without a pack"); + let body = receiver.finish().unwrap(); + assert_eq!(body.preamble(), &delete[..]); + assert!( + body.open_pack().unwrap().is_none(), + "a delete-only push has no pack" + ); +} diff --git a/knot2/crates/knot-pack/tests/serving.rs b/knot2/crates/knot-pack/tests/serving.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/tests/serving.rs @@ -0,0 +1,312 @@ +use std::sync::Arc; + +use axum::Router; +use axum::body::Body; +use axum::http::{Request, StatusCode, header}; +use http_body_util::BodyExt; +use knot_git::{ + EntryKind, Identity, Layout, NewCommit, RefUpdate, Repo, StagedAction, StagedChange, +}; +use knot_pack::{PackError, PackLimits, RepoLookup, RepoResolver, RepoTarget, ingest_pack}; +use knot_types::{AuthorName, BranchName, Email, ObjectFormat, Oid, RefName, RepoDid, UnixSeconds}; +use tempfile::TempDir; +use tower::ServiceExt; + +mod common; +use common::pkt; + +fn serve_dids() -> Arc { + Arc::new(|target: &RepoTarget| match target { + RepoTarget::Did(did) => RepoLookup::Hosted(did.clone()), + RepoTarget::OwnerRkey(_, _) => RepoLookup::Unhosted, + }) +} + +fn commit_blob(repo: &Repo, path: &str, content: &[u8], parents: Vec) -> Oid { + let base = Oid::from(gix::ObjectId::empty_tree(repo.object_format().kind())); + let id = Identity { + name: AuthorName::new("nel"), + email: Email::new("nel@oyster.cafe"), + time: UnixSeconds::new(1_700_000_000), + offset_seconds: 0, + }; + let tree = repo + .write_staged_tree( + base, + &[StagedChange { + path: knot_types::RepoPath::new(path).unwrap(), + action: StagedAction::Put { + content: content.to_vec(), + kind: EntryKind::Blob, + }, + }], + ) + .unwrap(); + repo.write_commit(&NewCommit { + tree, + parents, + author: id.clone(), + committer: id, + message: "c".to_string(), + extra_headers: Vec::new(), + }) + .unwrap() +} + +fn create_ref(repo: &Repo, name: &str, new: Oid) { + repo.update_ref(&RefUpdate::Create { + name: RefName::new(name).unwrap(), + new, + }) + .unwrap(); +} + +fn seed(format: ObjectFormat) -> (TempDir, Layout, RepoDid, Oid) { + let dir = tempfile::tempdir().unwrap(); + let layout = Layout::new(dir.path().join("scan")) + .with_object_format(format) + .with_default_branch(BranchName::new("main").unwrap()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let repo = layout.create(&did).unwrap(); + let tip = commit_blob(&repo, "reef.txt", b"kelp forest\n", Vec::new()); + create_ref(&repo, "refs/heads/main", tip); + (dir, layout, did, tip) +} + +fn v2_fetch_body(tip: Oid, server_option: bool) -> Vec { + let mut body = pkt(b"command=fetch\n"); + body.extend_from_slice(b"0001"); + body.extend(pkt(format!("want {tip}\n").as_bytes())); + if server_option { + body.extend(pkt(b"server-option=ci-skip\n")); + } + body.extend(pkt(b"done\n")); + body.extend_from_slice(b"0000"); + body +} + +async fn post(router: &Router, did: &str, body: Vec) -> axum::http::Response { + let request = Request::builder() + .method("POST") + .uri(format!("/{did}/git-upload-pack")) + .header("git-protocol", "version=2") + .header( + header::CONTENT_TYPE, + "application/x-git-upload-pack-request", + ) + .body(Body::from(body)) + .unwrap(); + router.clone().oneshot(request).await.unwrap() +} + +async fn post_upload(router: &Router, did: &str, body: Vec) -> Vec { + let response = post(router, did, body).await; + assert_eq!(response.status(), StatusCode::OK); + response + .into_body() + .collect() + .await + .unwrap() + .to_bytes() + .to_vec() +} + +fn hide_secret_ref(repo: &Repo) { + let path = repo.git().git_dir().join("config"); + let mut config = std::fs::read_to_string(&path).unwrap(); + config.push_str("\n[uploadpack]\n\thideRefs = refs/heads/secret\n"); + std::fs::write(&path, config).unwrap(); +} + +fn refused(result: Result, PackError>) -> bool { + matches!(result, Err(PackError::Protocol(_))) +} + +#[test] +fn the_v2_advertisement_offers_server_option_and_a_fetch_using_it_is_served() { + [ObjectFormat::SHA1, ObjectFormat::SHA256] + .into_iter() + .for_each(|format| { + let (_dir, layout, did, tip) = seed(format); + let repo = layout.open(&did).unwrap(); + let advert = knot_pack::advertise_upload(&repo).unwrap(); + assert!( + String::from_utf8_lossy(&advert).contains("server-option"), + "{format:?} advert" + ); + let served = knot_pack::upload_pack(&repo, &v2_fetch_body(tip, true)).unwrap(); + assert!( + String::from_utf8_lossy(&served).contains("packfile"), + "{format:?} server-option fetch" + ); + }); +} + +#[test] +fn upload_pack_refuses_malformed_unreachable_and_hidden_wants() { + let (_dir, layout, did, tip) = seed(ObjectFormat::SHA1); + let repo = layout.open(&did).unwrap(); + + let dangling = commit_blob(&repo, "dangle.txt", b"dangling\n", Vec::new()); + assert!( + refused(knot_pack::upload_pack( + &repo, + &v2_fetch_body(dangling, false) + )), + "unreachable want" + ); + + let mut malformed = pkt(b"command=fetch\n"); + malformed.extend_from_slice(b"0001"); + malformed.extend(pkt(b"want not-a-valid-object-id\n")); + malformed.extend(pkt(b"done\n")); + malformed.extend_from_slice(b"0000"); + assert!( + refused(knot_pack::upload_pack(&repo, &malformed)), + "malformed want line" + ); + + let writer = layout.open(&did).unwrap(); + let secret = commit_blob(&writer, "secret.txt", b"hidden\n", vec![tip]); + create_ref(&writer, "refs/heads/secret", secret); + hide_secret_ref(&writer); + + let repo = layout.open(&did).unwrap(); + let named = |scope| { + repo.advertised_refs_for(scope) + .unwrap() + .iter() + .any(|record| record.name.as_str() == "refs/heads/secret") + }; + assert!( + named(knot_git::AdvertScope::Receive), + "hidden ref still public on receive advert" + ); + assert!( + !named(knot_git::AdvertScope::Upload), + "hideRefs strips it from upload advert" + ); + assert!( + refused(knot_pack::upload_pack(&repo, &v2_fetch_body(secret, false))), + "upload-hidden ref by oid" + ); +} + +#[tokio::test] +async fn the_pack_cache_replays_then_invalidates_when_a_ref_is_hidden() { + let (dir, layout, did, tip) = seed(ObjectFormat::SHA1); + let writer = layout.open(&did).unwrap(); + let secret = commit_blob(&writer, "secret.txt", b"hidden\n", vec![tip]); + create_ref(&writer, "refs/heads/secret", secret); + + let router = knot_pack::router( + layout, + serve_dids(), + std::sync::Arc::new(knot_runtime::SystemClock), + ); + + let body = v2_fetch_body(tip, false); + let first = post_upload(&router, did.as_str(), body.clone()).await; + let second = post_upload(&router, did.as_str(), body).await; + assert_eq!( + first, second, + "a cache hit replays the leader's bytes exactly" + ); + let fork = Repo::create(dir.path().join("fork.git")).unwrap(); + ingest_pack( + &fork.objects_dir(), + &common::unsideband(&first), + &PackLimits::default(), + fork.object_format().kind(), + ) + .unwrap(); + assert!( + fork.contains(tip), + "the cached pack contains the wanted tip" + ); + + let secret_body = v2_fetch_body(secret, false); + let warm = post_upload(&router, did.as_str(), secret_body.clone()).await; + assert!( + !common::unsideband(&warm).is_empty(), + "the visible secret want is served and cached" + ); + + hide_secret_ref(&writer); + let after = post(&router, did.as_str(), secret_body).await.status(); + assert_eq!( + after, + StatusCode::BAD_REQUEST, + "once hidden the cached pack isn't replayed" + ); +} + +fn maint_opts() -> knot_maintenance::Options { + knot_maintenance::Options { + repack_max_objects: knot_maintenance::ObjectCount::new(1_000_000), + geometric_factor: knot_maintenance::GeometricFactor::full_repack(), + prune_grace: knot_maintenance::PruneGrace::from_secs(0), + reflog_floor: knot_maintenance::ReflogRetention::from_secs(i64::MAX as u64 / 4), + commit_graph: false, + multi_pack_index: false, + bitmap: true, + } +} + +fn pack_object_count(pack: &[u8]) -> u32 { + assert_eq!( + &pack[..4], + b"PACK", + "a served body begins with the pack signature" + ); + u32::from_be_bytes([pack[8], pack[9], pack[10], pack[11]]) +} + +#[test] +fn the_bitmap_fast_path_serves_the_same_pack_as_the_object_walk() { + [ObjectFormat::SHA1, ObjectFormat::SHA256] + .into_iter() + .for_each(|format| { + let (dir, layout, did, tip) = seed(format); + let repo = layout.open(&did).unwrap(); + let body = v2_fetch_body(tip, false); + let walk = common::unsideband(&knot_pack::upload_pack(&repo, &body).unwrap()); + + let now = UnixSeconds::new(1_700_000_500); + assert!( + knot_maintenance::run_repo(&repo, now, &maint_opts()) + .unwrap() + .bitmap, + "{format:?} seed packs a bitmap" + ); + + let fast = common::unsideband(&knot_pack::upload_pack(&repo, &body).unwrap()); + assert_eq!( + pack_object_count(&fast), + pack_object_count(&walk), + "{format:?} reuse vs walk count" + ); + + let did = RepoDid::new("did:plc:clam").unwrap(); + let fork = Layout::new(dir.path().join("fork")) + .with_object_format(format) + .create(&did) + .unwrap(); + ingest_pack( + &fork.objects_dir(), + &fast, + &PackLimits::default(), + fork.object_format().kind(), + ) + .unwrap(); + let closure: std::collections::HashSet = fork + .select_pack_objects(knot_git::Wants::new(&[tip]), knot_git::Haves::new(&[])) + .unwrap() + .into_iter() + .collect(); + assert!( + fork.contains(tip) && closure.len() as u32 == pack_object_count(&fast), + "{format:?} fast-path ingests as tip closure" + ); + }); +} diff --git a/knot2/crates/knot-pack/tests/soak.rs b/knot2/crates/knot-pack/tests/soak.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/tests/soak.rs @@ -0,0 +1,273 @@ +use std::net::SocketAddr; +use std::path::Path; +use std::process::{Child, Stdio}; +use std::time::Duration; + +use axum::Router; +use knot_bench::{ChurnCount, CommitCount, HistorySpec, PathCount, write_history}; +use knot_git::Layout; +use knot_pack::{RepoLookup, RepoResolver, RepoTarget}; +use knot_types::RepoDid; + +mod common; +use common::must; + +fn serve_dids() -> std::sync::Arc { + std::sync::Arc::new(|target: &RepoTarget| match target { + RepoTarget::Did(did) => RepoLookup::Hosted(did.clone()), + RepoTarget::OwnerRkey(_, _) => RepoLookup::Unhosted, + }) +} + +const BLOB_BYTES: usize = 16 * 1024 * 1024; +const CONCURRENCY: usize = 10; +const ROUNDS: usize = 5; +const PAGE_BYTES: u64 = 4096; +const OOM_CEILING: u64 = 1024 * 1024 * 1024; +const GROWTH_SLACK: u64 = 64 * 1024 * 1024; +const CURVE_LEVELS: [usize; 5] = [1, 2, 4, 8, 16]; +const PER_CONNECTION_CEILING: u64 = 96 * 1024 * 1024; +const SUBLINEAR_SLACK: u64 = 32 * 1024 * 1024; + +static RSS_GATE: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +fn incompressible(len: usize) -> Vec { + let mut state = 0x9e37_79b9_7f4a_7c15u64; + (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + (state & 0xff) as u8 + }) + .collect() +} + +async fn spawn(router: Router) -> SocketAddr { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, router).await.unwrap(); + }); + addr +} + +fn rss_bytes() -> u64 { + let statm = std::fs::read_to_string("/proc/self/statm").expect("/proc/self/statm is readable"); + statm + .split_whitespace() + .nth(1) + .and_then(|pages| pages.parse::().ok()) + .map(|pages| pages * PAGE_BYTES) + .expect("statm lists the resident page count") +} + +fn clone_child(remote: &str, dest: &Path) -> Child { + knot_fixtures::command(dest.parent().unwrap_or(dest)) + .args(["clone", "--bare", "--quiet", remote, dest.to_str().unwrap()]) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .expect("git clone spawns") +} + +struct Soak { + _scan: tempfile::TempDir, + scratch: tempfile::TempDir, + remote: String, + tip: String, +} + +async fn serve_large_repo() -> Soak { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + layout.create(&did).unwrap(); + let bare = layout.repo_path(&did).unwrap(); + + let scratch = tempfile::tempdir().unwrap(); + let work = scratch.path().join("work"); + std::fs::create_dir_all(&work).unwrap(); + must(&work, &["init", "-q", "-b", "main"]); + std::fs::write(work.join("big.bin"), incompressible(BLOB_BYTES)).unwrap(); + must(&work, &["add", "-A"]); + must(&work, &["commit", "-q", "-m", "large"]); + must(&work, &["push", "-q", bare.to_str().unwrap(), "main"]); + must(bare.as_path(), &["symbolic-ref", "HEAD", "refs/heads/main"]); + let tip = must(&work, &["rev-parse", "HEAD"]); + + let addr = spawn(knot_pack::router( + layout, + serve_dids(), + std::sync::Arc::new(knot_runtime::SystemClock), + )) + .await; + let remote = format!("http://{addr}/{}", did.as_str()); + + Soak { + _scan: scan, + scratch, + remote, + tip, + } +} + +async fn serve_wide_history() -> Soak { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + let repo = layout.create(&did).unwrap(); + let tip = write_history( + &repo, + HistorySpec { + commits: CommitCount::new(256), + paths: PathCount::new(4096), + churn: ChurnCount::new(16), + }, + ) + .to_hex(); + + let scratch = tempfile::tempdir().unwrap(); + let addr = spawn(knot_pack::router( + layout, + serve_dids(), + std::sync::Arc::new(knot_runtime::SystemClock), + )) + .await; + let remote = format!("http://{addr}/{}", did.as_str()); + + Soak { + _scan: scan, + scratch, + remote, + tip, + } +} + +fn drain_storm(remote: &str, scratch: &Path, round: usize, concurrency: usize, tip: &str) -> u64 { + let dests: Vec = (0..concurrency) + .map(|index| scratch.join(format!("clone-{round}-{index}"))) + .collect(); + let mut children: Vec = dests.iter().map(|dest| clone_child(remote, dest)).collect(); + + let mut peak = rss_bytes(); + let mut pending = true; + while pending { + peak = peak.max(rss_bytes()); + std::thread::sleep(Duration::from_millis(3)); + pending = children + .iter_mut() + .any(|child| matches!(child.try_wait(), Ok(None))); + } + + children.into_iter().enumerate().for_each(|(index, child)| { + let out = child.wait_with_output().unwrap(); + assert!( + out.status.success(), + "round {round} clone {index} failed:\n{}", + String::from_utf8_lossy(&out.stderr) + ); + }); + + dests.iter().for_each(|dest| { + assert_eq!( + must(dest, &["rev-parse", "HEAD"]), + tip, + "soak clone must reproduce repo tip" + ); + must(dest, &["fsck", "--connectivity-only", "--no-progress"]); + std::fs::remove_dir_all(dest).unwrap(); + }); + + peak.max(rss_bytes()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn concurrent_clone_memory_cost_curve() { + let _rss_gate = RSS_GATE.lock().await; + let soak = serve_wide_history().await; + + let baseline = rss_bytes(); + let mut peak = baseline; + let curve: Vec<(usize, u64)> = CURVE_LEVELS + .into_iter() + .map(|concurrency| { + peak = peak.max(drain_storm( + &soak.remote, + soak.scratch.path(), + concurrency, + concurrency, + &soak.tip, + )); + (concurrency, peak.saturating_sub(baseline)) + }) + .collect(); + + curve.iter().for_each(|(concurrency, delta)| { + let per_connection = delta / *concurrency as u64; + println!( + "{concurrency} concurrent clones: cumulative +{} MiB, ~{} MiB per connection", + delta / (1024 * 1024), + per_connection / (1024 * 1024) + ); + assert!( + per_connection <= PER_CONNECTION_CEILING, + "per-connection high-water for {concurrency} clones is {} MiB, past {} MiB ceiling", + per_connection / (1024 * 1024), + PER_CONNECTION_CEILING / (1024 * 1024) + ); + }); + + let (_, single) = curve[0]; + let (top, top_delta) = *curve.last().unwrap(); + let top_per_connection = top_delta / top as u64; + assert!( + top_per_connection <= single + SUBLINEAR_SLACK, + "memory grows super-linearly with concurrency. {top} clones cost {} MiB per connection \ + against {} MiB for single clone, past the {} MiB slack. Pack is shared, so the \ + per-connection high-water must stay flat as connections rise", + top_per_connection / (1024 * 1024), + single / (1024 * 1024), + SUBLINEAR_SLACK / (1024 * 1024) + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn concurrent_clones_of_a_large_repo_stay_bounded() { + let _rss_gate = RSS_GATE.lock().await; + let soak = serve_large_repo().await; + let scratch = &soak.scratch; + let remote = soak.remote.clone(); + let tip = soak.tip.clone(); + + let baseline = rss_bytes(); + let mut peak = baseline; + let after_round: Vec = (0..ROUNDS) + .map(|round| { + peak = peak.max(drain_storm( + &remote, + scratch.path(), + round, + CONCURRENCY, + &tip, + )); + rss_bytes() + }) + .collect(); + + assert!( + peak < OOM_CEILING, + "serving {CONCURRENCY} concurrent clones mustn't balloon resident memory: peak {} MiB exceeds {} MiB ceiling", + peak / (1024 * 1024), + OOM_CEILING / (1024 * 1024) + ); + + let settled = after_round[..ROUNDS - 1].iter().copied().max().unwrap(); + let last = after_round[ROUNDS - 1]; + assert!( + last <= settled + GROWTH_SLACK, + "resident memory is still climbing at final round, a leak: settled at {} MiB, round {ROUNDS} left {} MiB", + settled / (1024 * 1024), + last / (1024 * 1024) + ); +} diff --git a/knot2/crates/knot-postreceive/src/lib.rs b/knot2/crates/knot-postreceive/src/lib.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-postreceive/src/lib.rs @@ -0,0 +1,447 @@ +use std::collections::BTreeMap; +use std::time::{Duration, Instant}; + +use knot_events::{ + CommitCount, EmailCommitCount, GitRefUpdate, LanguageSize, RefUpdateMeta, Reservation, +}; +use knot_git::{EntryKind, Haves, RefUpdate, Repo, Wants}; +use knot_messages::{CiLogsKey, PushMessages, UrlKey}; +use knot_types::{ + AccountDid, AppviewEndpoint, BranchName, ChangedFiles, CiLogsAddr, Email, Handle, Listing, Oid, + OwnerDid, PushOptions, RefName, RefTransition, RepoDid, RepoPath, RepoRkey, +}; +use knot_workflow::{Compiled, RawWorkflow, Trigger, WorkflowName}; +use url::Url; + +const WORKFLOW_DIR: &str = ".tangled/workflows"; + +pub struct Actor { + pub committer: AccountDid, + pub owner: Option, + pub repo: RepoDid, +} + +pub enum Ci { + Skip, + Compile { + logs: Option, + verbose: bool, + }, +} + +pub enum OwnerLabel { + Handle(Handle), + Did(OwnerDid), +} + +impl OwnerLabel { + pub fn as_str(&self) -> &str { + match self { + OwnerLabel::Handle(handle) => handle.as_str(), + OwnerLabel::Did(did) => did.as_str(), + } + } +} + +pub struct PullLink { + pub appview: AppviewEndpoint, + pub owner: OwnerLabel, + pub rkey: RepoRkey, +} + +struct SourceBranch(BranchName); +struct TargetBranch(BranchName); + +knot_types::scalar_newtype! { + pub struct LanguagesPushBudget(Duration); +} + +struct PushContext<'a> { + actor: &'a Actor, + ci: &'a Ci, + push_options: &'a PushOptions, + pull: Option<&'a PullLink>, + languages_budget: LanguagesPushBudget, + messages: &'a PushMessages, +} + +#[allow(clippy::too_many_arguments)] +pub fn post_receive( + repo: &Repo, + actor: &Actor, + applied: Vec<(RefUpdate, Reservation)>, + ci: &Ci, + push_options: &PushOptions, + pull: Option<&PullLink>, + languages_budget: LanguagesPushBudget, + messages: &PushMessages, +) -> Vec { + let context = PushContext { + actor, + ci, + push_options, + pull, + languages_budget, + messages, + }; + applied + .into_iter() + .flat_map(|(update, reservation)| publish_one(repo, update, reservation, &context)) + .collect() +} + +fn publish_one( + repo: &Repo, + update: RefUpdate, + reservation: Reservation, + context: &PushContext, +) -> Vec { + let name = update.name(); + let transition = update.transition(); + let changed = match transition.new_oid() { + Some(new) => changed_paths(repo, name, transition.old_oid(), new), + None => ChangedFiles::none(), + }; + let pipeline = ci_messages(repo, name, transition, &changed, context); + let event = GitRefUpdate::new( + context.actor.repo.clone(), + context.actor.owner.clone(), + context.actor.committer.clone(), + ) + .on_ref(name.clone(), transition, repo.object_format()) + .with_push_options(context.push_options) + .with_changed_files(changed); + let event = match transition.new_oid() { + Some(new) => event.with_meta(ref_update_meta( + repo, + name, + transition.old_oid(), + new, + context.languages_budget, + )), + None => event, + }; + reservation.fulfill(&event); + + let pull_link = match (transition, context.pull) { + (RefTransition::Create { .. }, Some(link)) => { + pull_request_message(repo, link, name, context.messages).unwrap_or_default() + } + _ => Vec::new(), + }; + pull_link.into_iter().chain(pipeline).collect() +} + +fn changed_paths(repo: &Repo, name: &RefName, old: Option, new: Oid) -> ChangedFiles { + let range = knot_git::PatchRange { + base: old, + head: new, + }; + match repo.changed_paths(range) { + Ok(changed) => { + if changed.listing() == Listing::Truncated { + tracing::warn!( + ref_name = name.as_str(), + path = %repo.path().display(), + files = changed.paths().len(), + "changed-file listing truncated at the record budget, leaving every paths constraint assumed matched" + ); + } + changed + } + Err(error) => { + tracing::warn!( + ref_name = name.as_str(), + path = %repo.path().display(), + %error, + "changed-file listing failed, leaving every paths constraint assumed matched" + ); + ChangedFiles::unknown() + } + } +} + +fn pull_request_message( + repo: &Repo, + link: &PullLink, + name: &RefName, + messages: &PushMessages, +) -> Option> { + let branch = branch_short(name)?; + let default_ref = repo.default_branch()?; + let default = branch_short(&default_ref)?; + if branch == default { + return None; + } + repo.find_ref(&default_ref).ok().flatten()?; + if repo.origin_url().is_some() { + return None; + } + let url = pull_url( + &link.appview, + &link.owner, + &link.rkey, + &SourceBranch(branch), + &TargetBranch(default), + )?; + Some(messages.pull_request.lines(|UrlKey::Url| url.to_string())) +} + +fn pull_url( + appview: &AppviewEndpoint, + owner: &OwnerLabel, + repo: &RepoRkey, + source: &SourceBranch, + target: &TargetBranch, +) -> Option { + let mut url = Url::parse(appview.as_str()).ok()?; + url.path_segments_mut().ok()?.pop_if_empty().extend([ + owner.as_str(), + repo.as_str(), + "pulls", + "new", + ]); + url.query_pairs_mut() + .append_pair("source", "branch") + .append_pair("sourceBranch", source.0.as_str()) + .append_pair("targetBranch", target.0.as_str()); + Some(url) +} + +fn ref_update_meta( + repo: &Repo, + name: &RefName, + old: Option, + new: Oid, + languages_budget: LanguagesPushBudget, +) -> RefUpdateMeta { + let is_default_ref = is_default_branch(repo, name); + let by_email = commit_counts(repo, name, old, new); + let languages = match is_default_ref { + true => language_sizes(repo, new, languages_budget), + false => Vec::new(), + }; + RefUpdateMeta::new(is_default_ref, by_email, languages) +} + +fn is_default_branch(repo: &Repo, name: &RefName) -> bool { + match (branch_short(name), repo.default_branch()) { + (Some(short), Some(default)) => branch_short(&default) == Some(short), + _ => false, + } +} + +fn branch_short(name: &RefName) -> Option { + name.as_str() + .strip_prefix("refs/heads/") + .and_then(|short| BranchName::new(short).ok()) +} + +fn commit_counts(repo: &Repo, name: &RefName, old: Option, new: Oid) -> Vec { + let tip = match repo.peel_to_commit(new) { + Ok(tip) => tip, + Err(error) => { + tracing::warn!( + ref_name = name.as_str(), + path = %repo.path().display(), + %error, + "commit tally failed peeling new tip" + ); + return Vec::new(); + } + }; + let haves = match old { + Some(old) => match repo.peel_to_commit(old) { + Ok(base) => vec![base], + Err(error) => { + tracing::warn!( + ref_name = name.as_str(), + path = %repo.path().display(), + %error, + "commit tally failed reading prior tip" + ); + return Vec::new(); + } + }, + None => sibling_tips(repo, name), + }; + let walked = match repo.rev_walk(Wants::new(&[tip]), Haves::new(&haves)) { + Ok(oids) => oids, + Err(error) => { + tracing::warn!( + ref_name = name.as_str(), + path = %repo.path().display(), + %error, + "commit tally walk failed" + ); + return Vec::new(); + } + }; + let tallies = walked + .into_iter() + .filter_map(|oid| repo.find_commit(oid).ok()) + .fold( + BTreeMap::::new(), + |mut counts, commit| { + let slot = counts.entry(commit.author.email).or_default(); + *slot = slot.succ(); + counts + }, + ); + tallies + .into_iter() + .map(|(email, count)| EmailCommitCount::new(email, count)) + .collect() +} + +fn sibling_tips(repo: &Repo, name: &RefName) -> Vec { + match repo.references() { + Ok(records) => records + .into_iter() + .filter(|record| { + record.name.as_str() != name.as_str() + && record.name.as_str().starts_with("refs/heads/") + }) + .filter_map(|record| repo.peel_to_commit(record.target).ok()) + .collect(), + Err(error) => { + tracing::warn!( + ref_name = name.as_str(), + path = %repo.path().display(), + %error, + "sibling ref scan failed" + ); + Vec::new() + } + } +} + +fn language_sizes( + repo: &Repo, + new: Oid, + languages_budget: LanguagesPushBudget, +) -> Vec { + let deadline = Instant::now() + languages_budget.get(); + match knot_langs::analyze(repo, new, Some(deadline)) { + Ok(sizes) => sizes + .into_iter() + .filter(|(_, size)| size.get() > 0) + .map(|(name, size)| LanguageSize::new(name, size)) + .collect(), + Err(error) => { + tracing::warn!( + path = %repo.path().display(), + commit = %new.to_hex(), + %error, + "language breakdown failed" + ); + Vec::new() + } + } +} + +fn ci_messages( + repo: &Repo, + name: &RefName, + transition: RefTransition, + changed: &ChangedFiles, + context: &PushContext, +) -> Vec { + let Ci::Compile { logs, verbose } = context.ci else { + return Vec::new(); + }; + let Some(new) = transition.new_oid() else { + return Vec::new(); + }; + let templates = context.messages; + let raws = read_workflows(repo, new); + let compiled = knot_workflow::compile( + &raws, + &Trigger::Push { + ref_name: name.clone(), + }, + changed, + ); + let listed = compiled.any_listed_match(); + let Compiled { + workflows, + diagnostics, + } = compiled; + let mut messages = diagnostics.errors; + if *verbose { + let clean = messages.is_empty() && diagnostics.warnings.is_empty(); + messages.extend(diagnostics.warnings); + match (workflows.is_empty(), clean) { + (true, _) => messages.extend(templates.pipeline_none.text_lines()), + (false, true) => messages.extend(templates.pipeline_clean.text_lines()), + (false, false) => {} + } + } + if let Some(addr) = logs.as_ref().filter(|_| listed) { + messages.extend(templates.ci_logs.lines(|key| match key { + CiLogsKey::Host => addr.host().to_string(), + CiLogsKey::Port => addr.port().to_string(), + CiLogsKey::Repo => context.actor.repo.to_string(), + CiLogsKey::Sha => new.to_hex(), + })); + } + messages +} + +fn read_workflows(repo: &Repo, new: Oid) -> Vec { + let commit = match repo.peel_to_commit(new) { + Ok(commit) => commit, + Err(error) => { + tracing::warn!( + commit = %new.to_hex(), + path = %repo.path().display(), + %error, + "workflow read failed peeling commit" + ); + return Vec::new(); + } + }; + let workflow_dir = RepoPath::new(WORKFLOW_DIR).expect("literal workflow dir is well-formed"); + let entries = match repo.tree_entries_at(commit, Some(&workflow_dir)) { + Ok(entries) => entries.unwrap_or_default(), + Err(error) => { + tracing::warn!( + path = %repo.path().display(), + commit = %new.to_hex(), + %error, + "workflow directory read failed" + ); + return Vec::new(); + } + }; + entries + .into_iter() + .filter(|entry| matches!(entry.kind, EntryKind::Blob | EntryKind::BlobExecutable)) + .filter_map(|entry| { + let name = match WorkflowName::new(entry.name.as_str()) { + Ok(name) => name, + Err(error) => { + tracing::warn!( + workflow = %entry.name, + path = %repo.path().display(), + %error, + "workflow name rejected" + ); + return None; + } + }; + match repo.read_blob(entry.oid) { + Ok(contents) => Some(RawWorkflow { name, contents }), + Err(error) => { + tracing::warn!( + workflow = %entry.name, + path = %repo.path().display(), + %error, + "workflow unreadable" + ); + None + } + } + }) + .collect() +} diff --git a/knot2/crates/knot-postreceive/tests/post_receive.rs b/knot2/crates/knot-postreceive/tests/post_receive.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-postreceive/tests/post_receive.rs @@ -0,0 +1,709 @@ +use std::path::Path; +use std::time::Duration; + +use knot_events::{EventCursor, EventLog, Reservation}; +use knot_git::{Layout, RefUpdate, Repo}; +use knot_postreceive::{Actor, Ci, LanguagesPushBudget, OwnerLabel, PullLink, post_receive}; +use knot_runtime::{ManualClock, UnixMicros}; +use knot_types::{ + AccountDid, AppviewEndpoint, BranchName, CiLogsAddr, Handle, Oid, OwnerDid, PushOption, + PushOptions, RefName, RepoDid, RepoRkey, +}; + +const DID: &str = "did:plc:limpet"; +const OWNER: &str = "did:web:olaren.dev"; +const COMMITTER: &str = "did:plc:nel"; +const PUSH_BUDGET: LanguagesPushBudget = LanguagesPushBudget::new(Duration::from_secs(2)); + +fn git(cwd: &Path, args: &[&str]) -> String { + let output = knot_fixtures::command(cwd) + .args(args) + .output() + .expect("git is available"); + assert!( + output.status.success(), + "git {args:?} failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).unwrap().trim().to_string() +} + +fn commit_file(work: &Path, file: &str, contents: &str, message: &str) { + let path = work.join(file); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, contents).unwrap(); + git(work, &["add", "-A"]); + git(work, &["commit", "-q", "-m", message]); +} + +struct World { + _scan: tempfile::TempDir, + _work: tempfile::TempDir, + repo: Repo, + work: std::path::PathBuf, + bare: std::path::PathBuf, +} + +fn world() -> World { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()).with_default_branch(BranchName::new("main").unwrap()); + let did = RepoDid::new(DID).unwrap(); + layout.create(&did).unwrap(); + let bare = layout.repo_path(&did).unwrap(); + + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path().to_path_buf(); + git(&work, &["init", "-q", "-b", "main"]); + let repo = layout.open(&did).unwrap(); + World { + _scan: scan, + _work: work_dir, + repo, + work, + bare, + } +} + +fn push(world: &World, branch: &str) { + git( + &world.work, + &["push", "-q", world.bare.to_str().unwrap(), branch], + ); +} + +fn oid(world: &World, rev: &str) -> Oid { + Oid::from_hex(&git(&world.work, &["rev-parse", rev])).unwrap() +} + +fn actor() -> Actor { + Actor { + committer: AccountDid::new(COMMITTER).unwrap(), + owner: Some(OwnerDid::new(OWNER).unwrap()), + repo: RepoDid::new(DID).unwrap(), + } +} + +fn refname(name: &str) -> RefName { + RefName::new(name).unwrap() +} + +fn pull() -> PullLink { + PullLink { + appview: AppviewEndpoint::new("https://tangled.test").unwrap(), + owner: OwnerLabel::Handle(Handle::new_owned("nel.pet").unwrap()), + rkey: RepoRkey::new("anemone").unwrap(), + } +} + +fn compile(verbose: bool) -> Ci { + Ci::Compile { + logs: None, + verbose, + } +} + +fn compile_with_logs(addr: &str) -> Ci { + Ci::Compile { + logs: Some(CiLogsAddr::new(addr).unwrap()), + verbose: false, + } +} + +fn bounds() -> knot_events::ReplayBounds { + knot_events::ReplayBounds::new( + knot_events::ReplayEvents::new(32).unwrap(), + knot_events::ReplayBytes::new(16 << 20).unwrap(), + ) +} + +fn log() -> EventLog { + EventLog::new(ManualClock::new(UnixMicros::new(1_000_000_000)), bounds()) +} + +fn reserved(log: &EventLog, applied: &[RefUpdate]) -> Vec<(RefUpdate, Reservation)> { + applied + .iter() + .map(|update| (update.clone(), log.reserve())) + .collect() +} + +fn events(log: &EventLog) -> Vec<(String, serde_json::Value)> { + log.replay(EventCursor::START, bounds()) + .events + .into_iter() + .map(|event| { + let wire = serde_json::to_value(&*event).unwrap(); + (event.nsid.to_string(), wire["event"].clone()) + }) + .collect() +} + +fn run( + world: &World, + log: &EventLog, + applied: &[RefUpdate], + ci: &Ci, + pull: Option<&PullLink>, +) -> Vec { + run_with_options(world, log, applied, ci, &PushOptions::default(), pull) +} + +fn run_with_options( + world: &World, + log: &EventLog, + applied: &[RefUpdate], + ci: &Ci, + push_options: &PushOptions, + pull: Option<&PullLink>, +) -> Vec { + let repo = Repo::open(&world.bare).unwrap(); + post_receive( + &repo, + &actor(), + reserved(log, applied), + ci, + push_options, + pull, + PUSH_BUDGET, + &knot_messages::default_catalog().push, + ) +} + +fn created(branch: &str, head: Oid) -> [RefUpdate; 1] { + [RefUpdate::Create { + name: refname(&format!("refs/heads/{branch}")), + new: head, + }] +} + +fn create(world: &World, branch: &str) -> [RefUpdate; 1] { + commit_file(&world.work, "a.txt", "one\n", "first"); + push(world, branch); + created(branch, oid(world, "HEAD")) +} + +fn create_feature(world: &World) -> Oid { + commit_file(&world.work, "a.txt", "one\n", "first"); + push(world, "main"); + git(&world.work, &["checkout", "-q", "-b", "feature"]); + commit_file(&world.work, "c.txt", "three\n", "feature work"); + push(world, "feature"); + oid(world, "HEAD") +} + +#[test] +fn a_create_emits_a_ref_update_with_commit_counts_and_default_ref() { + let world = world(); + commit_file(&world.work, "a.txt", "one\n", "first"); + commit_file(&world.work, "b.txt", "two\n", "second"); + push(&world, "main"); + let head = oid(&world, "HEAD"); + + let log = log(); + let applied = created("main", head); + run(&world, &log, &applied, &Ci::Skip, None); + + let events = events(&log); + assert_eq!(events.len(), 1); + let (nsid, payload) = &events[0]; + assert_eq!(nsid, "sh.tangled.git.refUpdate"); + assert_eq!(payload["ref"], "refs/heads/main"); + assert_eq!(payload["newSha"], head.to_hex()); + assert_eq!( + payload["oldSha"], + world.repo.object_format().null_oid().to_string() + ); + assert_eq!(payload["committerDid"], COMMITTER); + assert_eq!(payload["meta"]["isDefaultRef"], true); + assert_eq!( + payload["meta"]["commitCount"]["byEmail"][0]["email"], + "nel@oyster.cafe" + ); + assert_eq!(payload["meta"]["commitCount"]["byEmail"][0]["count"], 2); +} + +#[test] +fn an_update_counts_only_the_new_commits() { + let world = world(); + commit_file(&world.work, "a.txt", "one\n", "first"); + push(&world, "main"); + let old = oid(&world, "HEAD"); + commit_file(&world.work, "b.txt", "two\n", "second"); + push(&world, "main"); + let new = oid(&world, "HEAD"); + + let log = log(); + let applied = [RefUpdate::Update { + name: refname("refs/heads/main"), + old, + new, + }]; + run(&world, &log, &applied, &Ci::Skip, None); + + let payload = &events(&log)[0].1; + assert_eq!(payload["meta"]["commitCount"]["byEmail"][0]["count"], 1); +} + +#[test] +fn a_non_default_branch_is_not_flagged_default() { + let world = world(); + commit_file(&world.work, "a.txt", "one\n", "first"); + push(&world, "main"); + git(&world.work, &["checkout", "-q", "-b", "feature"]); + commit_file(&world.work, "c.txt", "three\n", "feature work"); + push(&world, "feature"); + let head = oid(&world, "HEAD"); + + let log = log(); + let applied = created("feature", head); + run(&world, &log, &applied, &Ci::Skip, None); + + let payload = &events(&log)[0].1; + assert_eq!(payload["meta"]["isDefaultRef"], false); + assert_eq!(payload["meta"]["commitCount"]["byEmail"][0]["count"], 1); +} + +#[test] +fn a_large_push_counts_every_commit_without_a_limit() { + let world = world(); + let count = 110; + (0..count).for_each(|n| { + git( + &world.work, + &["commit", "-q", "--allow-empty", "-m", &format!("c{n}")], + ); + }); + push(&world, "main"); + let head = oid(&world, "HEAD"); + + let log = log(); + let applied = created("main", head); + run(&world, &log, &applied, &Ci::Skip, None); + + let payload = &events(&log)[0].1; + assert_eq!( + payload["meta"]["commitCount"]["byEmail"][0]["count"], count, + "every commit is tallied, none dropped" + ); +} + +#[test] +fn a_non_default_branch_omits_the_language_breakdown() { + let world = world(); + commit_file(&world.work, "src/main.rs", "fn main() {}\n", "rust"); + push(&world, "main"); + git(&world.work, &["checkout", "-q", "-b", "feature"]); + commit_file(&world.work, "src/extra.rs", "fn extra() {}\n", "more rust"); + push(&world, "feature"); + let head = oid(&world, "HEAD"); + + let log = log(); + let applied = created("feature", head); + run(&world, &log, &applied, &Ci::Skip, None); + + let payload = &events(&log)[0].1; + assert_eq!(payload["meta"]["isDefaultRef"], false); + assert_eq!( + payload["meta"]["langBreakdown"], + serde_json::Value::Null, + "non-default ref has no language breakdown" + ); + assert_eq!(payload["meta"]["commitCount"]["byEmail"][0]["count"], 1); +} + +#[test] +fn a_delete_emits_a_ref_update_without_meta() { + let world = world(); + commit_file(&world.work, "a.txt", "one\n", "first"); + push(&world, "main"); + let old = oid(&world, "HEAD"); + + let log = log(); + let applied = [RefUpdate::Delete { + name: refname("refs/heads/gone"), + old, + }]; + run(&world, &log, &applied, &Ci::Skip, None); + + let payload = &events(&log)[0].1; + assert_eq!(payload["ref"], "refs/heads/gone"); + assert_eq!( + payload["newSha"], + world.repo.object_format().null_oid().to_string() + ); + assert_eq!(payload["meta"], serde_json::Value::Null); + assert!( + payload.get("changedFiles").is_none(), + "a deletion has no tree to diff: {payload}" + ); + assert!(payload.get("pushOptions").is_none(), "{payload}"); +} + +#[test] +fn language_breakdown_reports_pushed_sources() { + let world = world(); + commit_file( + &world.work, + "src/main.rs", + "fn main() {\n println!(\"hello from nel\");\n}\n", + "rust", + ); + push(&world, "main"); + let head = oid(&world, "HEAD"); + + let log = log(); + let applied = created("main", head); + run(&world, &log, &applied, &Ci::Skip, None); + + let payload = &events(&log)[0].1; + let langs = payload["meta"]["langBreakdown"]["inputs"] + .as_array() + .expect("language breakdown is present"); + assert!( + langs.iter().any(|lang| lang["lang"] == "Rust"), + "expected Rust in {langs:?}" + ); +} + +#[test] +fn a_configured_logs_address_yields_an_ssh_command_for_compiled_workflows() { + let world = world(); + commit_file( + &world.work, + ".tangled/workflows/ci.yml", + "engine: nixery.dev/x\nwhen:\n - event: push\n branch: ['**']\n", + "add ci", + ); + push(&world, "main"); + let head = oid(&world, "HEAD"); + + let log = log(); + let applied = created("main", head); + let messages = run( + &world, + &log, + &applied, + &compile_with_logs("logs.oyster.cafe:3333"), + None, + ); + + assert!( + messages + .iter() + .any(|line| line.contains(&format!("ssh -t -p 3333 logs.oyster.cafe {DID} {head}"))), + "{messages:?}" + ); +} + +#[test] +fn a_push_without_workflows_yields_no_ssh_command_and_verbose_says_so() { + let world = world(); + let applied = create(&world, "main"); + let messages = run( + &world, + &log(), + &applied, + &compile_with_logs("logs.oyster.cafe:3333"), + None, + ); + assert!( + messages.iter().all(|line| !line.contains("ssh -t")), + "{messages:?}" + ); + + let messages = run(&world, &log(), &applied, &compile(true), None); + assert!( + messages + .iter() + .any(|line| line == "no pipelines to compile"), + "{messages:?}" + ); + assert!( + messages + .iter() + .all(|line| line != "pipeline compiled with no diagnostics"), + "{messages:?}" + ); +} + +#[test] +fn verbose_keeps_the_warning_that_explains_why_nothing_compiled() { + let world = world(); + commit_file( + &world.work, + ".tangled/workflows/ci.yml", + "engine: nixery.dev/x\nwhen:\n - event: push\n branch: [release]\n", + "add ci", + ); + push(&world, "main"); + let head = oid(&world, "HEAD"); + + let log = log(); + let messages = run(&world, &log, &created("main", head), &compile(true), None); + + assert!( + messages + .iter() + .any(|line| line.contains("workflow skipped")), + "a workflow that misses the trigger still reports why: {messages:?}" + ); + assert!( + messages + .iter() + .any(|line| line == "no pipelines to compile"), + "{messages:?}" + ); +} + +#[test] +fn the_ref_update_event_reports_changed_files_and_push_options() { + let world = world(); + commit_file(&world.work, "a.txt", "one\n", "first"); + commit_file(&world.work, "src/deep/main.rs", "fn main() {}\n", "nested"); + push(&world, "main"); + let head = oid(&world, "HEAD"); + git(&world.work, &["tag", "-a", "v1.0.0", "-m", "release one"]); + git( + &world.work, + &["push", "-q", "--tags", world.bare.to_str().unwrap()], + ); + let tag_object = oid(&world, "v1.0.0"); + assert_ne!(tag_object, oid(&world, "v1.0.0^{commit}")); + + let branch_log = log(); + let options = PushOptions::new([PushOption::new("verbose-ci").unwrap()]); + run_with_options( + &world, + &branch_log, + &created("main", head), + &compile(false), + &options, + None, + ); + + let payload = &events(&branch_log)[0].1; + assert_eq!( + payload["changedFiles"], + serde_json::json!(["a.txt", "src/deep/main.rs"]), + "a branch creation reports every blob in the tree and no directory of them" + ); + assert_eq!(payload["pushOptions"], serde_json::json!(["verbose-ci"])); + + let tag_log = log(); + let applied = [RefUpdate::Create { + name: refname("refs/tags/v1.0.0"), + new: tag_object, + }]; + run(&world, &tag_log, &applied, &Ci::Skip, None); + assert_eq!( + events(&tag_log)[0].1["changedFiles"], + serde_json::json!(["a.txt", "src/deep/main.rs"]), + "the tag object peels to its commit before the trees are diffed" + ); +} + +fn ci_yaml(paths: &str) -> String { + format!( + "engine: nixery.dev/x\nwhen:\n - event: push\n branch: ['**']\n paths: ['{paths}']\n" + ) +} + +#[test] +fn a_push_wider_than_the_record_prints_the_ssh_command_only_on_a_listed_glob_hit() { + let world = world(); + commit_file( + &world.work, + ".tangled/workflows/ci.yml", + &ci_yaml("never/**"), + "add ci", + ); + (0..20_000).for_each(|index| { + std::fs::write(world.work.join(format!("f{index}.txt")), "x\n").unwrap(); + }); + git(&world.work, &["add", "-A"]); + git(&world.work, &["commit", "-q", "-m", "many"]); + push(&world, "main"); + let unmatched = oid(&world, "HEAD"); + commit_file( + &world.work, + ".tangled/workflows/ci.yml", + &ci_yaml("f1.txt"), + "aim ci", + ); + push(&world, "main"); + let matched = oid(&world, "HEAD"); + + let wide_log = log(); + let messages = run( + &world, + &wide_log, + &created("main", unmatched), + &compile_with_logs("logs.oyster.cafe:3333"), + None, + ); + let listed = events(&wide_log)[0].1["changedFiles"] + .as_array() + .unwrap() + .len(); + assert!( + (1..20_000).contains(&listed), + "the listing stops at the byte budget instead of growing with the push: {listed}" + ); + assert!( + messages.iter().all(|line| !line.contains("ssh -t")), + "spindle reads the same truncated listing, rules the globs out, \ + and skips this run: {messages:?}" + ); + + let messages = run( + &world, + &log(), + &created("main", matched), + &compile_with_logs("logs.oyster.cafe:3333"), + None, + ); + assert!( + messages.iter().any(|line| line.contains("ssh -t -p 3333")), + "spindle sees the same listed path and runs this workflow: {messages:?}" + ); +} + +#[test] +fn a_compiled_workflow_emits_no_pipeline_event() { + let world = world(); + commit_file( + &world.work, + ".tangled/workflows/ci.yml", + "engine: nixery.dev/x\nwhen:\n - event: push\n branch: ['**']\n", + "add ci", + ); + push(&world, "main"); + let head = oid(&world, "HEAD"); + + let log = log(); + let applied = created("main", head); + run(&world, &log, &applied, &compile(false), None); + + let events = events(&log); + assert!(events.iter().all(|(nsid, _)| nsid != "sh.tangled.pipeline")); + assert_eq!(events.len(), 1); + assert_eq!(events[0].0, "sh.tangled.git.refUpdate"); +} + +#[test] +fn a_new_non_default_branch_yields_a_pull_request_link() { + let world = world(); + let log = log(); + let head = create_feature(&world); + + let applied = created("feature", head); + let messages = run(&world, &log, &applied, &Ci::Skip, Some(&pull())); + + let link = messages + .iter() + .find(|line| line.contains("/pulls/new")) + .expect("pull-request link is offered for new non-default branch"); + assert!( + link.contains("https://tangled.test/nel.pet/anemone/pulls/new"), + "{link}" + ); + assert!(link.contains("sourceBranch=feature"), "{link}"); + assert!(link.contains("targetBranch=main"), "{link}"); +} + +#[test] +fn no_pull_request_link_for_default_existing_forked_or_rootless_branches() { + type Case = (&'static str, fn(&World) -> [RefUpdate; 1]); + let cases: &[Case] = &[ + ("push to the default branch", |w| create(w, "main")), + ("update to an existing branch", |w| { + commit_file(&w.work, "a.txt", "one\n", "first"); + push(w, "main"); + git(&w.work, &["checkout", "-q", "-b", "feature"]); + commit_file(&w.work, "c.txt", "three\n", "feature"); + push(w, "feature"); + let old = oid(w, "HEAD"); + commit_file(&w.work, "d.txt", "four\n", "more feature"); + push(w, "feature"); + [RefUpdate::Update { + name: refname("refs/heads/feature"), + old, + new: oid(w, "HEAD"), + }] + }), + ("new branch on a fork with an origin remote", |w| { + let head = create_feature(w); + w.repo + .set_origin_url("https://oyster.cafe/did:plc:squid/anemone") + .unwrap(); + created("feature", head) + }), + ("new branch while the default branch is absent", |w| { + commit_file(&w.work, "a.txt", "one\n", "first"); + git(&w.work, &["checkout", "-q", "-b", "feature"]); + commit_file(&w.work, "c.txt", "three\n", "feature work"); + push(w, "feature"); + created("feature", oid(w, "HEAD")) + }), + ]; + cases.iter().for_each(|(label, build)| { + let world = world(); + let log = log(); + let applied = build(&world); + let messages = run(&world, &log, &applied, &Ci::Skip, Some(&pull())); + assert!( + messages.iter().all(|line| !line.contains("/pulls/new")), + "{label} must offer no pull-request link: {messages:?}" + ); + }); +} + +#[test] +fn verbose_ci_reports_a_clean_pipeline_and_quiet_ci_stays_silent() { + let world = world(); + let log = log(); + commit_file( + &world.work, + ".tangled/workflows/ci.yml", + "engine: nixery.dev/x\nwhen:\n - event: push\n branch: ['**']\n", + "add ci", + ); + push(&world, "main"); + let head = oid(&world, "HEAD"); + let applied = created("main", head); + + let verbose = run(&world, &log, &applied, &compile(true), None); + assert!( + verbose.iter().any(|line| line.contains("no diagnostics")), + "verbose ci announces clean compile: {verbose:?}" + ); + + let quiet = run(&world, &log, &applied, &compile(false), None); + assert!( + quiet.iter().all(|line| !line.contains("no diagnostics")), + "quiet push says nothing about clean compile: {quiet:?}" + ); +} + +#[test] +fn a_pipeline_compile_error_reaches_the_pusher_even_when_quiet() { + let world = world(); + let log = log(); + commit_file( + &world.work, + ".tangled/workflows/broken.yml", + "engine: : not valid : yaml :\n - [\n", + "add broken ci", + ); + push(&world, "main"); + let head = oid(&world, "HEAD"); + let applied = created("main", head); + + let messages = run(&world, &log, &applied, &compile(false), None); + assert!( + messages.iter().any(|line| line.starts_with("error:")), + "malformed workflow surfaces compile error to pusher: {messages:?}" + ); +} diff --git a/knot2/crates/knot-receive/src/lib.rs b/knot2/crates/knot-receive/src/lib.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-receive/src/lib.rs @@ -0,0 +1,329 @@ +//! # How we go about receiving a push! +//! +//! `land` will run the pack ingest and the pull-link lookup at the same time, +//! then will merge both into a single post-receive pass. +//! Each ref-update claims its event cursor during ingest +//! and fills the payload in afterward, +//! such that events replay in ref-order even though post-receive was what computed them. +//! +//! Or should I say: +//! +//! ```text +//! request task blocking pool +//! --------------------------------- ------------------------------ +//! read the push preamble +//! | +//! +---- spawn -------------------> open repo +//! | | +//! creates a branch? no -> no link receive pack +//! | yes | +//! look up pull link seal: 1 cursor per ref update +//! | repo-DID -> owner, rkey, handle | +//! v v +//! join <---------------------------- updates paired w/ reservations +//! | +//! any refs applied? no -> no messages +//! | yes +//! owner & rkey for this repo-DID +//! | +//! ci flags from push options +//! | +//! +---- spawn -------------------> ack lines +//! | | +//! | post_receive fulfills each +//! | | reservation +//! v v +//! messages <------------------------ ..and returns its messages +//! | +//! note this push for maintenance +//! | +//! frame report for the client +//! ``` +//! +//! *Figure 1: the tasks of a push.* +//! +//! `EventLog::replay` finishes at the oldest pending cursor, +//! meaning that a single reservation will hide every later-event until it resolves. +//! Every exit from `land` therefore has to either fulfill each one or drop it, +//! and `Reservation`'s `Drop` clears +//! the pending cursor such that replay advances again. +//! +//! ```text +//! one Reservation +//! --------------- +//! reserve -> pending, replay finishes here +//! | +//! +-- post_receive fulfills it -----> event is visible +//! +-- receive errors after seal ----> dropped, logged +//! +-- receive task panics ----------> dropped, silent +//! +-- post-receive task panics -----> dropped, logged +//! +-- caller drops the land future -> dropped, silent +//! ``` +//! +//! *Figure 2: every way in which a single reservation can end.* +//! +//! Note that last path in Figure 2 leaves the receive task running +//! with nowhere to return its result, +//! so the reservations inside it drop alongside the discarded value. + +use std::cell::RefCell; +use std::sync::Arc; + +use knot_atproto::Atproto; +use knot_cob::CobHome; +use knot_events::{EventLog, Reservation}; +use knot_git::{Layout, RefUpdate, Repo}; +use knot_index::{Index, Resolved}; +use knot_maintenance::{MaintenanceHandle, PushBytes}; +use knot_messages::{Catalog, PushAckKey, count_refs}; +use knot_pack::{ + PackError, PackLimits, PushGuard, ReceiveOutcome, ReceivedPack, frame_report, + receive_pack_guarded_streamed, receive_preflight, +}; +use knot_postreceive::{Actor, Ci, LanguagesPushBudget, OwnerLabel, PullLink, post_receive}; +use knot_resource::ResolveSlots; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{ + AccountDid, ActorId, AppviewEndpoint, CiLogsAddr, Handle, KnotHostname, OwnerDid, PushOptions, + RepoDid, +}; + +type Applied = Vec<(RefUpdate, Reservation)>; + +pub struct Push<'a, H: HttpTransport, C: Clock> { + pub layout: &'a Layout, + pub repo_did: &'a RepoDid, + pub received: ReceivedPack, + pub limits: PackLimits, + pub knot_actor: ActorId, + pub committer: AccountDid, + pub events: Arc>, + pub index: &'a Index, + pub atproto: &'a Atproto, + pub resolve_slots: &'a ResolveSlots, + pub appview: &'a AppviewEndpoint, + pub maintenance: &'a MaintenanceHandle, + pub hostname: &'a KnotHostname, + pub languages_push_budget: LanguagesPushBudget, + pub ci_logs: Option, + pub catalog: Arc, +} + +pub async fn land(push: Push<'_, H, C>) -> Result, PackError> { + let Push { + layout, + repo_did, + received, + limits, + knot_actor, + committer, + events, + index, + atproto, + resolve_slots, + appview, + maintenance, + hostname, + languages_push_budget, + catalog, + ci_logs, + } = push; + + let preflight = receive_preflight(received.preamble()); + let body_len = received.len(); + let home = CobHome::from(repo_did); + + let receive = { + let layout = layout.clone(); + let did = repo_did.clone(); + let events = Arc::clone(&events); + let catalog = Arc::clone(&catalog); + tokio::task::spawn_blocking( + move || -> Result<(Repo, ReceiveOutcome, Applied), PackError> { + let repo = layout.open(&did)?; + let guard = PushGuard { + cob_authority: knot_actor, + home, + messages: Arc::clone(&catalog), + }; + let stash: RefCell = RefCell::new(Vec::new()); + let seal = |updates: &[RefUpdate]| { + updates.iter().for_each(|update| { + stash.borrow_mut().push((update.clone(), events.reserve())) + }); + }; + let outcome = match receive_pack_guarded_streamed( + &repo, + &received, + &limits, + &guard, + &seal, + &catalog.reject, + ) { + Ok(outcome) => outcome, + Err(error) => { + match stash.borrow().len() { + 0 => {} + sealed => tracing::error!( + repo = did.as_str(), + sealed, + "receive failure dropped the events for the sealed ref updates" + ), + } + return Err(error); + } + }; + Ok((repo, outcome, stash.into_inner())) + }, + ) + }; + let pull = async { + match preflight.creates_branch { + true => resolve_pull_link(index, atproto, resolve_slots, appview, repo_did).await, + false => None, + } + }; + let (received, pull) = tokio::join!(receive, pull); + let (repo, outcome, applied) = match received { + Ok(Ok(triple)) => triple, + Ok(Err(error)) => return Err(error), + Err(_) => return Err(PackError::Pack("receive-pack task panicked".to_string())), + }; + + let messages = match applied.is_empty() { + true => Vec::new(), + false => { + let owner = registry_owner(index, repo_did); + let ci = ci_from_push_options(&outcome.push_options, ci_logs.clone()); + let push_options = outcome.push_options.clone(); + let did = repo_did.clone(); + let catalog = Arc::clone(&catalog); + let knot = hostname.clone(); + tokio::task::spawn_blocking(move || { + let actor = Actor { + committer, + owner, + repo: did, + }; + let ack = catalog.push.ack.lines(|key| match key { + PushAckKey::Knot => knot.as_str().to_string(), + PushAckKey::Refs => count_refs(applied.len()), + }); + ack.into_iter() + .chain(post_receive( + &repo, + &actor, + applied, + &ci, + &push_options, + pull.as_ref(), + languages_push_budget, + &catalog.push, + )) + .collect() + }) + .await + .unwrap_or_else(|error| { + tracing::error!(repo = repo_did.as_str(), %error, "post-receive task panicked and dropped the ref-update events for this push"); + Vec::new() + }) + } + }; + maintenance.note_push(repo_did, PushBytes::new(body_len as u64)); + Ok(frame_report(&outcome.report, &messages, outcome.side_band)) +} + +fn registry_owner(index: &Index, repo: &RepoDid) -> Option { + let ready = |owner: Resolved>| match owner { + Resolved::Ready(owner) => owner, + Resolved::Warming => None, + }; + match index.owner_of(repo) { + resolved @ Resolved::Ready(_) => ready(resolved), + Resolved::Warming => { + if let Err(error) = index.refresh_registry() { + tracing::warn!( + repo = repo.as_str(), + %error, + "registry refresh during post-receive failed, ref-update event omits the owner" + ); + } + ready(index.owner_of(repo)) + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum PushDirective { + SkipCi, + VerboseCi, +} + +impl PushDirective { + fn parse(option: &str) -> Option { + match option { + "skip-ci" | "ci-skip" => Some(Self::SkipCi), + "verbose-ci" | "ci-verbose" => Some(Self::VerboseCi), + _ => None, + } + } +} + +pub fn ci_from_push_options(options: &PushOptions, logs: Option) -> Ci { + let directives: Vec = options + .as_slice() + .iter() + .filter_map(|option| PushDirective::parse(option.as_str())) + .collect(); + match directives.contains(&PushDirective::SkipCi) { + true => Ci::Skip, + false => Ci::Compile { + logs, + verbose: directives.contains(&PushDirective::VerboseCi), + }, + } +} + +async fn resolve_pull_link( + index: &Index, + atproto: &Atproto, + resolve_slots: &ResolveSlots, + appview: &AppviewEndpoint, + repo_did: &RepoDid, +) -> Option { + let owner = match index.owner_of(repo_did) { + Resolved::Ready(Some(owner)) => owner, + _ => return None, + }; + let rkey = match index.rkey_of(repo_did) { + Resolved::Ready(Some(rkey)) => rkey, + _ => return None, + }; + Some(PullLink { + appview: appview.clone(), + owner: resolve_owner_label(atproto, resolve_slots, &owner).await, + rkey, + }) +} + +async fn resolve_owner_label( + atproto: &Atproto, + resolve_slots: &ResolveSlots, + owner: &OwnerDid, +) -> OwnerLabel { + let did = AccountDid::from(owner.clone()); + match resolve_handle(atproto, resolve_slots, &did).await { + Some(handle) => OwnerLabel::Handle(handle), + None => OwnerLabel::Did(owner.clone()), + } +} + +pub async fn resolve_handle( + atproto: &Atproto, + resolve_slots: &ResolveSlots, + did: &AccountDid, +) -> Option { + let _permit = resolve_slots.try_acquire()?; + let identity = atproto.resolve_identity(did).await.ok()?; + identity.primary_handle().cloned() +} diff --git a/knot2/crates/knot-resource/src/admission.rs b/knot2/crates/knot-resource/src/admission.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-resource/src/admission.rs @@ -0,0 +1,497 @@ +use std::collections::HashMap; +use std::net::IpAddr; +use std::sync::{Arc, Mutex}; + +use knot_types::UnixMicros; + +const MAX_TRACKED_PEERS: usize = 100_000; + +const SWEEP_INTERVAL_MICROS: u64 = 1_000_000; + +knot_types::scalar_newtype! { + pub struct Burst(u32); + pub struct RefillMicros(u64); + pub struct PerPeerInflight(usize); + pub struct GlobalInflight(usize); +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RateLimit { + pub burst: Burst, + pub refill: RefillMicros, +} + +impl RateLimit { + const fn interval(self) -> u64 { + match self.refill.get() { + 0 => 1, + micros => micros, + } + } +} + +#[derive(Debug, Clone, Copy)] +pub struct LimitConfig { + pub rate: Option, + pub per_peer_inflight: Option, + pub global_inflight: Option, +} + +impl Default for LimitConfig { + fn default() -> Self { + Self { + rate: Some(RateLimit { + burst: Burst::new(20), + refill: RefillMicros::new(100_000), + }), + per_peer_inflight: Some(PerPeerInflight::new(8)), + global_inflight: Some(GlobalInflight::new(64)), + } + } +} + +impl LimitConfig { + pub const fn per_peer_only(per_peer_inflight: PerPeerInflight) -> Self { + Self { + rate: None, + per_peer_inflight: Some(per_peer_inflight), + global_inflight: None, + } + } + + pub const fn unmetered() -> Self { + Self { + rate: None, + per_peer_inflight: None, + global_inflight: None, + } + } +} + +struct Bucket { + tokens: u32, + last_refill: UnixMicros, +} + +impl Bucket { + fn new(rate: RateLimit, now: UnixMicros) -> Self { + Self { + tokens: rate.burst.get(), + last_refill: now, + } + } + + fn tokens_at(&self, rate: RateLimit, now: UnixMicros) -> u32 { + let elapsed = now.get().saturating_sub(self.last_refill.get()); + let gained = (elapsed / rate.interval()).min(u64::from(rate.burst.get())) as u32; + self.tokens.saturating_add(gained).min(rate.burst.get()) + } + + fn replenish(&mut self, rate: RateLimit, now: UnixMicros) -> bool { + if now.get().saturating_sub(self.last_refill.get()) >= rate.interval() { + self.tokens = self.tokens_at(rate, now); + self.last_refill = now; + } + self.tokens > 0 + } + + fn full(&self, rate: RateLimit, now: UnixMicros) -> bool { + self.tokens_at(rate, now) >= rate.burst.get() + } +} + +struct PeerState { + bucket: Option, + inflight: usize, +} + +impl PeerState { + fn forgettable(&self) -> bool { + self.inflight == 0 && self.bucket.is_none() + } + + fn worth_tracking(&self, rate: Option, now: UnixMicros) -> bool { + match (self.inflight, &self.bucket, rate) { + (0, Some(bucket), Some(rate)) => !bucket.full(rate, now), + (0, _, _) => false, + _ => true, + } + } +} + +struct Inner { + peers: HashMap, PeerState>, + global_inflight: usize, + last_sweep: UnixMicros, +} + +pub struct PreAuthLimiter { + config: LimitConfig, + inner: Mutex, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Refusal { + RateLimited, + Saturated, +} + +impl Default for PreAuthLimiter { + fn default() -> Self { + Self::with_config(LimitConfig::default()) + } +} + +impl PreAuthLimiter { + pub fn with_config(config: LimitConfig) -> Self { + Self { + config, + inner: Mutex::new(Inner { + peers: HashMap::new(), + global_inflight: 0, + last_sweep: UnixMicros::new(0), + }), + } + } + + fn lock(&self) -> std::sync::MutexGuard<'_, Inner> { + self.inner + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + pub fn admit( + self: &Arc, + peer: Option, + now: UnixMicros, + ) -> Result { + let mut inner = self.lock(); + let rate = self.config.rate; + let over_global = self + .config + .global_inflight + .is_some_and(|limit| inner.global_inflight >= limit.get()); + + // Only a rate budget outlives the work it admitted, so only a rate + // budget can pile up entries for peers that have gone away. Without one + // the last operation removes the entry and the table is already bounded + // by live concurrency, so capping it would shed callers over a table + // that can't grow. + if rate.is_some() + && !inner.peers.contains_key(&peer) + && inner.peers.len() >= MAX_TRACKED_PEERS + { + let sweep_due = + now.get().saturating_sub(inner.last_sweep.get()) >= SWEEP_INTERVAL_MICROS; + if sweep_due { + inner.last_sweep = now; + inner + .peers + .retain(|_, state| state.worth_tracking(rate, now)); + } + if inner.peers.len() >= MAX_TRACKED_PEERS { + return Err(Refusal::Saturated); + } + } + + let per_peer = self.config.per_peer_inflight; + let decision = { + let state = inner.peers.entry(peer).or_insert_with(|| PeerState { + bucket: rate.map(|rate| Bucket::new(rate, now)), + inflight: 0, + }); + let ready = match (rate, state.bucket.as_mut()) { + (Some(rate), Some(bucket)) => bucket.replenish(rate, now), + _ => true, + }; + let over_peer = per_peer.is_some_and(|limit| state.inflight >= limit.get()); + match (ready, over_peer || over_global) { + (false, _) => Err(Refusal::RateLimited), + (_, true) => Err(Refusal::Saturated), + (true, false) => { + if let Some(bucket) = state.bucket.as_mut() { + bucket.tokens -= 1; + } + state.inflight += 1; + Ok(()) + } + } + }; + match decision { + Err(refusal) => { + if inner.peers.get(&peer).is_some_and(PeerState::forgettable) { + inner.peers.remove(&peer); + } + Err(refusal) + } + Ok(()) => { + inner.global_inflight += 1; + Ok(AdmitGuard { + limiter: Arc::clone(self), + peer, + }) + } + } + } + + fn leave(&self, peer: Option) { + let mut inner = self.lock(); + inner.global_inflight = inner.global_inflight.saturating_sub(1); + let Some(state) = inner.peers.get_mut(&peer) else { + return; + }; + state.inflight = state.inflight.saturating_sub(1); + if state.forgettable() { + inner.peers.remove(&peer); + } + } + + fn repay(&self, peer: Option) { + let Some(rate) = self.config.rate else { + return; + }; + let mut inner = self.lock(); + if let Some(bucket) = inner + .peers + .get_mut(&peer) + .and_then(|state| state.bucket.as_mut()) + { + bucket.tokens = bucket.tokens.saturating_add(1).min(rate.burst.get()); + } + } +} + +pub struct AdmitGuard { + limiter: Arc, + peer: Option, +} + +impl AdmitGuard { + pub fn refund(self) { + self.limiter.repay(self.peer); + } +} + +impl Drop for AdmitGuard { + fn drop(&mut self) { + self.limiter.leave(self.peer); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::Ipv4Addr; + + fn limiter(config: LimitConfig) -> Arc { + Arc::new(PreAuthLimiter::with_config(config)) + } + + fn rate(burst: u32, refill_micros: u64) -> Option { + Some(RateLimit { + burst: Burst::new(burst), + refill: RefillMicros::new(refill_micros), + }) + } + + fn peer(last: u8) -> Option { + Some(IpAddr::V4(Ipv4Addr::new(127, 0, 0, last))) + } + + fn rotating(index: u64) -> Option { + let octets = (index as u32).to_be_bytes(); + Some(IpAddr::V4(Ipv4Addr::new( + 1, octets[1], octets[2], octets[3], + ))) + } + + fn at(micros: u64) -> UnixMicros { + UnixMicros::new(micros) + } + + fn tracked(limiter: &Arc) -> usize { + limiter.lock().peers.len() + } + + #[test] + fn a_rate_budget_refuses_a_flood_refills_over_time_and_stays_per_peer() { + let limiter = limiter(LimitConfig { + rate: rate(2, 1_000), + per_peer_inflight: Some(PerPeerInflight::new(100)), + global_inflight: Some(GlobalInflight::new(100)), + }); + assert!(limiter.admit(peer(1), at(0)).is_ok()); + assert!(limiter.admit(peer(1), at(0)).is_ok()); + assert_eq!( + limiter.admit(peer(1), at(0)).err(), + Some(Refusal::RateLimited), + "the limiter refuses a third request inside the same instant even when nothing is in flight" + ); + assert!( + limiter.admit(peer(2), at(0)).is_ok(), + "a second peer keeps its own rate budget" + ); + assert_eq!( + limiter.admit(peer(1), at(600)).err(), + Some(Refusal::RateLimited) + ); + assert!( + limiter.admit(peer(1), at(1_000)).is_ok(), + "a refusal partway through the interval mustn't reset the clock the refill measures from" + ); + } + + #[test] + fn inflight_shedding_frees_on_drop_and_spends_no_rate_budget() { + let limiter = limiter(LimitConfig { + rate: rate(4, 1_000_000), + per_peer_inflight: Some(PerPeerInflight::new(1)), + global_inflight: Some(GlobalInflight::new(2)), + }); + let held = limiter + .admit(peer(1), at(0)) + .expect("the limiter admits the first request"); + let other = limiter + .admit(peer(2), at(0)) + .expect("a second peer fills the global budget"); + assert_eq!( + limiter.admit(peer(1), at(0)).err(), + Some(Refusal::Saturated), + "the limiter sheds a second concurrent request from one peer" + ); + assert_eq!( + limiter.admit(peer(3), at(0)).err(), + Some(Refusal::Saturated), + "the limiter sheds a third peer once the global in-flight limit is reached" + ); + drop(other); + drop( + limiter + .admit(peer(3), at(0)) + .expect("freeing a global slot admits the peer that was shed"), + ); + drop(held); + (0..50).for_each(|_| { + limiter + .admit(peer(1), at(0)) + .expect("a refunded admission must leave the full budget available") + .refund(); + }); + (0..3).for_each(|_| { + limiter + .admit(peer(1), at(0)) + .expect("a shed request mustn't spend the rate budget it never used"); + }); + assert_eq!( + limiter.admit(peer(1), at(0)).err(), + Some(Refusal::RateLimited), + "a dropped guard without a refund keeps its token spent" + ); + } + + #[test] + fn a_budget_without_a_rate_never_rate_limits_and_keeps_no_idle_state() { + let overflowing = MAX_TRACKED_PEERS as u64 + 1_000; + + let per_peer = limiter(LimitConfig::per_peer_only(PerPeerInflight::new(2))); + (0..1_000).for_each(|_| { + per_peer + .admit(peer(1), at(0)) + .expect("a budget with no rate has nothing for a sequential flood to exhaust"); + }); + let concurrent: Vec = (0..2) + .map(|_| { + per_peer + .admit(peer(1), at(0)) + .expect("the limiter admits both concurrent operations from one peer") + }) + .collect(); + assert_eq!( + per_peer.admit(peer(1), at(0)).err(), + Some(Refusal::Saturated), + "only the per-peer count refuses a request in this budget" + ); + drop(concurrent); + let held: Vec = (0..overflowing) + .map(|index| { + per_peer + .admit(rotating(index), at(0)) + .expect("a budget that keeps no idle state has no table to overflow") + }) + .collect(); + assert_eq!(tracked(&per_peer), held.len()); + drop(held); + assert_eq!( + tracked(&per_peer), + 0, + "with no tokens to remember, an idle peer leaves no entry, \ + so address rotation mustn't fill the table and start shedding newcomers" + ); + + let global = limiter(LimitConfig { + rate: None, + per_peer_inflight: None, + global_inflight: Some(GlobalInflight::new(1)), + }); + let _saturating = global + .admit(peer(1), at(0)) + .expect("the limiter admits the first peer"); + (0..overflowing).for_each(|index| { + assert_eq!( + global.admit(rotating(index), at(0)).err(), + Some(Refusal::Saturated) + ); + }); + assert_eq!( + tracked(&global), + 1, + "a refusal returns no guard, so an entry it left behind would never be freed, \ + and the sweep that bounds the table only reclaims idle rate state" + ); + + let unmetered = limiter(LimitConfig::unmetered()); + let guards: Vec = (0..512) + .map(|_| { + unmetered + .admit(peer(1), at(0)) + .expect("an unmetered budget admits every request from every peer") + }) + .collect(); + drop(guards); + assert_eq!(tracked(&unmetered), 0); + } + + #[test] + fn a_rate_budget_bounds_its_peer_map_and_sweeps_at_most_once_per_interval() { + let limiter = limiter(LimitConfig { + rate: rate(1, 1_000), + per_peer_inflight: Some(PerPeerInflight::new(8)), + global_inflight: Some(GlobalInflight::new(64)), + }); + (0..MAX_TRACKED_PEERS as u64).for_each(|index| { + let _ = limiter.admit(rotating(index), at(0)); + }); + assert_eq!( + limiter + .admit(peer(201), at(SWEEP_INTERVAL_MICROS - 1)) + .err(), + Some(Refusal::Saturated), + "inside the sweep interval a full map sheds unseen peers without rescanning" + ); + assert!( + limiter.admit(peer(202), at(SWEEP_INTERVAL_MICROS)).is_ok(), + "once the interval elapses the sweep evicts replenished entries and admits the newcomer" + ); + (0..(MAX_TRACKED_PEERS as u64 + 50_000)).for_each(|index| { + let _ = limiter.admit( + rotating(MAX_TRACKED_PEERS as u64 + index), + at(SWEEP_INTERVAL_MICROS + index), + ); + }); + let tracked = tracked(&limiter); + assert!( + tracked <= MAX_TRACKED_PEERS, + "a flood of distinct source addresses mustn't grow the peer map past its limit, saw {tracked}" + ); + } +} diff --git a/knot2/crates/knot-resource/src/cpu.rs b/knot2/crates/knot-resource/src/cpu.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-resource/src/cpu.rs @@ -0,0 +1,258 @@ +use std::sync::OnceLock; +use std::sync::atomic::{AtomicUsize, Ordering}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct ThreadCount(usize); + +impl ThreadCount { + pub const fn new(count: usize) -> Self { + Self(if count == 0 { 1 } else { count }) + } + + pub const fn get(self) -> usize { + self.0 + } +} + +knot_types::scalar_newtype! { + struct WorkUnits(usize); +} + +struct Budget { + ceiling: usize, + available: AtomicUsize, +} + +static BUDGET: OnceLock = OnceLock::new(); + +fn detected_threads() -> ThreadCount { + ThreadCount::new( + std::thread::available_parallelism() + .map(std::num::NonZeroUsize::get) + .unwrap_or(1), + ) +} + +pub(crate) fn install(ceiling: ThreadCount) -> ThreadCount { + let budget = BUDGET.get_or_init(|| Budget { + ceiling: ceiling.get(), + available: AtomicUsize::new(ceiling.get().saturating_sub(1)), + }); + ThreadCount::new(budget.ceiling) +} + +fn budget() -> &'static Budget { + BUDGET.get_or_init(|| { + let ceiling = detected_threads().get(); + Budget { + ceiling, + available: AtomicUsize::new(ceiling.saturating_sub(1)), + } + }) +} + +pub fn threads() -> ThreadCount { + ThreadCount::new(budget().ceiling) +} + +pub fn gix_thread_limit() -> Option { + let budget = budget(); + (budget.ceiling < detected_threads().get()).then_some(ThreadCount::new(budget.ceiling)) +} + +pub(crate) fn ceiling() -> usize { + budget().ceiling +} + +static SATURATE: AtomicUsize = AtomicUsize::new(0); + +pub struct Saturate(()); + +impl Drop for Saturate { + fn drop(&mut self) { + SATURATE.fetch_sub(1, Ordering::Relaxed); + } +} + +// the eat my machine button +pub fn saturate() -> Saturate { + SATURATE.fetch_add(1, Ordering::Relaxed); + Saturate(()) +} + +struct Lease { + extra: usize, + saturated: bool, +} + +impl Lease { + fn none() -> Self { + Self { + extra: 0, + saturated: false, + } + } + + fn lanes(&self) -> usize { + self.extra + 1 + } +} + +impl Drop for Lease { + fn drop(&mut self) { + if !self.saturated && self.extra > 0 { + budget().available.fetch_add(self.extra, Ordering::AcqRel); + } + } +} + +fn lease(units: WorkUnits) -> Lease { + let budget = budget(); + let want = units + .get() + .saturating_sub(1) + .min(budget.ceiling.saturating_sub(1)); + if want == 0 { + return Lease::none(); + } + if SATURATE.load(Ordering::Relaxed) > 0 { + return Lease { + extra: want, + saturated: true, + }; + } + let mut available = budget.available.load(Ordering::Relaxed); + loop { + let grant = want.min(available); + if grant == 0 { + return Lease::none(); + } + match budget.available.compare_exchange_weak( + available, + available - grant, + Ordering::AcqRel, + Ordering::Relaxed, + ) { + Ok(_) => { + return Lease { + extra: grant, + saturated: false, + }; + } + Err(observed) => available = observed, + } + } +} + +pub fn map_spans(len: usize, f: F) -> Result, E> +where + R: Send, + E: Send, + F: Fn(usize, usize) -> Result, E> + Sync, +{ + if len == 0 { + return Ok(Vec::new()); + } + let f = &f; + let lease = lease(WorkUnits::new(len)); + let lanes = lease.lanes().min(len); + if lanes <= 1 { + return f(0, len); + } + let chunk = len.div_ceil(lanes); + let spans: Vec<(usize, usize)> = (0..lanes) + .map(|lane| (lane * chunk, ((lane + 1) * chunk).min(len))) + .filter(|(start, end)| start < end) + .collect(); + let (head, tail) = spans + .split_first() + .expect("a positive lane count yields at least one span"); + let ordered: Vec, E>> = std::thread::scope(|scope| { + let handles: Vec<_> = tail + .iter() + .map(|&(start, end)| scope.spawn(move || f(start, end))) + .collect(); + let head = f(head.0, head.1); + std::iter::once(head) + .chain( + handles + .into_iter() + .map(|handle| handle.join().expect("resource worker panicked")), + ) + .collect() + }); + ordered.into_iter().try_fold(Vec::new(), |mut acc, part| { + acc.extend(part?); + Ok(acc) + }) +} + +pub fn map_chunks(items: &[T], f: F) -> Result, E> +where + T: Sync, + R: Send, + E: Send, + F: Fn(&[T]) -> Result, E> + Sync, +{ + map_spans(items.len(), |start, end| f(&items[start..end])) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_positive_span_count_covers_every_index_in_order() { + let doubled = map_spans::(1000, |start, end| { + Ok((start..end).map(|value| value * 2).collect()) + }) + .unwrap(); + assert_eq!(doubled.len(), 1000); + assert!( + doubled + .iter() + .enumerate() + .all(|(index, value)| *value == index * 2) + ); + } + + #[test] + fn an_empty_span_produces_nothing() { + assert_eq!( + map_spans::(0, |_, _| Ok(vec![1])).unwrap(), + Vec::::new() + ); + } + + #[test] + fn a_worker_error_propagates() { + let outcome = + map_spans::( + 500, + |start, _| { + if start == 0 { Err("boom") } else { Ok(vec![]) } + }, + ); + assert_eq!(outcome, Err("boom")); + } + + #[test] + fn chunks_preserve_element_order() { + let items: Vec = (0..777).collect(); + let echoed = map_chunks::(&items, |batch| Ok(batch.to_vec())).unwrap(); + assert_eq!(echoed, items); + } + + #[test] + fn a_lease_never_reserves_more_than_the_ceiling() { + let lease = lease(WorkUnits::new(usize::MAX)); + assert!(lease.lanes() <= threads().get()); + } + + #[test] + fn a_saturated_lease_still_respects_the_ceiling() { + let _boost = saturate(); + let lease = lease(WorkUnits::new(usize::MAX)); + assert!(lease.lanes() <= threads().get()); + } +} diff --git a/knot2/crates/knot-resource/src/disk.rs b/knot2/crates/knot-resource/src/disk.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-resource/src/disk.rs @@ -0,0 +1,146 @@ +use std::io; +use std::path::Path; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +// 3 kinds of u64 that denote "bytes" in their own way +// & look identical at a callsite. +// `reserve(path, floor)` *used to* compile perfectly happily. +knot_types::scalar_newtype! { + pub struct DiskFloorBytes(u64); + pub struct ReserveBytes(u64); + pub struct FreeBytes(u64); +} + +pub fn free_bytes(path: &Path) -> io::Result { + rustix::fs::statvfs(path) + .map(|stat| FreeBytes::new(stat.f_bavail.saturating_mul(stat.f_frsize))) + .map_err(io::Error::from) +} + +#[derive(Debug)] +pub enum ReserveError { + BelowFloor { + free: FreeBytes, + floor: DiskFloorBytes, + }, + Probe(io::Error), +} + +struct Ledger { + floor: DiskFloorBytes, + reserved: AtomicU64, +} + +#[derive(Clone)] +pub struct DiskGovernor(Arc); + +impl DiskGovernor { + pub fn new(floor: DiskFloorBytes) -> Self { + Self(Arc::new(Ledger { + floor, + reserved: AtomicU64::new(0), + })) + } + + pub fn reserved_bytes(&self) -> u64 { + self.0.reserved.load(Ordering::SeqCst) + } + + pub fn reserve( + &self, + path: &Path, + bytes: ReserveBytes, + ) -> Result { + let amount = bytes.get(); + let projected = self.0.reserved.fetch_add(amount, Ordering::SeqCst) + amount; + let free = match free_bytes(path) { + Ok(free) => free, + Err(source) => { + self.0.reserved.fetch_sub(amount, Ordering::SeqCst); + return Err(ReserveError::Probe(source)); + } + }; + if free.get() < self.0.floor.get().saturating_add(projected) { + self.0.reserved.fetch_sub(amount, Ordering::SeqCst); + return Err(ReserveError::BelowFloor { + free, + floor: self.0.floor, + }); + } + Ok(DiskReservation { + ledger: Arc::clone(&self.0), + bytes: amount, + }) + } +} + +pub struct DiskReservation { + ledger: Arc, + bytes: u64, +} + +impl Drop for DiskReservation { + fn drop(&mut self) { + self.ledger.reserved.fetch_sub(self.bytes, Ordering::SeqCst); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_real_filesystem_reports_headroom() { + let dir = std::env::temp_dir(); + assert!(free_bytes(&dir).unwrap().get() > 0); + } + + #[test] + fn a_missing_path_reports_the_fault() { + assert!(free_bytes(Path::new("/definitely/not/a/mounted/path")).is_err()); + } + + #[test] + fn a_reservation_holds_bytes_until_it_drops() { + let dir = std::env::temp_dir(); + let governor = DiskGovernor::new(DiskFloorBytes::new(0)); + assert_eq!(governor.reserved_bytes(), 0); + { + let _held = governor.reserve(&dir, ReserveBytes::new(4_096)).unwrap(); + assert_eq!(governor.reserved_bytes(), 4_096); + let _also = governor.reserve(&dir, ReserveBytes::new(1_024)).unwrap(); + assert_eq!(governor.reserved_bytes(), 5_120); + } + assert_eq!(governor.reserved_bytes(), 0); + } + + #[test] + fn concurrent_reservations_cannot_jointly_punch_through_the_floor() { + let dir = std::env::temp_dir(); + let free = free_bytes(&dir).unwrap(); + let floor = DiskFloorBytes::new(free.get().saturating_sub(6_144)); + let governor = DiskGovernor::new(floor); + let first = governor.reserve(&dir, ReserveBytes::new(4_096)).unwrap(); + let denied = governor.reserve(&dir, ReserveBytes::new(4_096)); + assert!( + matches!(denied, Err(ReserveError::BelowFloor { .. })), + "the second reservation must see the first still held" + ); + assert_eq!(governor.reserved_bytes(), 4_096); + drop(first); + assert_eq!(governor.reserved_bytes(), 0); + assert!(governor.reserve(&dir, ReserveBytes::new(4_096)).is_ok()); + } + + #[test] + fn a_probe_fault_leaves_the_ledger_untouched() { + let governor = DiskGovernor::new(DiskFloorBytes::new(0)); + let fault = governor.reserve( + Path::new("/definitely/not/a/mounted/path"), + ReserveBytes::new(4_096), + ); + assert!(matches!(fault, Err(ReserveError::Probe(_)))); + assert_eq!(governor.reserved_bytes(), 0); + } +} diff --git a/knot2/crates/knot-resource/src/fsio.rs b/knot2/crates/knot-resource/src/fsio.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-resource/src/fsio.rs @@ -0,0 +1,347 @@ +use std::fs::File; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, SystemTime}; + +const STAGING_INFIX: &str = ".knot-tmp."; + +// A staging file belongs to whoever is filling it, and another process filling +// the same target is normal for a knot sharing a volume with a migrate +// or maintenance run. Only reclaim one old enough that no live writer could +// still own it. +const STAGING_REAP_AFTER: Duration = Duration::from_secs(3600); + +#[derive(Debug)] +pub struct FsError { + pub path: PathBuf, + pub source: io::Error, +} + +impl FsError { + fn at(path: &Path, source: io::Error) -> Self { + Self { + path: path.to_path_buf(), + source, + } + } +} + +impl std::fmt::Display for FsError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}: {}", self.path.display(), self.source) + } +} + +impl std::error::Error for FsError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.source) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FileMode { + Inherited, + Private, +} + +pub fn staging_nonce() -> String { + static COUNTER: AtomicU64 = AtomicU64::new(0); + format!( + "{}.{}", + std::process::id(), + COUNTER.fetch_add(1, Ordering::Relaxed) + ) +} + +fn staging_prefix(name: &str) -> String { + format!(".{name}{STAGING_INFIX}") +} + +fn pre_hidden_staging_prefix(name: &str) -> String { + format!("{name}{STAGING_INFIX}") +} + +fn remove_matching( + dir: &Path, + prefixes: &[&str], + reclaimable: impl Fn(&std::fs::DirEntry) -> bool, +) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + entries + .filter_map(Result::ok) + .filter(|entry| { + entry + .file_name() + .to_str() + .is_some_and(|name| prefixes.iter().any(|prefix| name.starts_with(prefix))) + }) + .filter(|entry| reclaimable(entry)) + .for_each(|entry| { + let _ = std::fs::remove_file(entry.path()); + }); +} + +pub fn clear_temps(dir: &Path, prefix: &str) { + remove_matching(dir, &[prefix], |_| true); +} + +pub fn clear_stale(dir: &Path, prefix: &str) { + let now = SystemTime::now(); + remove_matching(dir, &[prefix], |entry| abandoned(entry, now)); +} + +fn abandoned(entry: &std::fs::DirEntry, now: SystemTime) -> bool { + entry + .metadata() + .and_then(|meta| meta.modified()) + .ok() + .and_then(|modified| now.duration_since(modified).ok()) + .is_some_and(|age| age >= STAGING_REAP_AFTER) +} + +pub fn clear_staging(path: &Path) { + let (Some(parent), Some(name)) = ( + path.parent(), + path.file_name().and_then(|name| name.to_str()), + ) else { + return; + }; + let (hidden, pre_hidden) = (staging_prefix(name), pre_hidden_staging_prefix(name)); + let now = SystemTime::now(); + remove_matching(parent, &[&hidden, &pre_hidden], |entry| { + abandoned(entry, now) + }); +} + +pub fn fsync_path(path: &Path) -> Result<(), FsError> { + match File::open(path) { + Ok(file) => file.sync_all().map_err(|error| FsError::at(path, error)), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(FsError::at(path, error)), + } +} + +fn create(path: &Path, mode: FileMode) -> io::Result { + let mut options = std::fs::OpenOptions::new(); + options.write(true).create(true).truncate(true); + #[cfg(unix)] + if mode == FileMode::Private { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + options.open(path) +} + +pub fn atomic_write(path: &Path, mode: FileMode, fill: F) -> Result<(), E> +where + F: FnOnce(&mut File) -> Result<(), E>, + E: From, +{ + let (Some(parent), Some(name)) = ( + path.parent(), + path.file_name().and_then(|name| name.to_str()), + ) else { + return Err(FsError::at(path, io::Error::from(io::ErrorKind::InvalidInput)).into()); + }; + clear_staging(path); + let staging = parent.join(format!("{}{}", staging_prefix(name), staging_nonce())); + + let outcome = create(&staging, mode) + .map_err(|error| E::from(FsError::at(path, error))) + .and_then(|mut file| { + fill(&mut file)?; + file.sync_all() + .map_err(|error| E::from(FsError::at(path, error))) + }) + .and_then(|()| { + std::fs::rename(&staging, path).map_err(|error| E::from(FsError::at(path, error))) + }); + + match outcome { + Ok(()) => fsync_path(parent).map_err(Into::into), + Err(error) => { + let _ = std::fs::remove_file(&staging); + Err(error) + } + } +} + +pub fn atomic_write_bytes(path: &Path, contents: &[u8], mode: FileMode) -> Result<(), FsError> { + let target = path.to_path_buf(); + atomic_write(path, mode, move |file| { + file.write_all(contents) + .map_err(|error| FsError::at(&target, error)) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn names_in(dir: &Path) -> Vec { + std::fs::read_dir(dir) + .unwrap() + .filter_map(Result::ok) + .filter_map(|entry| entry.file_name().to_str().map(str::to_string)) + .collect() + } + + fn age(path: &Path, by: Duration) { + let when = SystemTime::now() - by; + File::options() + .write(true) + .open(path) + .unwrap() + .set_times(std::fs::FileTimes::new().set_modified(when)) + .unwrap(); + } + + #[test] + fn a_write_stages_under_a_hidden_name_and_leaves_only_the_target_behind() { + let dir = tempfile::tempdir().unwrap(); + let target = dir.path().join("keys.sealed"); + let sibling = dir.path().join("keys.sealed.v2"); + let staging = std::sync::Mutex::new(Vec::new()); + atomic_write::(&target, FileMode::Private, |file| { + *staging.lock().unwrap() = names_in(dir.path()); + file.write_all(b"first") + .map_err(|error| FsError::at(&target, error)) + }) + .unwrap(); + atomic_write_bytes(&target, b"second", FileMode::Private).unwrap(); + atomic_write_bytes(&sibling, b"two", FileMode::Inherited).unwrap(); + + let staging = staging.into_inner().unwrap(); + assert_eq!( + staging + .iter() + .filter(|name| name.starts_with("keys.sealed")) + .count(), + 0, + "a tool matching on the target's own prefix mustn't find the half-written staging file" + ); + assert!( + staging + .iter() + .any(|name| name.starts_with(".keys.sealed.knot-tmp.")), + "saw {staging:?}" + ); + assert_eq!(std::fs::read(&target).unwrap(), b"second"); + assert_eq!( + std::fs::read(&sibling).unwrap(), + b"two", + "a name that extends another mustn't share its staging path" + ); + let left = names_in(dir.path()); + assert_eq!(left.len(), 2, "left staging files behind: {left:?}"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(&target).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o600, "sealed material stays owner-only"); + } + assert!(fsync_path(&dir.path().join("never-written")).is_ok()); + } + + #[test] + fn a_write_that_fails_or_overlaps_another_keeps_what_is_already_stored() { + let dir = tempfile::tempdir().unwrap(); + let target = dir.path().join("packed-refs"); + atomic_write_bytes(&target, b"kept", FileMode::Inherited).unwrap(); + let failed: Result<(), FsError> = atomic_write(&target, FileMode::Inherited, |_| { + Err(FsError::at(&target, io::Error::other("fill failed"))) + }); + assert!(failed.is_err()); + assert_eq!( + std::fs::read(&target).unwrap(), + b"kept", + "a failed write mustn't destroy what was already stored" + ); + assert_eq!( + names_in(dir.path()).len(), + 1, + "a failed write removes its staging file" + ); + + let overlapping: Result<(), FsError> = atomic_write(&target, FileMode::Inherited, |file| { + atomic_write_bytes( + &target, + b"the second writer's contents", + FileMode::Inherited, + )?; + file.write_all(b"first writer finishes after") + .map_err(|error| FsError::at(&target, error)) + }); + assert!( + overlapping.is_ok(), + "an overlapping write mustn't delete the staging file this one is filling: \ + {overlapping:?}" + ); + assert_eq!( + std::fs::read(&target).unwrap(), + b"first writer finishes after" + ); + } + + #[test] + fn a_write_reclaims_aged_staging_files_of_either_naming_and_spares_a_fresh_one() { + let dir = tempfile::tempdir().unwrap(); + let target = dir.path().join("packed-refs"); + let crashed = dir.path().join(".packed-refs.knot-tmp.4242.7"); + let pre_hidden = dir.path().join("packed-refs.knot-tmp.4242.8"); + let in_flight = dir.path().join(".packed-refs.knot-tmp.4243.0"); + std::fs::write(&crashed, b"left by a crashed run").unwrap(); + std::fs::write(&pre_hidden, b"left beside the target by an older build").unwrap(); + std::fs::write(&in_flight, b"another process is mid-write").unwrap(); + let aged = STAGING_REAP_AFTER + Duration::from_secs(60); + age(&crashed, aged); + age(&pre_hidden, aged); + + atomic_write_bytes(&target, b"fresh", FileMode::Inherited).unwrap(); + + assert!( + !crashed.exists(), + "the write reclaims a crashed run's staging file" + ); + assert!( + !pre_hidden.exists(), + "moving staging under a dot mustn't strand the temps the previous naming left" + ); + assert!( + in_flight.exists(), + "a second knot on the same volume is a supported deployment \ + whose live staging file this write mustn't sweep out from under its rename" + ); + } + + #[test] + fn a_prefix_sweep_spares_a_live_file_only_where_another_writer_could_own_it() { + let dir = tempfile::tempdir().unwrap(); + let crashed = dir.path().join(".knot-repack.4242.7.pack"); + let in_flight = dir.path().join(".knot-repack.4243.0.pack"); + let installed = dir.path().join("multi-pack-index"); + let bitmap = dir.path().join("multi-pack-index-abc.bitmap"); + std::fs::write(&crashed, b"left by a crashed run").unwrap(); + std::fs::write(&in_flight, b"another process is streaming into this").unwrap(); + std::fs::write(&installed, b"index").unwrap(); + std::fs::write(&bitmap, b"bitmap").unwrap(); + age(&crashed, STAGING_REAP_AFTER + Duration::from_secs(60)); + + clear_stale(dir.path(), ".knot-repack."); + clear_temps(dir.path(), "multi-pack-index"); + + assert!(!crashed.exists()); + assert!( + in_flight.exists(), + "deleting the staging pack another repack is streaming into fails that run's rename" + ); + assert!(!installed.exists()); + assert!( + !bitmap.exists(), + "removing the index without its bitmap would leave a bitmap describing an index that is gone" + ); + } +} diff --git a/knot2/crates/knot-resource/src/lib.rs b/knot2/crates/knot-resource/src/lib.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-resource/src/lib.rs @@ -0,0 +1,64 @@ +mod admission; +mod cpu; +mod disk; +mod fsio; +mod mem; +mod slots; + +pub use admission::{ + AdmitGuard, Burst, GlobalInflight, LimitConfig, PerPeerInflight, PreAuthLimiter, RateLimit, + RefillMicros, Refusal, +}; +pub use cpu::{Saturate, ThreadCount, gix_thread_limit, map_chunks, map_spans, saturate, threads}; +pub use disk::{ + DiskFloorBytes, DiskGovernor, DiskReservation, FreeBytes, ReserveBytes, ReserveError, + free_bytes as disk_free_bytes, +}; +pub use fsio::{ + FileMode, FsError, atomic_write, atomic_write_bytes, clear_staging, clear_stale, clear_temps, + fsync_path, staging_nonce, +}; +pub use mem::{ + AvailableBytes, BudgetSource, ChurnBytes, ConnectivityObjects, DecayMs, MemoryBudget, + MemoryHighBytes, PayloadBytes, WorkingSetBytes, advert_cache_bytes, available_bytes, + cache_shed_warranted, decay_warrants_apply, externalize_connectivity, ingest_admits, + ingest_admits_churn, ingest_base_budget, ingest_thread_limit, object_cache_bytes, + pack_cache_bytes, target_decay, +}; +pub use slots::{PackSlots, ReceiveSlots, ResolveSlots, SlotPermit, Slots}; + +#[derive(Clone, Copy, Debug, Default)] +pub struct Ceilings { + pub max_threads: Option, + pub max_memory: Option, +} + +#[derive(Clone, Copy, Debug)] +pub struct Report { + pub threads: ThreadCount, + pub memory: Option, + pub memory_source: BudgetSource, + pub memory_high_bytes: Option, +} + +pub fn init(ceilings: Ceilings) -> Report { + let threads = cpu::install( + ceilings + .max_threads + .unwrap_or_else(|| ThreadCount::new(default_threads())), + ); + let (memory, memory_source) = mem::install(ceilings.max_memory); + let memory_high_bytes = mem::try_set_memory_high(); + Report { + threads, + memory, + memory_source, + memory_high_bytes, + } +} + +fn default_threads() -> usize { + std::thread::available_parallelism() + .map(std::num::NonZeroUsize::get) + .unwrap_or(1) +} diff --git a/knot2/crates/knot-resource/src/mem.rs b/knot2/crates/knot-resource/src/mem.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-resource/src/mem.rs @@ -0,0 +1,827 @@ +//! # Memory sensing & sizing. +//! +//! Everything is derived from: +//! - `MemoryBudget`: how much mem this process may use in total, +//! which we detect once then cache. +//! - `AvailableBytes`: how much is free right now, +//! which we read afresh every call. +//! +//! Sizing decisions that have to stay fixed for the lifetime of +//! a given process will refer to the mem budget, +//! while decisions that react to load will take a live reading +//! on the fly like a pulse. +//! +//! ```text +//! MemoryBudget, which is detected then cached +//! +//! configured cgroup MemTotal the budget source +//! ---------- ------ -------- ---------------- -------------------- +//! set any any configured Configured +//! - reads reads the smaller whichever won +//! - reads - the cgroup limit CgroupV2 or CgroupV1 +//! - - reads MemTotal ProcMeminfo +//! - - - None Unconstrained +//! +//! AvailableBytes, read every call +//! +//! cgroup v2 max -> anon, else v1 limit -> usage, else MemAvailable, else None +//! +//! MemoryBudget ---> object_cache_bytes, pack_cache_bytes, +//! \ ingest_base_budget +//! +--------> target_decay +//! / +//! AvailableBytes -> ingest_thread_limit, ingest_admits, +//! ingest_admits_churn, externalize_connectivity +//! ``` +//! +//! *Figure 1: where `MemoryBudget` & `AvailableBytes` originate, along with receivers.* +//! +//! In Figure 1, +//! a `-` means that source is absent or unreadable, +//! and `any` means we don't consult it at all. +//! The `cgroup` column is cgroup v2 `memory.max`, +//! or cgroup v1 `memory.limit_in_bytes` when the v2 file is +//! missing or just reads `max`. +//! +//! Every cgroup v2 reading reads this cgroup +//! up to root and takes the smallest `memory.max` it finds up the chain. +//! A `None` budget means unconstrained, +//! so every clamp returns its caller's +//! ceiling and `target_decay` reports a healthy interval. +//! +//! A `None` live-reading means the sensor is unavailable, +//! so every check takes the most permissive variant: +//! admission returns true, the thread limit stays at the CPU ceiling, +//! and `externalize_connectivity` returns false such that the +//! connectivity-map stays in RAM. +//! +//! Long story short, if the knot can't detect any limits it'll assume +//! it's allowed everything it can handle. +//! +//! In contrast, +//! `memory_high_target` uses neither of the above, +//! since `try_set_memory_high` passes it the cgroup v2 max directly, +//! which means a configured budget, +//! a `MemTotal` budget, and a cgroup v1 limit all don't affect it. +//! +//! `clamp_to_budget` will choose its cache size from a percentage of the budget, +//! between a floor and a ceiling. +//! Each of the following steps wins in some situation, +//! which Table 1 traces with the `object_cache_bytes` constants of +//! ceiling 64M, percent 2, floor 8M: +//! +//! ```text +//! ceiling.min(max(budget / 100 * percent, floor)).min(budget) +//! +//! budget budget*pct max(.,floor) min(ceiling,.) min(.,budget) winner +//! ------ ---------- ------------ -------------- ------------- ------- +//! 4M 0.08M 8M 8M 4M budget +//! 256M 5.1M 8M 8M 8M floor +//! 1G 20.5M 20.5M 20.5M 20.5M percent +//! 8G 163.8M 163.8M 64M 64M ceiling +//! ``` +//! +//! *Table 1: a budget per row, and which step decided it.* +//! +//! The 4M row is the only one where trailing `min` actually does anything, +//! since it covers a host whose entire budget is below the floor. +//! +//! `decay_for_headroom` maps headroom, meaning available over-budget, +//! onto the jemalloc dirty-page decay interval. +//! +//! // TODO: research if I can do this with mimalloc. +//! +//! When there's a lot of headroom, pages will stay cached for ten seconds, +//! but while under pressure the decay drops to zero such that pages go +//! back to the OS immediately. +//! Between those thresholds it interpolates like: +//! +//! ```text +//! decay ms +//! 10000 | ------------------ +//! | ,-' +//! | ,-' +//! | ,-' +//! | ,-' +//! | ,-' +//! 0 +===+-------------+-----------------+ +//! 0% 10% 50% 100% +//! headroom = available / budget +//! ``` +//! +//! *Figure 2: headroom mapped onto dirty-page decay interval.* +//! +//! That `=` run below 10% in Figure 2 is the curve itself, +//! I meant flat at zero, not the axis. :P +//! Between the thresholds the ramp climbs 250ms per point of headroom. +//! +//! `decay_warrants_apply` judges writes against the above curve. +//! Any move smaller than one second will be ignored, +//! so the reading has to shift like 4 points of +//! headroom before we rewrite the setting. +//! A target of 0 is exempt and always applies, +//! unless it happens to be the applied value already. + +use std::path::{Path, PathBuf}; +use std::sync::OnceLock; + +const CGROUP_V2_ROOT: &str = "/sys/fs/cgroup"; +const PROC_SELF_CGROUP: &str = "/proc/self/cgroup"; +const MEMORY_V1_LIMIT_PATH: &str = "/sys/fs/cgroup/memory/memory.limit_in_bytes"; +const MEMORY_V1_USAGE_PATH: &str = "/sys/fs/cgroup/memory/memory.usage_in_bytes"; +const MEMINFO_PATH: &str = "/proc/meminfo"; + +const CGROUP_V1_UNLIMITED: u64 = 0x7FFF_FFFF_FFFF_F000; + +const HIGH_HEADROOM_PERCENT: u64 = 10; +const HIGH_HEADROOM_LIMIT: u64 = 1024 * 1024 * 1024; + +const OBJECT_CACHE_CEILING: u64 = 64 * 1024 * 1024; +const OBJECT_CACHE_PERCENT: Percent = Percent::new(2); +const OBJECT_CACHE_FLOOR: u64 = 8 * 1024 * 1024; + +const PACK_CACHE_PERCENT: Percent = Percent::new(25); +const PACK_CACHE_FLOOR: u64 = 32 * 1024 * 1024; + +const ADVERT_CACHE_CEILING: u64 = 128 * 1024 * 1024; +const ADVERT_CACHE_PERCENT: Percent = Percent::new(5); +const ADVERT_CACHE_FLOOR: u64 = 8 * 1024 * 1024; + +const CACHE_SHED_PERCENT: u64 = 10; + +const INGEST_BASE_PERCENT: u64 = 40; + +const DECAY_HEALTHY_MS: isize = 10_000; +const DECAY_PRESSURE_MS: isize = 0; +const DECAY_HYSTERESIS_MS: isize = 1_000; +const HEADROOM_RELAXED_PERCENT: u64 = 50; +const HEADROOM_TIGHT_PERCENT: u64 = 10; + +const CONNECTIVITY_BYTES_PER_OBJECT: u64 = 96; +const INGEST_FIXED_BYTES: u64 = 16 * 1024 * 1024; +const INGEST_THREAD_WORKING_BYTES: u64 = 12 * 1024 * 1024; +const INGEST_CONCURRENCY_BYTES: u64 = 256 * 1024 * 1024; +const INGEST_CHURN_MULTIPLE: u64 = 4; + +knot_types::scalar_newtype! { + pub struct MemoryBudget(u64); +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct Percent(u64); + +impl Percent { + pub const fn new(percent: u64) -> Self { + // Const so a `250` would fail build for example. + // `clamp_to_budget` would otherwise have major problems + // at runtime. + assert!(percent <= 100, "percent exceeds 100"); + Self(percent) + } + + pub const fn get(self) -> u64 { + self.0 + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BudgetSource { + Configured, + CgroupV2, + CgroupV1, + ProcMeminfo, + Unconstrained, +} + +static BUDGET: OnceLock<(Option, BudgetSource)> = OnceLock::new(); + +pub(crate) fn install(configured: Option) -> (Option, BudgetSource) { + let computed = match configured { + Some(budget) => (Some(budget), BudgetSource::Configured), + None => detect_memory_budget(), + }; + *BUDGET.get_or_init(|| computed) +} + +fn resolved() -> Option { + BUDGET.get_or_init(detect_memory_budget).0 +} + +fn detect_cgroup_limit() -> Option<(u64, BudgetSource)> { + read_cgroup_max() + .map(|limit| (limit, BudgetSource::CgroupV2)) + .or_else(|| read_cgroup_v1_max().map(|limit| (limit, BudgetSource::CgroupV1))) +} + +fn detect_memory_budget() -> (Option, BudgetSource) { + match (detect_cgroup_limit(), read_meminfo_total()) { + (Some((cgroup, source)), Some(total)) => { + if cgroup <= total { + (Some(MemoryBudget::new(cgroup)), source) + } else { + (Some(MemoryBudget::new(total)), BudgetSource::ProcMeminfo) + } + } + (Some((cgroup, source)), None) => (Some(MemoryBudget::new(cgroup)), source), + (None, Some(total)) => (Some(MemoryBudget::new(total)), BudgetSource::ProcMeminfo), + (None, None) => (None, BudgetSource::Unconstrained), + } +} + +fn cgroup_v2_dir() -> Option { + let content = std::fs::read_to_string(PROC_SELF_CGROUP).ok()?; + let relative = content + .lines() + .find_map(|line| line.strip_prefix("0::"))? + .trim(); + Some(Path::new(CGROUP_V2_ROOT).join(relative.trim_start_matches('/'))) +} + +fn read_cgroup_max() -> Option { + let root = Path::new(CGROUP_V2_ROOT); + let mut dir = cgroup_v2_dir()?; + let mut effective: Option = None; + loop { + if let Some(limit) = std::fs::read_to_string(dir.join("memory.max")) + .ok() + .and_then(|raw| parse_cgroup_max(&raw)) + { + effective = Some(effective.map_or(limit, |current| current.min(limit))); + } + if dir == root { + break; + } + match dir.parent() { + Some(parent) if parent.starts_with(root) => dir = parent.to_path_buf(), + _ => break, + } + } + effective +} + +fn parse_cgroup_max(raw: &str) -> Option { + match raw.trim() { + "max" => None, + bytes => bytes.parse::().ok(), + } +} + +fn read_meminfo_total() -> Option { + parse_meminfo_field(&std::fs::read_to_string(MEMINFO_PATH).ok()?, "MemTotal:") +} + +fn parse_meminfo_field(raw: &str, key: &str) -> Option { + raw.lines() + .find_map(|line| line.strip_prefix(key)) + .and_then(|rest| rest.trim().strip_suffix("kB")) + .and_then(|kb| kb.trim().parse::().ok()) + .map(|kb| kb.saturating_mul(1024)) +} + +fn read_u64_file(path: &str) -> Option { + std::fs::read_to_string(path) + .ok()? + .trim() + .parse::() + .ok() +} + +fn read_memory_stat_field(stat: &str, key: &str) -> Option { + stat.lines().find_map(|line| { + let mut parts = line.split_whitespace(); + match (parts.next(), parts.next()) { + (Some(name), Some(value)) if name == key => value.parse::().ok(), + _ => None, + } + }) +} + +fn cgroup_v2_available() -> Option { + let dir = cgroup_v2_dir()?; + let stat = std::fs::read_to_string(dir.join("memory.stat")).ok()?; + let anon = read_memory_stat_field(&stat, "anon")?; + Some(read_cgroup_max()?.saturating_sub(anon)) +} + +fn read_cgroup_v1_max() -> Option { + read_u64_file(MEMORY_V1_LIMIT_PATH).filter(|&limit| limit < CGROUP_V1_UNLIMITED) +} + +fn cgroup_v1_available() -> Option { + let limit = read_cgroup_v1_max()?; + Some(limit.saturating_sub(read_u64_file(MEMORY_V1_USAGE_PATH)?)) +} + +fn meminfo_available() -> Option { + parse_meminfo_field( + &std::fs::read_to_string(MEMINFO_PATH).ok()?, + "MemAvailable:", + ) +} + +pub fn available_bytes() -> Option { + cgroup_v2_available() + .or_else(cgroup_v1_available) + .or_else(meminfo_available) + .map(AvailableBytes::new) +} + +fn clamp_to_budget( + budget: Option, + ceiling: u64, + percent: Percent, + floor: u64, +) -> u64 { + match budget { + None => ceiling, + Some(budget) => ceiling + .min((budget.get() / 100 * percent.get()).max(floor)) + .min(budget.get()), + } +} + +pub fn object_cache_bytes() -> usize { + let sized = clamp_to_budget( + resolved(), + OBJECT_CACHE_CEILING, + OBJECT_CACHE_PERCENT, + OBJECT_CACHE_FLOOR, + ); + usize::try_from(sized).unwrap_or(usize::MAX) +} + +pub fn pack_cache_bytes(configured: u64) -> u64 { + clamp_to_budget(resolved(), configured, PACK_CACHE_PERCENT, PACK_CACHE_FLOOR) +} + +pub fn advert_cache_bytes() -> u64 { + clamp_to_budget( + resolved(), + ADVERT_CACHE_CEILING, + ADVERT_CACHE_PERCENT, + ADVERT_CACHE_FLOOR, + ) +} + +pub fn cache_shed_warranted() -> bool { + shed_warranted_at(available_bytes(), resolved()) +} + +fn shed_warranted_at(available: Option, budget: Option) -> bool { + match (available, budget) { + (Some(available), Some(budget)) if budget.get() > 0 => { + available.get().saturating_mul(100) / budget.get() < CACHE_SHED_PERCENT + } + _ => false, + } +} + +fn ingest_base_budget_for(budget: MemoryBudget) -> usize { + let sized = budget.get() / 100 * INGEST_BASE_PERCENT; + usize::try_from(sized).unwrap_or(usize::MAX) +} + +pub fn ingest_base_budget() -> Option { + resolved().map(ingest_base_budget_for) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DecayMs(isize); + +impl DecayMs { + pub const fn ms(self) -> isize { + self.0 + } +} + +fn decay_for_headroom(available: Option, budget: Option) -> DecayMs { + let (available, budget) = match (available, budget) { + (Some(available), Some(budget)) if budget.get() > 0 => (available.get(), budget.get()), + _ => return DecayMs(DECAY_HEALTHY_MS), + }; + let headroom_percent = available.saturating_mul(100) / budget; + let decay = if headroom_percent >= HEADROOM_RELAXED_PERCENT { + DECAY_HEALTHY_MS + } else if headroom_percent <= HEADROOM_TIGHT_PERCENT { + DECAY_PRESSURE_MS + } else { + let span = (HEADROOM_RELAXED_PERCENT - HEADROOM_TIGHT_PERCENT) as isize; + let above = (headroom_percent - HEADROOM_TIGHT_PERCENT) as isize; + DECAY_HEALTHY_MS * above / span + }; + DecayMs(decay) +} + +pub fn target_decay() -> DecayMs { + decay_for_headroom(available_bytes(), resolved()) +} + +pub fn decay_warrants_apply(applied: DecayMs, target: DecayMs) -> bool { + if applied == target { + false + } else if target.ms() == DECAY_PRESSURE_MS { + true + } else { + (target.ms() - applied.ms()).abs() >= DECAY_HYSTERESIS_MS + } +} + +fn connectivity_fits(count: ConnectivityObjects, available: Option) -> bool { + match available { + Some(available) => { + count.get().saturating_mul(CONNECTIVITY_BYTES_PER_OBJECT) <= available.get() / 2 + } + None => true, + } +} + +pub fn externalize_connectivity(count: ConnectivityObjects) -> bool { + !connectivity_fits(count, available_bytes()) +} + +fn ingest_threads_for(ceiling: usize, available: Option) -> usize { + match available { + Some(available) => { + let funded = + available.get().saturating_sub(INGEST_FIXED_BYTES) / INGEST_CONCURRENCY_BYTES; + ceiling + .min(usize::try_from(funded).unwrap_or(ceiling)) + .max(1) + } + None => ceiling, + } +} + +fn ingest_floor_for(ceiling: usize, available: AvailableBytes) -> u64 { + INGEST_FIXED_BYTES + + ingest_threads_for(ceiling, Some(available)) as u64 * INGEST_THREAD_WORKING_BYTES +} + +fn ingest_admits_for( + ceiling: usize, + available: Option, + payload_bytes: PayloadBytes, +) -> bool { + match available { + Some(available) => { + ingest_floor_for(ceiling, available).saturating_add(payload_bytes.get()) + <= available.get() + } + None => true, + } +} + +pub fn ingest_thread_limit() -> usize { + ingest_threads_for(crate::cpu::ceiling(), available_bytes()) +} + +pub fn ingest_admits(payload: PayloadBytes) -> bool { + ingest_admits_for(crate::cpu::ceiling(), available_bytes(), payload) +} + +knot_types::scalar_newtype! { + pub struct WorkingSetBytes(u64); + pub struct ChurnBytes(u64); + pub struct AvailableBytes(u64); + pub struct PayloadBytes(u64); + pub struct ConnectivityObjects(u64); + pub struct MemoryHighBytes(u64); +} + +fn ingest_admits_churn_for( + ceiling: usize, + available: Option, + working_set: WorkingSetBytes, + churn: ChurnBytes, +) -> bool { + match available { + Some(available) => { + ingest_admits_for(ceiling, Some(available), PayloadBytes::new(working_set.0)) + && churn.0 <= available.get().saturating_mul(INGEST_CHURN_MULTIPLE) + } + None => true, + } +} + +pub fn ingest_admits_churn(working_set: WorkingSetBytes, churn: ChurnBytes) -> bool { + ingest_admits_churn_for(crate::cpu::ceiling(), available_bytes(), working_set, churn) +} + +fn memory_high_target(max: u64) -> u64 { + let headroom = (max / 100 * HIGH_HEADROOM_PERCENT).min(HIGH_HEADROOM_LIMIT); + max.saturating_sub(headroom) +} + +pub(crate) fn try_set_memory_high() -> Option { + let dir = cgroup_v2_dir()?; + let high = memory_high_target(read_cgroup_max()?); + std::fs::write(dir.join("memory.high"), high.to_string()) + .ok() + .map(|()| MemoryHighBytes::new(high)) +} + +#[cfg(test)] +mod tests { + use super::*; + + const GIB: u64 = 1024 * 1024 * 1024; + const MIB: u64 = 1024 * 1024; + + #[test] + fn an_unlimited_cgroup_reads_as_no_budget() { + assert_eq!(parse_cgroup_max("max\n"), None); + assert_eq!(parse_cgroup_max("104857600\n"), Some(104_857_600)); + assert_eq!(parse_cgroup_max("garbage"), None); + } + + #[test] + fn meminfo_fields_parse_kilobytes_into_bytes() { + let sample = "MemTotal: 16384 kB\nMemFree: 100 kB\nMemAvailable: 8192 kB\n"; + assert_eq!(parse_meminfo_field(sample, "MemTotal:"), Some(16384 * 1024)); + assert_eq!( + parse_meminfo_field(sample, "MemAvailable:"), + Some(8192 * 1024) + ); + assert_eq!(parse_meminfo_field(sample, "Nothing:"), None); + } + + #[test] + fn memory_stat_matches_the_whole_key_not_a_prefix() { + let sample = "anon 2097152\nfile 8388608\nanon_thp 0\nkernel 65536\n"; + assert_eq!(read_memory_stat_field(sample, "anon"), Some(2_097_152)); + assert_eq!(read_memory_stat_field(sample, "file"), Some(8_388_608)); + assert_eq!(read_memory_stat_field(sample, "anon_thp"), Some(0)); + assert_eq!(read_memory_stat_field(sample, "missing"), None); + } + + #[test] + fn the_sensor_reads_live_memory_on_this_host() { + let available = available_bytes() + .expect("a Linux host must expose live memory availability") + .get(); + assert!( + available > 0, + "available memory must be positive, got {available}" + ); + } + + #[test] + fn an_unconstrained_host_keeps_the_ceiling() { + assert_eq!( + clamp_to_budget(None, 500_000_000, Percent::new(25), 32), + 500_000_000 + ); + } + + #[test] + fn a_constrained_host_clamps_to_the_fraction() { + let budget = Some(MemoryBudget::new(400 * MIB)); + assert_eq!( + clamp_to_budget(budget, 4 * GIB, PACK_CACHE_PERCENT, PACK_CACHE_FLOOR), + 100 * MIB + ); + } + + #[test] + fn a_tiny_host_holds_the_floor_but_never_exceeds_the_budget() { + let budget = Some(MemoryBudget::new(16 * MIB)); + assert_eq!( + clamp_to_budget(budget, 4 * GIB, PACK_CACHE_PERCENT, PACK_CACHE_FLOOR), + 16 * MIB + ); + } + + #[test] + fn a_small_host_reserves_the_headroom_percent() { + let max = 4 * GIB; + let headroom = max / 100 * HIGH_HEADROOM_PERCENT; + assert!(headroom < HIGH_HEADROOM_LIMIT); + assert_eq!(memory_high_target(max), max - headroom); + } + + #[test] + fn a_large_host_bounds_the_reclaim_headroom() { + let max = 128 * GIB; + assert!(max / 100 * HIGH_HEADROOM_PERCENT > HIGH_HEADROOM_LIMIT); + assert_eq!(memory_high_target(max), max - HIGH_HEADROOM_LIMIT); + } + + #[test] + fn a_healthy_host_keeps_the_allocator_lazy_and_a_squeezed_one_reclaims() { + let budget = Some(MemoryBudget::new(4 * GIB)); + assert_eq!( + decay_for_headroom(Some(AvailableBytes::new(3 * GIB)), budget).ms(), + DECAY_HEALTHY_MS, + "ample headroom stays fast" + ); + assert_eq!( + decay_for_headroom(Some(AvailableBytes::new(GIB / 4)), budget).ms(), + DECAY_PRESSURE_MS, + "near-exhaustion reclaims at once" + ); + assert_eq!( + decay_for_headroom(Some(AvailableBytes::new(2 * GIB)), budget).ms(), + DECAY_HEALTHY_MS, + "half-free sits at the relaxed threshold" + ); + } + + #[test] + fn cache_shedding_triggers_only_under_tight_headroom() { + let budget = Some(MemoryBudget::new(4 * GIB)); + assert!( + !shed_warranted_at(Some(AvailableBytes::new(2 * GIB)), budget), + "ample headroom keeps caches" + ); + assert!( + shed_warranted_at(Some(AvailableBytes::new(GIB / 4)), budget), + "tight headroom sheds caches" + ); + assert!( + !shed_warranted_at(Some(AvailableBytes::new(GIB)), None), + "an unmeasured budget never sheds" + ); + } + + #[test] + fn the_decay_interpolates_across_the_pressure_band() { + let budget = Some(MemoryBudget::new(100 * MIB)); + assert_eq!( + decay_for_headroom(Some(AvailableBytes::new(30 * MIB)), budget).ms(), + DECAY_HEALTHY_MS * 20 / 40, + "30% headroom is halfway through the 10..50 band" + ); + } + + #[test] + fn ingest_parallelism_backs_off_as_memory_tightens() { + assert_eq!( + ingest_threads_for(8, Some(AvailableBytes::new(4 * GIB))), + 8, + "a roomy host keeps the full cpu ceiling" + ); + assert_eq!( + ingest_threads_for(8, Some(AvailableBytes::new(GIB))), + 3, + "a 1GB limit funds only ~3 ingest threads, far below a many-core ceiling, so \ + decompression churn cannot outrun the munmap-on-free page return and grow unbounded" + ); + assert_eq!( + ingest_threads_for(8, Some(AvailableBytes::new(176 * MIB))), + 1, + "a squeezed host drops to a single ingest thread, shrinking the working set" + ); + assert_eq!( + ingest_threads_for(8, None), + 8, + "an unmeasurable host keeps the ceiling" + ); + } + + #[test] + fn the_base_spill_budget_stays_a_fraction_so_it_can_bound_a_small_host() { + assert_eq!( + ingest_base_budget_for(MemoryBudget::new(64 * GIB)), + (64 * GIB / 100 * INGEST_BASE_PERCENT) as usize, + "a roomy host spills only after the working set passes 40% of its RAM" + ); + assert!( + (ingest_base_budget_for(MemoryBudget::new(300 * MIB)) as u64) < 300 * MIB, + "a squeezed host keeps the spill threshold under its total, or it OOMs before paging" + ); + } + + #[test] + fn ingest_admission_scales_its_floor_with_the_threads_it_will_actually_use() { + assert!( + ingest_admits_for( + 8, + Some(AvailableBytes::new(32 * MIB)), + PayloadBytes::new(MIB) + ), + "a small push fits a 32MB host by running a single ~28MB-floor ingest thread" + ); + assert!( + !ingest_admits_for( + 8, + Some(AvailableBytes::new(20 * MIB)), + PayloadBytes::new(MIB) + ), + "below the one-thread floor the push is shed, never OOM-ed part way through" + ); + assert!( + !ingest_admits_for( + 8, + Some(AvailableBytes::new(64 * MIB)), + PayloadBytes::new(200 * MIB) + ), + "a payload that dwarfs free memory is declined" + ); + assert!( + ingest_admits_for(8, None, PayloadBytes::new(u64::MAX)), + "an unmeasurable host proceeds optimistically" + ); + } + + #[test] + fn ingest_churn_sheds_a_pack_whose_decompressed_volume_dwarfs_free_memory() { + assert!( + ingest_admits_churn_for( + 8, + Some(AvailableBytes::new(GIB)), + WorkingSetBytes(MIB), + ChurnBytes(3 * GIB) + ), + "churn within a few multiples of free memory rides on the working-set floor" + ); + assert!( + !ingest_admits_churn_for( + 8, + Some(AvailableBytes::new(GIB)), + WorkingSetBytes(MIB), + ChurnBytes(5 * GIB) + ), + "decompression volume past the multiple of free memory is shed, never OOM-ed" + ); + assert!( + !ingest_admits_churn_for( + 8, + Some(AvailableBytes::new(20 * MIB)), + WorkingSetBytes(MIB), + ChurnBytes(MIB) + ), + "below the one-thread working floor the pack is shed even with trivial churn" + ); + assert!( + ingest_admits_churn_for(8, None, WorkingSetBytes(u64::MAX), ChurnBytes(u64::MAX)), + "an unmeasurable host proceeds optimistically on both gates" + ); + } + + #[test] + fn connectivity_externalizes_only_when_the_in_ram_map_would_crowd_the_host() { + assert!( + connectivity_fits( + ConnectivityObjects::new(1_000_000), + Some(AvailableBytes::new(4 * GIB)) + ), + "a small closure fits with headroom to spare" + ); + assert!( + !connectivity_fits( + ConnectivityObjects::new(7_700_000), + Some(AvailableBytes::new(512 * MIB)) + ), + "nixpkgs cannot hold its connectivity map on a 512MB box" + ); + assert!( + connectivity_fits(ConnectivityObjects::new(u64::MAX), None), + "an unmeasurable host stays on the fast in-ram path" + ); + } + + #[test] + fn an_unmeasurable_host_stays_on_the_fast_default() { + assert_eq!(decay_for_headroom(None, None).ms(), DECAY_HEALTHY_MS); + assert_eq!( + decay_for_headroom(Some(AvailableBytes::new(GIB)), None).ms(), + DECAY_HEALTHY_MS, + "no budget means no pressure signal, so don't throttle" + ); + } + + #[test] + fn a_big_host_scales_up_to_the_ceiling_only() { + let budget = Some(MemoryBudget::new(256 * GIB)); + assert_eq!( + clamp_to_budget( + budget, + OBJECT_CACHE_CEILING, + OBJECT_CACHE_PERCENT, + OBJECT_CACHE_FLOOR + ), + OBJECT_CACHE_CEILING + ); + } + + #[test] + fn decay_hysteresis_absorbs_small_wobble_but_honors_the_pressure_floor() { + let healthy = DecayMs(DECAY_HEALTHY_MS); + assert!( + !decay_warrants_apply(healthy, healthy), + "an unchanged target never rewrites the arenas" + ); + assert!( + !decay_warrants_apply(DecayMs(5_000), DecayMs(5_200)), + "a sub-band change is ignored so a percent of headroom wobble doesn't churn" + ); + assert!( + decay_warrants_apply(DecayMs(5_000), DecayMs(7_000)), + "a change past the hysteresis band is applied" + ); + assert!( + decay_warrants_apply(DecayMs(200), DecayMs(DECAY_PRESSURE_MS)), + "a move to the pressure floor is always honored so RSS reclaim is never delayed" + ); + } +} diff --git a/knot2/crates/knot-resource/src/slots.rs b/knot2/crates/knot-resource/src/slots.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-resource/src/slots.rs @@ -0,0 +1,135 @@ +use std::sync::{Arc, OnceLock}; + +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; + +use crate::cpu::threads; + +const RESOLVE_SLOTS_PER_TRANSPORT: usize = 16; + +const SHARING_TRANSPORTS: usize = 2; + +const RESOLVE_SLOTS: usize = RESOLVE_SLOTS_PER_TRANSPORT * SHARING_TRANSPORTS; + +static PROCESS: OnceLock = OnceLock::new(); + +pub struct SlotPermit(#[allow(dead_code)] OwnedSemaphorePermit); + +macro_rules! slot_kind { + ($($name:ident),+ $(,)?) => {$( + #[derive(Clone)] + pub struct $name(Arc); + + impl $name { + pub fn new(permits: usize) -> Self { + Self(Arc::new(Semaphore::new(permits.max(1)))) + } + + pub async fn acquire(&self) -> SlotPermit { + SlotPermit( + Arc::clone(&self.0) + .acquire_owned() + .await + .expect("a slot budget closes only with the process that owns it"), + ) + } + + pub fn available(&self) -> usize { + self.0.available_permits() + } + } + )+}; +} + +slot_kind!(ResolveSlots, ReceiveSlots, PackSlots); + +impl ResolveSlots { + pub fn try_acquire(&self) -> Option { + Arc::clone(&self.0).try_acquire_owned().ok().map(SlotPermit) + } +} + +#[derive(Clone)] +pub struct Slots { + pub resolve: ResolveSlots, + pub receive: ReceiveSlots, + pub pack: PackSlots, +} + +impl Slots { + pub fn for_machine() -> Self { + PROCESS + .get_or_init(|| Self { + resolve: ResolveSlots::new(RESOLVE_SLOTS), + receive: ReceiveSlots::new(threads().get()), + pack: PackSlots::new(threads().get()), + }) + .clone() + } + + pub fn testing(permits: usize) -> Self { + Self { + resolve: ResolveSlots::new(permits), + receive: ReceiveSlots::new(permits), + pack: PackSlots::new(permits), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn a_clone_shares_one_pool_per_kind_of_work() { + let slots = Slots::testing(1); + let other = slots.clone(); + let held = slots.receive.acquire().await; + assert_eq!( + other.receive.available(), + 0, + "a cloned budget mustn't grant a second permit for the one slot" + ); + assert_eq!( + other.pack.available(), + 1, + "spending a receive slot mustn't spend the pack budget" + ); + let _resolving = slots.resolve.acquire().await; + assert!( + other.resolve.try_acquire().is_none(), + "a cosmetic lookup mustn't wait, \ + or a push queues on it while its receive and pack slots stay spent" + ); + drop(held); + assert_eq!(other.receive.available(), 1); + } + + #[tokio::test] + async fn every_caller_of_for_machine_shares_one_budget_that_no_test_budget_touches() { + let ssh = Slots::for_machine(); + let http = Slots::for_machine(); + let isolated = Slots::testing(1); + let before = http.receive.available(); + let _held = ssh.receive.acquire().await; + assert_eq!( + http.receive.available(), + before - 1, + "two transports asking the machine for a budget must get the same one, \ + or the process grants twice the concurrency it was configured for" + ); + let resolving: Vec = (0..RESOLVE_SLOTS_PER_TRANSPORT) + .filter_map(|_| ssh.resolve.try_acquire()) + .collect(); + assert_eq!(resolving.len(), RESOLVE_SLOTS_PER_TRANSPORT); + assert!( + http.resolve.try_acquire().is_some(), + "collapsing a per-transport pool into a process-wide one mustn't give a deployment \ + running both transports less outbound resolution than either had on its own" + ); + assert_eq!( + isolated.receive.available(), + 1, + "whatever the process budget is doing mustn't spend a test budget" + ); + } +} diff --git a/knot2/crates/knot-runtime/src/clock.rs b/knot2/crates/knot-runtime/src/clock.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-runtime/src/clock.rs @@ -0,0 +1,84 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +pub use knot_types::UnixMicros; + +pub trait Clock: Send + Sync + 'static { + fn now_unix_micros(&self) -> UnixMicros; +} + +impl Clock for Arc { + fn now_unix_micros(&self) -> UnixMicros { + (**self).now_unix_micros() + } +} + +pub struct SystemClock; + +impl Clock for SystemClock { + fn now_unix_micros(&self) -> UnixMicros { + let micros = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|elapsed| elapsed.as_micros() as u64) + .unwrap_or(0); + UnixMicros::new(micros) + } +} + +pub struct ManualClock { + micros: AtomicU64, +} + +impl ManualClock { + pub fn new(start: UnixMicros) -> Self { + Self { + micros: AtomicU64::new(start.get()), + } + } + + pub fn advance(&self, delta: Duration) { + self.micros + .fetch_add(delta.as_micros() as u64, Ordering::SeqCst); + } +} + +impl Clock for ManualClock { + fn now_unix_micros(&self) -> UnixMicros { + UnixMicros::new(self.micros.load(Ordering::SeqCst)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn manual_clock_advances() { + let clock = ManualClock::new(UnixMicros::new(1_000)); + assert_eq!(clock.now_unix_micros().get(), 1_000); + clock.advance(Duration::from_micros(500)); + assert_eq!(clock.now_unix_micros().get(), 1_500); + } + + #[test] + fn manual_clock_same_start_same_sequence() { + let one = ManualClock::new(UnixMicros::new(1_000)); + let two = ManualClock::new(UnixMicros::new(1_000)); + let advances = [10, 250, 7, 1_000]; + advances.iter().for_each(|&step| { + one.advance(Duration::from_micros(step)); + two.advance(Duration::from_micros(step)); + assert_eq!(one.now_unix_micros(), two.now_unix_micros()); + }); + } + + #[test] + fn a_shared_clock_advances_through_the_trait_object() { + let shared = Arc::new(ManualClock::new(UnixMicros::new(1_000))); + let view: Arc = Arc::clone(&shared) as Arc; + assert_eq!(view.now_unix_micros().get(), 1_000); + shared.advance(Duration::from_micros(250)); + assert_eq!(view.now_unix_micros().get(), 1_250); + } +} diff --git a/knot2/crates/knot-runtime/src/dns.rs b/knot2/crates/knot-runtime/src/dns.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-runtime/src/dns.rs @@ -0,0 +1,86 @@ +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; + +use tokio::sync::OnceCell; + +use crate::http::NetworkError; + +pub type DnsFuture = Pin, NetworkError>> + Send>>; + +pub trait DnsTxtResolver: Send + Sync + 'static { + fn lookup_txt(&self, name: String) -> DnsFuture; +} + +pub struct SystemDns { + resolver: Arc>, +} + +impl SystemDns { + pub fn new() -> Self { + Self { + resolver: Arc::new(OnceCell::new()), + } + } +} + +impl Default for SystemDns { + fn default() -> Self { + Self::new() + } +} + +impl DnsTxtResolver for SystemDns { + fn lookup_txt(&self, name: String) -> DnsFuture { + let cell = self.resolver.clone(); + Box::pin(async move { + let resolver = cell + .get_or_try_init(|| async { + hickory_resolver::TokioAsyncResolver::tokio_from_system_conf() + .map_err(|error| NetworkError::Build(error.to_string())) + }) + .await?; + match resolver.txt_lookup(name).await { + Ok(lookup) => Ok(lookup.iter().map(render_txt).collect()), + Err(error) => match error.kind() { + hickory_resolver::error::ResolveErrorKind::NoRecordsFound { .. } => { + Ok(Vec::new()) + } + _ => Err(NetworkError::Request(error.to_string())), + }, + } + }) + } +} + +fn render_txt(record: &hickory_resolver::proto::rr::rdata::TXT) -> String { + let bytes: Vec = record + .txt_data() + .iter() + .flat_map(|chunk| chunk.iter().copied()) + .collect(); + String::from_utf8_lossy(&bytes).into_owned() +} + +pub struct FakeDns { + responder: F, +} + +impl FakeDns +where + F: Fn(&str) -> Result, NetworkError> + Send + Sync + 'static, +{ + pub fn new(responder: F) -> Self { + Self { responder } + } +} + +impl DnsTxtResolver for FakeDns +where + F: Fn(&str) -> Result, NetworkError> + Send + Sync + 'static, +{ + fn lookup_txt(&self, name: String) -> DnsFuture { + let result = (self.responder)(&name); + Box::pin(async move { result }) + } +} diff --git a/knot2/crates/knot-runtime/src/entropy.rs b/knot2/crates/knot-runtime/src/entropy.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-runtime/src/entropy.rs @@ -0,0 +1,146 @@ +use std::sync::atomic::{AtomicU64, Ordering}; + +// Golden gamma gem alert +const GOLDEN_GAMMA: u64 = 0x9E37_79B9_7F4A_7C15; + +fn splitmix64(z: u64) -> u64 { + let z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + let z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) +} + +pub trait Entropy: Send + Sync + 'static { + fn next_u64(&self) -> u64; + fn fill(&self, buffer: &mut [u8]); + fn derive(&self, label: u64) -> Box; +} + +pub struct OsEntropy; + +impl Entropy for OsEntropy { + fn next_u64(&self) -> u64 { + let mut bytes = [0u8; 8]; + getrandom::fill(&mut bytes).expect("OS entropy unavailable"); + u64::from_le_bytes(bytes) + } + + fn fill(&self, buffer: &mut [u8]) { + getrandom::fill(buffer).expect("OS entropy unavailable"); + } + + fn derive(&self, _label: u64) -> Box { + Box::new(OsEntropy) + } +} + +pub struct SeededEntropy { + seed: u64, + state: AtomicU64, +} + +impl SeededEntropy { + pub fn new(seed: u64) -> Self { + Self { + seed, + state: AtomicU64::new(seed), + } + } + + pub fn derive(&self, label: u64) -> SeededEntropy { + SeededEntropy::new(splitmix64(self.seed ^ splitmix64(label))) + } +} + +impl Entropy for SeededEntropy { + fn next_u64(&self) -> u64 { + let z = self + .state + .fetch_add(GOLDEN_GAMMA, Ordering::SeqCst) + .wrapping_add(GOLDEN_GAMMA); + splitmix64(z) + } + + fn fill(&self, buffer: &mut [u8]) { + buffer.chunks_mut(8).for_each(|chunk| { + let value = self.next_u64().to_le_bytes(); + chunk.copy_from_slice(&value[..chunk.len()]); + }); + } + + fn derive(&self, label: u64) -> Box { + Box::new(SeededEntropy::derive(self, label)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn stream(entropy: &SeededEntropy, count: usize) -> Vec { + std::iter::repeat_with(|| entropy.next_u64()) + .take(count) + .collect() + } + + #[test] + fn seeded_entropy_is_deterministic() { + let one = SeededEntropy::new(42); + let two = SeededEntropy::new(42); + assert_eq!(stream(&one, 256), stream(&two, 256)); + } + + #[test] + fn distinct_seeds_diverge() { + assert_ne!( + stream(&SeededEntropy::new(1), 64), + stream(&SeededEntropy::new(2), 64) + ); + } + + #[test] + fn derive_is_independent_of_parent_draw_timing() { + let early = SeededEntropy::new(42); + let child_before = early.derive(7); + + let late = SeededEntropy::new(42); + let _ = stream(&late, 100); + let child_after = late.derive(7); + + assert_eq!(stream(&child_before, 128), stream(&child_after, 128)); + } + + #[test] + fn derived_streams_differ_by_label() { + let parent = SeededEntropy::new(42); + assert_ne!(stream(&parent.derive(1), 64), stream(&parent.derive(2), 64)); + } + + fn first_fill(entropy: &dyn Entropy, label: u64) -> [u8; 16] { + let mut buffer = [0u8; 16]; + entropy.derive(label).fill(&mut buffer); + buffer + } + + #[test] + fn trait_object_derive_is_independent_of_draw_order() { + let parent: &dyn Entropy = &SeededEntropy::new(77); + let in_order = [first_fill(parent, 10), first_fill(parent, 20)]; + + let parent: &dyn Entropy = &SeededEntropy::new(77); + let reversed = [first_fill(parent, 20), first_fill(parent, 10)]; + + assert_eq!(in_order[0], reversed[1]); + assert_eq!(in_order[1], reversed[0]); + assert_ne!(in_order[0], in_order[1]); + } + + #[test] + fn seeded_fill_matches_stream() { + let stream = SeededEntropy::new(7); + let expected = stream.next_u64().to_le_bytes(); + let bytes = SeededEntropy::new(7); + let mut buffer = [0u8; 8]; + bytes.fill(&mut buffer); + assert_eq!(buffer, expected); + } +} diff --git a/knot2/crates/knot-runtime/src/http.rs b/knot2/crates/knot-runtime/src/http.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-runtime/src/http.rs @@ -0,0 +1,496 @@ +use std::future::Future; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; +use std::pin::Pin; +use std::sync::Arc; +use std::time::Duration; + +use bytes::Bytes; +use futures::TryStreamExt; +use http::{HeaderMap, Method, StatusCode}; +use url::{Host, Url}; + +#[derive(Debug, Clone, thiserror::Error)] +pub enum NetworkError { + #[error("build: {0}")] + Build(String), + #[error("connect: {0}")] + Connect(String), + #[error("timeout: {0}")] + Timeout(String), + #[error("request: {0}")] + Request(String), + #[error("body: {0}")] + Body(String), + #[error("response exceeds {limit} bytes")] + TooLarge { limit: u64 }, + #[error("refusing to reach non-public address {host}")] + Blocked { host: String }, +} + +pub fn is_blocked_ip(ip: IpAddr) -> bool { + match ip { + IpAddr::V4(v4) => is_blocked_v4(v4), + IpAddr::V6(v6) => match embedded_ipv4(v6) { + Some(embedded) => is_blocked_v4(embedded), + None => { + v6.is_loopback() + || v6.is_unspecified() + || v6.is_multicast() + || (v6.segments()[0] & 0xfe00) == 0xfc00 + || (v6.segments()[0] & 0xffc0) == 0xfe80 + } + }, + } +} + +fn is_blocked_v4(v4: Ipv4Addr) -> bool { + v4.is_loopback() + || v4.is_private() + || v4.is_link_local() + || v4.is_unspecified() + || v4.is_broadcast() + || v4.is_documentation() + || v4.is_multicast() + || v4.octets()[0] == 0 + || v4.octets()[0] >= 240 + || matches!(v4.octets(), [100, second, ..] if (64..=127).contains(&second)) + || matches!(v4.octets(), [198, second, ..] if (18..=19).contains(&second)) +} + +fn embedded_ipv4(v6: Ipv6Addr) -> Option { + if let Some(mapped) = v6.to_ipv4() { + return Some(mapped); + } + let segments = v6.segments(); + if segments[0] == 0x2002 { + return Some(Ipv4Addr::new( + (segments[1] >> 8) as u8, + segments[1] as u8, + (segments[2] >> 8) as u8, + segments[2] as u8, + )); + } + if segments[0] == 0x0064 && segments[1] == 0xff9b && segments[2..6] == [0, 0, 0, 0] { + return Some(Ipv4Addr::new( + (segments[6] >> 8) as u8, + segments[6] as u8, + (segments[7] >> 8) as u8, + segments[7] as u8, + )); + } + None +} + +struct GuardedResolver; + +impl reqwest::dns::Resolve for GuardedResolver { + fn resolve(&self, name: reqwest::dns::Name) -> reqwest::dns::Resolving { + Box::pin(async move { + let host = name.as_str().to_owned(); + let resolved = tokio::net::lookup_host((host.as_str(), 0)).await?; + let allowed: Vec = + resolved.filter(|addr| !is_blocked_ip(addr.ip())).collect(); + if allowed.is_empty() { + return Err(Box::::from(format!( + "{host} resolves only to non-public addresses" + ))); + } + Ok(Box::new(allowed.into_iter()) as reqwest::dns::Addrs) + }) + } +} + +#[derive(Debug, Clone, Copy)] +pub struct HttpLimits { + pub connect_timeout: Duration, + pub read_timeout: Duration, + pub request_timeout: Duration, + pub max_response_bytes: u64, + pub block_private_addresses: bool, +} + +impl Default for HttpLimits { + fn default() -> Self { + Self { + connect_timeout: Duration::from_secs(5), + read_timeout: Duration::from_secs(30), + request_timeout: Duration::from_secs(60), + max_response_bytes: 16 * 1024 * 1024, + block_private_addresses: true, + } + } +} + +pub struct HttpRequest { + pub method: Method, + pub url: Url, + pub headers: HeaderMap, + pub body: Option, +} + +impl HttpRequest { + pub fn get(url: Url) -> Self { + Self { + method: Method::GET, + url, + headers: HeaderMap::new(), + body: None, + } + } + + pub fn post(url: Url, body: Bytes) -> Self { + Self { + method: Method::POST, + url, + headers: HeaderMap::new(), + body: Some(body), + } + } +} + +#[derive(Debug)] +pub struct HttpResponse { + pub status: StatusCode, + pub headers: HeaderMap, + pub body: Bytes, +} + +pub type HttpFuture = Pin> + Send>>; + +pub type ByteStream = Pin> + Send>>; + +pub struct StreamedResponse { + pub status: StatusCode, + pub headers: HeaderMap, + pub body: ByteStream, +} + +pub type StreamFuture = + Pin> + Send>>; + +pub trait HttpTransport: Send + Sync + 'static { + fn execute(&self, request: HttpRequest) -> HttpFuture; + + fn execute_streamed(&self, request: HttpRequest) -> StreamFuture { + let response = self.execute(request); + Box::pin(async move { + let response = response.await?; + Ok(StreamedResponse { + status: response.status, + headers: response.headers, + body: Box::pin(futures::stream::once(std::future::ready(Ok(response.body)))), + }) + }) + } +} + +pub struct ReqwestHttp { + client: reqwest::Client, + max_response_bytes: u64, + block_private_addresses: bool, +} + +impl ReqwestHttp { + pub fn new(limits: HttpLimits) -> Result { + let mut builder = reqwest::Client::builder() + .connect_timeout(limits.connect_timeout) + .read_timeout(limits.read_timeout) + .timeout(limits.request_timeout) + .redirect(reqwest::redirect::Policy::none()); + if limits.block_private_addresses { + builder = builder.dns_resolver(Arc::new(GuardedResolver)); + } + if let Some(path) = std::env::var_os("KNOT_EXTRA_CA_FILE") { + let pem = + std::fs::read(&path).map_err(|error| NetworkError::Build(error.to_string()))?; + builder = reqwest::Certificate::from_pem_bundle(&pem) + .map_err(|error| NetworkError::Build(error.to_string()))? + .into_iter() + .fold(builder, reqwest::ClientBuilder::add_root_certificate); + } + let client = builder + .build() + .map_err(|error| NetworkError::Build(error.to_string()))?; + Ok(Self { + client, + max_response_bytes: limits.max_response_bytes, + block_private_addresses: limits.block_private_addresses, + }) + } +} + +impl HttpTransport for ReqwestHttp { + fn execute(&self, request: HttpRequest) -> HttpFuture { + let client = self.client.clone(); + let limit = self.max_response_bytes; + let guard = self.block_private_addresses; + Box::pin(async move { + if let Some(host) = guard.then(|| blocked_literal(&request.url)).flatten() { + return Err(NetworkError::Blocked { host }); + } + let mut builder = client + .request(request.method, request.url) + .headers(request.headers); + if let Some(body) = request.body { + builder = builder.body(body); + } + let response = builder.send().await.map_err(map_reqwest)?; + let status = response.status(); + let headers = response.headers().clone(); + if response.content_length().is_some_and(|len| len > limit) { + return Err(NetworkError::TooLarge { limit }); + } + let body = bounded_body(response, limit).await?; + Ok(HttpResponse { + status, + headers, + body, + }) + }) + } + + fn execute_streamed(&self, request: HttpRequest) -> StreamFuture { + let client = self.client.clone(); + let guard = self.block_private_addresses; + Box::pin(async move { + if let Some(host) = guard.then(|| blocked_literal(&request.url)).flatten() { + return Err(NetworkError::Blocked { host }); + } + let mut builder = client + .request(request.method, request.url) + .headers(request.headers); + if let Some(body) = request.body { + builder = builder.body(body); + } + let response = builder.send().await.map_err(map_reqwest)?; + let status = response.status(); + let headers = response.headers().clone(); + let body: ByteStream = Box::pin(response.bytes_stream().map_err(|error| { + if error.is_timeout() { + NetworkError::Timeout(error.to_string()) + } else { + NetworkError::Body(error.to_string()) + } + })); + Ok(StreamedResponse { + status, + headers, + body, + }) + }) + } +} + +async fn bounded_body(response: reqwest::Response, limit: u64) -> Result { + response + .bytes_stream() + .map_err(|error| { + if error.is_timeout() { + NetworkError::Timeout(error.to_string()) + } else { + NetworkError::Body(error.to_string()) + } + }) + .try_fold(Vec::new(), |mut buffer, chunk| async move { + if buffer.len() as u64 + chunk.len() as u64 > limit { + return Err(NetworkError::TooLarge { limit }); + } + buffer.extend_from_slice(&chunk); + Ok(buffer) + }) + .await + .map(Bytes::from) +} + +fn blocked_literal(url: &Url) -> Option { + match url.host()? { + Host::Ipv4(ip) if is_blocked_ip(IpAddr::V4(ip)) => Some(ip.to_string()), + Host::Ipv6(ip) if is_blocked_ip(IpAddr::V6(ip)) => Some(ip.to_string()), + _ => None, + } +} + +fn map_reqwest(error: reqwest::Error) -> NetworkError { + if error.is_timeout() { + NetworkError::Timeout(error.to_string()) + } else if error.is_connect() { + NetworkError::Connect(error.to_string()) + } else { + NetworkError::Request(error.to_string()) + } +} + +pub struct FakeHttp { + responder: F, +} + +impl FakeHttp +where + F: Fn(&HttpRequest) -> Result + Send + Sync + 'static, +{ + pub fn new(responder: F) -> Self { + Self { responder } + } +} + +impl HttpTransport for FakeHttp +where + F: Fn(&HttpRequest) -> Result + Send + Sync + 'static, +{ + fn execute(&self, request: HttpRequest) -> HttpFuture { + let result = (self.responder)(&request); + Box::pin(async move { result }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use futures::StreamExt; + use std::io::{Read, Write}; + use std::net::{SocketAddr, TcpListener}; + + #[test] + fn blocked_addresses_cover_the_internal_ranges() { + let blocked = [ + "127.0.0.1", + "10.0.0.5", + "192.168.1.1", + "172.16.0.1", + "169.254.169.254", + "100.64.0.1", + "198.18.0.1", + "240.0.0.1", + "0.0.0.0", + "::1", + "::ffff:127.0.0.1", + "fd00::1", + "fe80::1", + "2002:7f00:1::", + "64:ff9b::7f00:1", + ]; + for raw in blocked { + assert!( + is_blocked_ip(raw.parse().unwrap()), + "{raw} should be blocked" + ); + } + let allowed = [ + "1.1.1.1", + "8.8.8.8", + "93.184.216.34", + "2606:4700:4700::1111", + "2002:808:808::", + "64:ff9b::808:808", + ]; + for raw in allowed { + assert!( + !is_blocked_ip(raw.parse().unwrap()), + "{raw} should be allowed" + ); + } + } + + fn tiny_limits(max_response_bytes: u64, request_timeout: Duration) -> HttpLimits { + HttpLimits { + connect_timeout: Duration::from_millis(200), + read_timeout: Duration::from_millis(200), + request_timeout, + max_response_bytes, + block_private_addresses: false, + } + } + + fn serve_body(body: Vec, with_content_length: bool) -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind"); + let addr = listener.local_addr().expect("addr"); + std::thread::spawn(move || { + if let Ok((mut stream, _)) = listener.accept() { + let _ = stream.read(&mut [0u8; 1024]); + let header = if with_content_length { + format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n", body.len()) + } else { + "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n".to_string() + }; + let _ = stream.write_all(header.as_bytes()); + let _ = stream.write_all(&body); + } + }); + addr + } + + fn serve_hang() -> SocketAddr { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind"); + let addr = listener.local_addr().expect("addr"); + std::thread::spawn(move || { + if let Ok((mut stream, _)) = listener.accept() { + let _ = stream.read(&mut [0u8; 1024]); + std::thread::sleep(Duration::from_secs(5)); + drop(stream); + } + }); + addr + } + + async fn fetch(addr: SocketAddr, limits: HttpLimits) -> Result { + let transport = ReqwestHttp::new(limits).expect("client builds"); + let url = Url::parse(&format!("http://{addr}/")).expect("url"); + transport.execute(HttpRequest::get(url)).await + } + + #[tokio::test] + async fn oversized_response_is_rejected_whether_declared_or_streamed() { + futures::stream::iter([true, false]) + .for_each(|with_content_length| async move { + let addr = serve_body(vec![0u8; 4096], with_content_length); + let result = fetch(addr, tiny_limits(64, Duration::from_secs(2))).await; + assert!(matches!(result, Err(NetworkError::TooLarge { limit: 64 }))); + }) + .await; + } + + #[tokio::test] + async fn small_response_within_limit_succeeds() { + let addr = serve_body(b"pong".to_vec(), true); + let response = fetch(addr, tiny_limits(64, Duration::from_secs(2))) + .await + .expect("response within limit"); + assert_eq!(response.body.as_ref(), b"pong"); + } + + #[tokio::test] + async fn unresponsive_server_times_out() { + let addr = serve_hang(); + let result = fetch(addr, tiny_limits(1024, Duration::from_millis(150))).await; + assert!(matches!(result, Err(NetworkError::Timeout(_)))); + } + + #[tokio::test] + async fn connect_failure_surfaces_typed_error() { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind"); + let addr = listener.local_addr().expect("addr"); + drop(listener); + let result = fetch(addr, tiny_limits(1024, Duration::from_secs(2))).await; + assert!(matches!( + result, + Err(NetworkError::Connect(_) | NetworkError::Request(_) | NetworkError::Timeout(_)) + )); + } + + #[test] + fn fake_http_returns_canned_response() { + let transport = FakeHttp::new(|request: &HttpRequest| { + assert_eq!(request.method, Method::GET); + Ok(HttpResponse { + status: StatusCode::OK, + headers: HeaderMap::new(), + body: Bytes::from_static(b"pong"), + }) + }); + let request = HttpRequest::get(Url::parse("https://oyster.cafe/ping").unwrap()); + let response = + futures::executor::block_on(transport.execute(request)).expect("fake response"); + assert_eq!(response.status, StatusCode::OK); + assert_eq!(response.body.as_ref(), b"pong"); + } +} diff --git a/knot2/crates/knot-runtime/src/lib.rs b/knot2/crates/knot-runtime/src/lib.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-runtime/src/lib.rs @@ -0,0 +1,85 @@ +mod clock; +mod dns; +mod entropy; +mod http; +mod signer; + +pub use clock::{Clock, ManualClock, SystemClock, UnixMicros}; +pub use dns::{DnsFuture, DnsTxtResolver, FakeDns, SystemDns}; +pub use entropy::{Entropy, OsEntropy, SeededEntropy}; +pub use http::{ + ByteStream, FakeHttp, HttpFuture, HttpLimits, HttpRequest, HttpResponse, HttpTransport, + NetworkError, ReqwestHttp, StreamFuture, StreamedResponse, is_blocked_ip, +}; +pub use signer::{ + K256Signer, MAX_SCALAR_ATTEMPTS, PublicKeyBytes, Signature, SignatureScheme, Signer, + SignerError, verify, +}; + +#[cfg(test)] +mod contract { + use super::*; + + fn clock_contract(clock: &dyn Clock) { + let first = clock.now_unix_micros(); + let second = clock.now_unix_micros(); + assert!(second >= first); + } + + #[test] + fn every_clock_is_non_decreasing() { + clock_contract(&SystemClock); + clock_contract(&ManualClock::new(UnixMicros::new(1_000))); + } + + fn entropy_contract(entropy: &dyn Entropy) { + let mut buffer = [0u8; 16]; + entropy.fill(&mut buffer); + assert!(buffer.iter().any(|byte| *byte != 0)); + entropy.fill(&mut []); + let _ = entropy.next_u64(); + } + + #[test] + fn every_entropy_produces_output() { + entropy_contract(&OsEntropy); + entropy_contract(&SeededEntropy::new(1)); + } + + fn signer_contract(signer: &dyn Signer) { + let signature = signer.sign(b"the message"); + assert!(verify(&signer.public_key(), b"the message", &signature)); + assert!(!verify( + &signer.public_key(), + b"another message", + &signature + )); + } + + #[test] + fn the_seeded_signer_is_the_test_double() { + signer_contract(&K256Signer::generate(&SeededEntropy::new(3))); + } + + fn ok_response() -> HttpResponse { + HttpResponse { + status: ::http::StatusCode::OK, + headers: ::http::HeaderMap::new(), + body: bytes::Bytes::from_static(b"ok"), + } + } + + #[test] + fn http_transport_surfaces_ok_and_typed_error() { + let url = url::Url::parse("https://oyster.cafe/").unwrap(); + let okay = FakeHttp::new(|_| Ok(ok_response())); + let response = + futures::executor::block_on(okay.execute(HttpRequest::get(url.clone()))).unwrap(); + assert_eq!(response.body.as_ref(), b"ok"); + + let failing = FakeHttp::new(|_| Err(NetworkError::Timeout("slow".to_string()))); + let error = + futures::executor::block_on(failing.execute(HttpRequest::get(url))).unwrap_err(); + assert!(matches!(error, NetworkError::Timeout(_))); + } +} diff --git a/knot2/crates/knot-runtime/src/signer.rs b/knot2/crates/knot-runtime/src/signer.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-runtime/src/signer.rs @@ -0,0 +1,141 @@ +use k256::ecdsa::signature::{Signer as _, Verifier as _}; +use k256::ecdsa::{Signature as K256Signature, SigningKey, VerifyingKey}; + +use crate::Entropy; + +pub const MAX_SCALAR_ATTEMPTS: usize = 64; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Signature(Vec); + +impl Signature { + pub fn from_bytes(bytes: Vec) -> Self { + Self(bytes) + } + + pub fn as_bytes(&self) -> &[u8] { + &self.0 + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PublicKeyBytes(Vec); + +impl PublicKeyBytes { + pub fn from_bytes(bytes: Vec) -> Self { + Self(bytes) + } + + pub fn as_bytes(&self) -> &[u8] { + &self.0 + } +} + +#[derive(Debug, thiserror::Error)] +pub enum SignerError { + #[error("invalid signing key bytes")] + InvalidKey, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SignatureScheme { + Secp256k1, + P256, +} + +pub trait Signer: Send + Sync + 'static { + fn sign(&self, message: &[u8]) -> Signature; + fn public_key(&self) -> PublicKeyBytes; + fn scheme(&self) -> SignatureScheme; +} + +pub struct K256Signer { + key: SigningKey, +} + +impl K256Signer { + pub fn from_slice(bytes: &[u8]) -> Result { + SigningKey::from_slice(bytes) + .map(|key| Self { key }) + .map_err(|_| SignerError::InvalidKey) + } + + // peak dice rolling + pub fn generate(entropy: &dyn Entropy) -> Self { + std::iter::repeat_with(|| { + let mut bytes = [0u8; 32]; + entropy.fill(&mut bytes); + SigningKey::from_slice(&bytes).ok() + }) + .take(MAX_SCALAR_ATTEMPTS) + .flatten() + .next() + .map(|key| Self { key }) + .unwrap_or_else(|| { + panic!( + "entropy failed to yield valid secp256k1 scalar in {MAX_SCALAR_ATTEMPTS} attempts" + ) + }) + } +} + +impl Signer for K256Signer { + fn sign(&self, message: &[u8]) -> Signature { + let signature: K256Signature = self.key.sign(message); + Signature(signature.to_bytes().to_vec()) + } + + fn public_key(&self) -> PublicKeyBytes { + let point = self.key.verifying_key().to_encoded_point(true); + PublicKeyBytes(point.as_bytes().to_vec()) + } + + fn scheme(&self) -> SignatureScheme { + SignatureScheme::Secp256k1 + } +} + +pub fn verify(public_key: &PublicKeyBytes, message: &[u8], signature: &Signature) -> bool { + let Ok(verifying_key) = VerifyingKey::from_sec1_bytes(public_key.as_bytes()) else { + return false; + }; + let Ok(parsed) = K256Signature::from_slice(signature.as_bytes()) else { + return false; + }; + verifying_key.verify(message, &parsed).is_ok() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::SeededEntropy; + + #[test] + fn generate_is_deterministic_from_seed() { + let one = K256Signer::generate(&SeededEntropy::new(99)); + let two = K256Signer::generate(&SeededEntropy::new(99)); + assert_eq!(one.public_key(), two.public_key()); + } + + struct BrokenEntropy; + + impl crate::Entropy for BrokenEntropy { + fn next_u64(&self) -> u64 { + 0 + } + + fn fill(&self, buffer: &mut [u8]) { + buffer.fill(0); + } + + fn derive(&self, _label: u64) -> Box { + Box::new(BrokenEntropy) + } + } + + #[test] + #[should_panic(expected = "entropy failed to yield valid secp256k1 scalar")] + fn broken_entropy_fails_stop_instead_of_spinning() { + let _ = K256Signer::generate(&BrokenEntropy); + } +} diff --git a/knot2/crates/knot-secrets/src/lib.rs b/knot2/crates/knot-secrets/src/lib.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-secrets/src/lib.rs @@ -0,0 +1,771 @@ +use std::collections::BTreeMap; +use std::fmt; +use std::path::{Path, PathBuf}; +use std::sync::{Mutex, MutexGuard, RwLock}; + +use aes_gcm::aead::Aead; +use aes_gcm::{Aes256Gcm, KeyInit}; +use base64::Engine; +use base64::engine::general_purpose::STANDARD; +use hkdf::Hkdf; +use knot_runtime::{Entropy, K256Signer, MAX_SCALAR_ATTEMPTS, PublicKeyBytes, Signer}; +use knot_types::KnotId; +use serde::{Deserialize, Serialize}; +use sha2::Sha256; +use zeroize::{Zeroize, ZeroizeOnDrop, Zeroizing}; + +// why oh why didn't I just call this knot.sealed-key-store.v1. the +// wind changed and now we're stuck with this +const HKDF_INFO: &[u8] = b"knot2.sealed-key-store.v1"; +const NONCE_LEN: usize = 12; +const SCALAR_LEN: usize = 32; +const VAULT_VERSION: u32 = 1; +const MIN_MASTER_KEY_LEN: usize = 32; + +#[derive(Debug, thiserror::Error)] +pub enum SecretsError { + #[error("sealed key store {path}: {source}")] + Io { + path: PathBuf, + #[source] + source: std::io::Error, + }, + #[error( + "sealed key store couldn't be decrypted. Master key may be wrong or file may be corrupt." + )] + Decrypt, + #[error("sealed key store is malformed: {0}")] + Malformed(String), + #[error("no signing key sealed for {0}")] + Missing(String), + #[error("signing key is already sealed for {0}")] + Occupied(String), + #[error("{len}-byte master key is shorter than the {MIN_MASTER_KEY_LEN}-byte minimum")] + WeakMasterKey { len: usize }, +} + +#[derive(Clone, Zeroize, ZeroizeOnDrop)] +pub struct MasterKey(Vec); + +impl MasterKey { + pub fn new(bytes: impl Into>) -> Result { + let bytes = bytes.into(); + if bytes.len() < MIN_MASTER_KEY_LEN { + return Err(SecretsError::WeakMasterKey { len: bytes.len() }); + } + Ok(Self(bytes)) + } + + pub fn as_bytes(&self) -> &[u8] { + &self.0 + } +} + +impl fmt::Debug for MasterKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("MasterKey()") + } +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize)] +#[serde(transparent)] +pub struct SealedKeyId(String); + +impl SealedKeyId { + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl From<&KnotId> for SealedKeyId { + fn from(did: &KnotId) -> Self { + Self(did.as_str().to_string()) + } +} + +impl<'de> serde::Deserialize<'de> for SealedKeyId { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + KnotId::new(raw) + .map(|did| Self::from(&did)) + .map_err(serde::de::Error::custom) + } +} + +#[derive(Clone, Zeroize, ZeroizeOnDrop)] +struct SecretScalar([u8; SCALAR_LEN]); + +impl SecretScalar { + fn generate(entropy: &dyn Entropy) -> Self { + std::iter::repeat_with(|| { + let mut bytes = [0u8; SCALAR_LEN]; + entropy.fill(&mut bytes); + k256::ecdsa::SigningKey::from_slice(&bytes) + .is_ok() + .then_some(bytes) + }) + .take(MAX_SCALAR_ATTEMPTS) + .flatten() + .next() + .map(Self) + .unwrap_or_else(|| { + panic!( + "entropy failed to yield valid secp256k1 scalar in {MAX_SCALAR_ATTEMPTS} attempts" + ) + }) + } + + fn signer(&self) -> K256Signer { + K256Signer::from_slice(&self.0).expect("stored scalar is always a valid signing key") + } +} + +#[derive(Zeroize, ZeroizeOnDrop)] +struct VaultKey([u8; 32]); + +impl VaultKey { + fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } +} + +pub struct KeyMaterial { + scalar: SecretScalar, +} + +impl KeyMaterial { + pub fn signer(&self) -> K256Signer { + self.scalar.signer() + } + + pub fn public_key(&self) -> PublicKeyBytes { + self.scalar.signer().public_key() + } +} + +#[derive(Zeroize, ZeroizeOnDrop, Serialize, Deserialize)] +#[serde(transparent)] +struct EncodedSecret(String); + +impl EncodedSecret { + fn new(value: String) -> Self { + Self(value) + } +} + +#[derive(Serialize, Deserialize)] +struct VaultFile { + version: u32, + entries: BTreeMap, +} + +pub struct SealedStore { + path: PathBuf, + enc_key: VaultKey, + entropy: Box, + entries: RwLock>, + persist_lock: Mutex<()>, +} + +impl SealedStore { + pub fn open( + path: impl Into, + master_key: &MasterKey, + entropy: Box, + ) -> Result { + let path = path.into(); + knot_resource::clear_staging(&path); + sweep_pre_knot_tmp_vaults(&path); + let enc_key = derive_enc_key(master_key); + let entries = match std::fs::read(&path) { + Ok(sealed) => decode_vault(&unseal(&enc_key, &sealed)?)?, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => BTreeMap::new(), + Err(source) => { + return Err(SecretsError::Io { + path: path.clone(), + source, + }); + } + }; + Ok(Self { + path, + enc_key, + entropy, + entries: RwLock::new(entries), + persist_lock: Mutex::new(()), + }) + } + + pub fn signer(&self, id: impl Into) -> Result { + let id = id.into(); + self.entries + .read() + .expect("sealed store lock") + .get(&id) + .map(SecretScalar::signer) + .ok_or(SecretsError::Missing(id.0)) + } + + pub fn public_key(&self, id: impl Into) -> Result { + let id = id.into(); + self.entries + .read() + .expect("sealed store lock") + .get(&id) + .map(|scalar| scalar.signer().public_key()) + .ok_or(SecretsError::Missing(id.0)) + } + + pub fn generate(&self) -> KeyMaterial { + KeyMaterial { + scalar: SecretScalar::generate(&*self.entropy), + } + } + + pub fn store( + &self, + id: impl Into, + material: &KeyMaterial, + ) -> Result<(), SecretsError> { + let id = id.into(); + let guard = self.persist_guard(); + let mut staged = self.staged(); + if staged.contains_key(&id) { + return Err(SecretsError::Occupied(id.0)); + } + staged.insert(id, material.scalar.clone()); + self.commit_locked(&guard, staged) + } + + pub fn ensure(&self, id: impl Into) -> Result { + let id = id.into(); + let guard = self.persist_guard(); + if let Some(public) = self + .entries + .read() + .expect("sealed store lock") + .get(&id) + .map(|scalar| scalar.signer().public_key()) + { + return Ok(public); + } + let material = self.generate(); + let public = material.public_key(); + let mut staged = self.staged(); + staged.insert(id, material.scalar.clone()); + self.commit_locked(&guard, staged)?; + Ok(public) + } + + pub fn len(&self) -> usize { + self.entries.read().expect("sealed store lock").len() + } + + pub fn is_empty(&self) -> bool { + self.entries.read().expect("sealed store lock").is_empty() + } + + pub fn remove(&self, id: impl Into) -> Result { + let id = id.into(); + let guard = self.persist_guard(); + let mut staged = self.staged(); + if staged.remove(&id).is_none() { + return Ok(false); + } + self.commit_locked(&guard, staged)?; + Ok(true) + } + + fn persist_guard(&self) -> MutexGuard<'_, ()> { + self.persist_lock + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + fn staged(&self) -> BTreeMap { + self.entries.read().expect("sealed store lock").clone() + } + + fn commit_locked( + &self, + _guard: &MutexGuard<'_, ()>, + staged: BTreeMap, + ) -> Result<(), SecretsError> { + let plaintext = encode_vault(&staged); + let sealed = seal(&self.enc_key, &plaintext, &*self.entropy); + atomic_write(&self.path, &sealed)?; + *self.entries.write().expect("sealed store lock") = staged; + Ok(()) + } +} + +fn derive_enc_key(master_key: &MasterKey) -> VaultKey { + let hk = Hkdf::::new(None, master_key.as_bytes()); + let mut okm = VaultKey([0u8; 32]); + hk.expand(HKDF_INFO, &mut okm.0) + .expect("32 bytes is valid HKDF-SHA256 output length"); + okm +} + +fn seal(enc_key: &VaultKey, plaintext: &[u8], entropy: &dyn Entropy) -> Vec { + let cipher = Aes256Gcm::new_from_slice(enc_key.as_bytes()).expect("32-byte AES-256-GCM key"); + let mut nonce_bytes = [0u8; NONCE_LEN]; + entropy.fill(&mut nonce_bytes); + let ciphertext = cipher + .encrypt(&nonce_bytes.into(), plaintext) + .expect("AES-256-GCM encryption doesn't fail on valid inputs"); + [&nonce_bytes[..], &ciphertext[..]].concat() +} + +fn unseal(enc_key: &VaultKey, sealed: &[u8]) -> Result>, SecretsError> { + if sealed.len() < NONCE_LEN { + return Err(SecretsError::Decrypt); + } + let (nonce_bytes, ciphertext) = sealed.split_at(NONCE_LEN); + let nonce: [u8; NONCE_LEN] = nonce_bytes + .try_into() + .expect("split_at(NONCE_LEN) yields exactly NONCE_LEN bytes"); + let cipher = Aes256Gcm::new_from_slice(enc_key.as_bytes()).expect("32-byte AES-256-GCM key"); + cipher + .decrypt(&nonce.into(), ciphertext) + .map(Zeroizing::new) + .map_err(|_| SecretsError::Decrypt) +} + +fn encode_vault(entries: &BTreeMap) -> Zeroizing> { + let file = VaultFile { + version: VAULT_VERSION, + entries: entries + .iter() + .map(|(id, scalar)| { + ( + id.clone(), + EncodedSecret::new(STANDARD.encode(scalar.0.as_slice())), + ) + }) + .collect(), + }; + Zeroizing::new(serde_json::to_vec(&file).expect("vault always serializes")) +} + +fn decode_vault(plaintext: &[u8]) -> Result, SecretsError> { + let VaultFile { version, entries } = serde_json::from_slice(plaintext) + .map_err(|error| SecretsError::Malformed(error.to_string()))?; + if version != VAULT_VERSION { + return Err(SecretsError::Malformed(format!( + "unsupported vault version {version}" + ))); + } + entries + .into_iter() + .map(|(id, mut encoded)| { + let encoded = Zeroizing::new(std::mem::take(&mut encoded.0)); + let bytes = Zeroizing::new( + STANDARD + .decode(encoded.as_bytes()) + .map_err(|error| SecretsError::Malformed(error.to_string()))?, + ); + let scalar: [u8; SCALAR_LEN] = bytes.as_slice().try_into().map_err(|_| { + SecretsError::Malformed(format!("scalar for {} isn't 32 bytes", id.as_str())) + })?; + k256::ecdsa::SigningKey::from_slice(&scalar).map_err(|_| { + SecretsError::Malformed(format!("scalar for {} isn't a valid key", id.as_str())) + })?; + Ok((id, SecretScalar(scalar))) + }) + .collect() +} + +fn effective_parent(path: &Path) -> &Path { + path.parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")) +} + +fn sweep_pre_knot_tmp_vaults(path: &Path) { + let Some(stem) = path.file_name().and_then(|name| name.to_str()) else { + return; + }; + let Ok(listing) = std::fs::read_dir(effective_parent(path)) else { + return; + }; + let prefix = format!(".{stem}."); + listing + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with(&prefix) && name.ends_with(".tmp")) + }) + .for_each(|path| { + let _ = std::fs::remove_file(path); + }); +} + +fn atomic_write(path: &Path, bytes: &[u8]) -> Result<(), SecretsError> { + let parent = effective_parent(path); + std::fs::create_dir_all(parent).map_err(|source| SecretsError::Io { + path: path.to_path_buf(), + source, + })?; + knot_resource::atomic_write_bytes(path, bytes, knot_resource::FileMode::Private) + .map_err(Into::into) +} + +impl From for SecretsError { + fn from(error: knot_resource::FsError) -> Self { + SecretsError::Io { + path: error.path, + source: error.source, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use knot_runtime::{SeededEntropy, Signer, verify}; + + fn entropy(seed: u64) -> Box { + Box::new(SeededEntropy::new(seed)) + } + + fn master() -> MasterKey { + MasterKey::new([7u8; 32]).unwrap() + } + + fn kid(value: &str) -> KnotId { + KnotId::new(value).unwrap() + } + + fn store_at(path: &Path, seed: u64) -> SealedStore { + SealedStore::open(path, &master(), entropy(seed)).unwrap() + } + + fn store(seed: u64) -> (tempfile::TempDir, SealedStore) { + let dir = tempfile::tempdir().unwrap(); + let store = store_at(&dir.path().join("keys.sealed"), seed); + (dir, store) + } + + #[test] + fn an_absent_file_opens_as_an_empty_store() { + let (_dir, store) = store(1); + assert!(matches!( + store.signer(&kid("did:web:nel.pet")), + Err(SecretsError::Missing(_)) + )); + } + + #[test] + fn a_sealed_key_survives_a_reopen() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("keys.sealed"); + let public = store_at(&path, 1).ensure(&kid("did:web:nel.pet")).unwrap(); + let reopened = store_at(&path, 2); + assert_eq!( + reopened.public_key(&kid("did:web:nel.pet")).unwrap(), + public + ); + } + + #[test] + fn ensure_is_idempotent() { + let (_dir, store) = store(1); + let first = store.ensure(&kid("did:plc:squid")).unwrap(); + let second = store.ensure(&kid("did:plc:squid")).unwrap(); + assert_eq!(first, second); + } + + #[test] + fn a_generated_key_signs_verifiably_and_round_trips_through_storage() { + let (_dir, store) = store(5); + let material = store.generate(); + let public = material.public_key(); + store.store(&kid("did:plc:limpet"), &material).unwrap(); + + let signer = store.signer(&kid("did:plc:limpet")).unwrap(); + let signature = signer.sign(b"a meta-repo cob change"); + assert!(verify(&public, b"a meta-repo cob change", &signature)); + assert_eq!(signer.public_key(), public); + } + + #[test] + fn storing_over_a_sealed_key_is_refused() { + let (_dir, store) = store(1); + let original = store.ensure(&kid("did:plc:limpet")).unwrap(); + let intruder = store.generate(); + assert!(matches!( + store.store(&kid("did:plc:limpet"), &intruder), + Err(SecretsError::Occupied(_)) + )); + assert_eq!( + store.public_key(&kid("did:plc:limpet")).unwrap(), + original, + "refused overwrite must leave the sealed key untouched" + ); + } + + #[test] + fn master_key_length_is_enforced_at_construction() { + [(31usize, false), (32usize, true)] + .iter() + .for_each(|&(len, accepted)| { + let result = MasterKey::new(vec![7u8; len]); + assert_eq!(result.is_ok(), accepted, "{len}-byte master key acceptance"); + assert!( + accepted + || matches!(result, Err(SecretsError::WeakMasterKey { len: reported }) if reported == len), + "a refused master key reports its short length" + ); + }); + } + + #[test] + fn the_master_key_debug_redacts_its_bytes() { + let rendered = format!("{:?}", MasterKey::new([7u8; 32]).unwrap()); + assert_eq!(rendered, "MasterKey()"); + assert!(!rendered.contains('7')); + } + + #[test] + fn a_wrong_master_key_fails_to_decrypt() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("k"); + store_at(&path, 1).ensure(&kid("did:web:nel.pet")).unwrap(); + assert!(matches!( + SealedStore::open(&path, &MasterKey::new([9u8; 32]).unwrap(), entropy(1)), + Err(SecretsError::Decrypt) + )); + } + + #[test] + fn a_removed_key_is_gone_after_reopen() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("k"); + let store = store_at(&path, 1); + store.ensure(&kid("did:plc:squid")).unwrap(); + store.ensure(&kid("did:plc:clam")).unwrap(); + assert!(store.remove(&kid("did:plc:squid")).unwrap()); + assert!(!store.remove(&kid("did:plc:squid")).unwrap()); + + let reopened = store_at(&path, 1); + assert!(reopened.signer(&kid("did:plc:squid")).is_err()); + assert!(reopened.signer(&kid("did:plc:clam")).is_ok()); + } + + #[test] + fn concurrent_writers_never_lose_a_sealed_key_or_error() { + use std::sync::Arc; + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("keys.sealed"); + let store = Arc::new( + SealedStore::open(&path, &master(), Box::new(knot_runtime::OsEntropy)).unwrap(), + ); + let dids: Vec = (0..64).map(|i| kid(&format!("did:plc:race{i}"))).collect(); + + std::thread::scope(|scope| { + dids.chunks(8).for_each(|chunk| { + let store = Arc::clone(&store); + let chunk = chunk.to_vec(); + scope.spawn(move || { + chunk.iter().for_each(|did| { + store.ensure(did).expect("concurrent seal mustn't error"); + }); + }); + }); + }); + + let reopened = SealedStore::open(&path, &master(), entropy(99)).unwrap(); + let missing: Vec<&KnotId> = dids + .iter() + .filter(|did| reopened.signer(*did).is_err()) + .collect(); + assert!( + missing.is_empty(), + "keys acknowledged in memory were lost from sealed file on disk: {missing:?}" + ); + } + + #[test] + fn concurrent_ensures_of_one_did_agree_on_a_single_key() { + use std::sync::Arc; + + let dir = tempfile::tempdir().unwrap(); + let store = Arc::new( + SealedStore::open( + dir.path().join("keys.sealed"), + &master(), + Box::new(knot_runtime::OsEntropy), + ) + .unwrap(), + ); + let contested = kid("did:plc:whelk"); + + let publics: Vec = std::thread::scope(|scope| { + let handles: Vec<_> = (0..8) + .map(|_| { + let store = Arc::clone(&store); + let did = contested.clone(); + scope.spawn(move || store.ensure(&did).unwrap()) + }) + .collect(); + handles + .into_iter() + .map(|handle| handle.join().unwrap()) + .collect() + }); + + assert!( + publics.windows(2).all(|pair| pair[0] == pair[1]), + "every racing ensure must acknowledge same sealed key" + ); + assert_eq!(store.public_key(&contested).unwrap(), publics[0]); + } + + #[test] + fn the_sealed_file_does_not_contain_raw_scalars() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("k"); + let store = store_at(&path, 1); + let material = store.generate(); + let scalar = material.scalar.0; + store.store(&kid("did:plc:squid"), &material).unwrap(); + let sealed = std::fs::read(&path).unwrap(); + assert!( + !sealed.windows(SCALAR_LEN).any(|window| window == scalar), + "plaintext scalar must never appear in the sealed file" + ); + } + + #[cfg(unix)] + fn set_mode(path: &Path, mode: u32) { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode)).unwrap(); + } + + #[cfg(unix)] + fn make_read_only(dir: &Path) -> bool { + set_mode(dir, 0o555); + let enforced = std::fs::write(dir.join("probe"), b"probe").is_err(); + if !enforced { + set_mode(dir, 0o755); + eprintln!("skipping permission fault injection, this user bypasses read-only modes"); + } + enforced + } + + #[cfg(unix)] + #[test] + fn a_failed_persist_acknowledges_no_key() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("keys.sealed"); + let store = store_at(&path, 1); + store.ensure(&kid("did:plc:squid")).unwrap(); + + if !make_read_only(dir.path()) { + return; + } + let material = store.generate(); + assert!(matches!( + store.store(&kid("did:plc:clam"), &material), + Err(SecretsError::Io { .. }) + )); + assert!( + matches!( + store.signer(&kid("did:plc:clam")), + Err(SecretsError::Missing(_)) + ), + "key whose persist failed mustn't be served from memory" + ); + assert!(matches!( + store.ensure(&kid("did:plc:clam")), + Err(SecretsError::Io { .. }) + )); + set_mode(dir.path(), 0o755); + + let public = store.ensure(&kid("did:plc:clam")).unwrap(); + let reopened = store_at(&path, 2); + assert_eq!(reopened.public_key(&kid("did:plc:clam")).unwrap(), public); + assert!(reopened.signer(&kid("did:plc:squid")).is_ok()); + } + + #[cfg(unix)] + #[test] + fn a_failed_removal_keeps_the_key_served_and_sealed() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("keys.sealed"); + let store = store_at(&path, 1); + store.ensure(&kid("did:plc:squid")).unwrap(); + + if !make_read_only(dir.path()) { + return; + } + assert!(matches!( + store.remove(&kid("did:plc:squid")), + Err(SecretsError::Io { .. }) + )); + assert!( + store.signer(&kid("did:plc:squid")).is_ok(), + "removal that failed to persist must leave the key in service" + ); + set_mode(dir.path(), 0o755); + + let reopened = store_at(&path, 2); + assert!(reopened.signer(&kid("did:plc:squid")).is_ok()); + } + + struct BrokenEntropy; + + impl Entropy for BrokenEntropy { + fn next_u64(&self) -> u64 { + 0 + } + + fn fill(&self, buffer: &mut [u8]) { + buffer.fill(0); + } + + fn derive(&self, _label: u64) -> Box { + Box::new(BrokenEntropy) + } + } + + #[test] + #[should_panic(expected = "entropy failed to yield valid secp256k1 scalar")] + fn broken_entropy_fails_stop_instead_of_spinning() { + let _ = SecretScalar::generate(&BrokenEntropy); + } + + #[test] + fn stale_temp_files_are_swept_on_open() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("keys.sealed"); + let stale = dir.path().join(".keys.sealed.knot-tmp.4242.7"); + let in_flight = dir.path().join(".keys.sealed.knot-tmp.4243.0"); + let pre_rename = dir.path().join(".keys.sealed.4242.7.tmp"); + std::fs::write(&stale, b"abandoned by a crashed run").unwrap(); + std::fs::write(&in_flight, b"another process is sealing right now").unwrap(); + std::fs::write(&pre_rename, b"abandoned before the staging rename").unwrap(); + let long_ago = std::time::SystemTime::now() - std::time::Duration::from_secs(7 * 3600); + std::fs::File::options() + .write(true) + .open(&stale) + .unwrap() + .set_times(std::fs::FileTimes::new().set_modified(long_ago)) + .unwrap(); + + store_at(&path, 1); + assert!(!stale.exists(), "abandoned temp file must be swept at open"); + assert!( + in_flight.exists(), + "the sweep at open must keep the staging file a second process is filling" + ); + assert!( + !pre_rename.exists(), + "a vault sealed by an older build still leaves temps this build has to reclaim" + ); + } +} diff --git a/knot2/crates/knot-server/src/allocator.rs b/knot2/crates/knot-server/src/allocator.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-server/src/allocator.rs @@ -0,0 +1,148 @@ +use std::time::Duration; + +use knot_resource::DecayMs; +use tokio_util::sync::CancellationToken; + +const SAMPLE_INTERVAL: Duration = Duration::from_secs(2); +const BACKGROUND_THREAD: &[u8] = b"background_thread\0"; +const NARENAS: &[u8] = b"arenas.narenas\0"; +const DIRTY_DECAY_NEW_ARENAS: &[u8] = b"arenas.dirty_decay_ms\0"; + +#[derive(Clone, Copy)] +struct ArenaIndex(u32); + +impl ArenaIndex { + fn write_decay(self, decay: DecayMs) -> bool { + let key = format!("arena.{}.dirty_decay_ms\0", self.0); + unsafe { tikv_jemalloc_ctl::raw::write(key.as_bytes(), decay.ms()) }.is_ok() + } +} + +#[derive(Clone, Copy)] +struct DecayCoverage { + reached: usize, + skipped: usize, +} + +impl DecayCoverage { + const EMPTY: Self = Self { + reached: 0, + skipped: 0, + }; + + fn record(self, reached: bool) -> Self { + match reached { + true => Self { + reached: self.reached + 1, + ..self + }, + false => Self { + skipped: self.skipped + 1, + ..self + }, + } + } +} + +fn write_or_warn(name: &[u8], value: T, control: &str) { + if let Err(error) = unsafe { tikv_jemalloc_ctl::raw::write(name, value) } { + tracing::warn!(%error, control, "jemalloc control unavailable"); + } +} + +fn apply_decay(decay: DecayMs) -> tikv_jemalloc_ctl::Result { + unsafe { tikv_jemalloc_ctl::raw::write(DIRTY_DECAY_NEW_ARENAS, decay.ms())? }; + let narenas: u32 = unsafe { tikv_jemalloc_ctl::raw::read(NARENAS)? }; + Ok((0..narenas) + .map(ArenaIndex) + .map(|arena| arena.write_decay(decay)) + .fold(DecayCoverage::EMPTY, DecayCoverage::record)) +} + +pub fn install() { + match apply_decay(knot_resource::target_decay()) { + Ok(coverage) => tracing::info!( + arenas_reached = coverage.reached, + arenas_skipped = coverage.skipped, + "jemalloc dirty_decay applied to reachable arenas" + ), + Err(error) => tracing::warn!(%error, "jemalloc dirty_decay governor unavailable"), + } + write_or_warn(BACKGROUND_THREAD, true, "background_thread"); + let background: bool = + unsafe { tikv_jemalloc_ctl::raw::read(b"opt.background_thread\0") }.unwrap_or(false); + let retain: bool = unsafe { tikv_jemalloc_ctl::raw::read(b"opt.retain\0") }.unwrap_or(true); + let dirty_decay_ms: isize = + unsafe { tikv_jemalloc_ctl::raw::read(b"arena.0.dirty_decay_ms\0") }.unwrap_or(-1); + tracing::info!( + background_thread = background, + retain, + dirty_decay_ms, + "jemalloc page-return configured" + ); +} + +pub async fn govern_decay(shutdown: CancellationToken) { + let mut ticker = tokio::time::interval(SAMPLE_INTERVAL); + let mut applied = knot_resource::target_decay(); + loop { + tokio::select! { + () = shutdown.cancelled() => return, + _ = ticker.tick() => { + let target = knot_resource::target_decay(); + if knot_resource::decay_warrants_apply(applied, target) { + match apply_decay(target) { + Ok(coverage) => tracing::info!( + target_ms = target.ms(), + arenas_reached = coverage.reached, + arenas_skipped = coverage.skipped, + "jemalloc dirty_decay retuned" + ), + Err(error) => tracing::warn!(%error, "jemalloc dirty_decay retune failed"), + } + applied = target; + } + if knot_resource::cache_shed_warranted() + && let Some(freed) = knot_cache::reclaim_largest() + { + tracing::warn!( + freed_bytes = freed.get(), + "shedding largest cache under memory pressure" + ); + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn apply_decay_reaches_reachable_arenas_and_persists() { + let narenas: u32 = + unsafe { tikv_jemalloc_ctl::raw::read(NARENAS) }.expect("arenas.narenas is readable"); + assert!(narenas >= 1, "a live jemalloc has at least one arena"); + + let coverage = apply_decay(knot_resource::target_decay()) + .expect("new-arena default write and narenas read succeed"); + assert!( + coverage.reached >= 1, + "arena 0 is always initialized, so the sweep reaches at least one arena" + ); + assert_eq!( + coverage.reached + coverage.skipped, + narenas as usize, + "every arena index is accounted as reached or skipped" + ); + + unsafe { + tikv_jemalloc_ctl::raw::write(b"arena.0.dirty_decay_ms\0", 5_000isize) + .expect("per-arena dirty_decay_ms is writable"); + let back: isize = tikv_jemalloc_ctl::raw::read(b"arena.0.dirty_decay_ms\0") + .expect("per-arena dirty_decay_ms is readable"); + assert_eq!(back, 5_000, "a per-arena write reads back"); + } + } +} diff --git a/knot2/crates/knot-server/src/homepage.html b/knot2/crates/knot-server/src/homepage.html new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-server/src/homepage.html @@ -0,0 +1,21 @@ + + + + + + knot + + + +

Baby's first Knot

+

This is a Knot server: a git host on a network of federated servers that make up Tangled.

+

+ Operator! Please customize this page! It's your time to shine! Something unique? Something creative? + We made it easy using the homepage.path in the knot config. +

+ + diff --git a/knot2/crates/knot-server/src/main.rs b/knot2/crates/knot-server/src/main.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-server/src/main.rs @@ -0,0 +1,704 @@ +mod allocator; + +#[global_allocator] +static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +#[allow(non_upper_case_globals)] +#[unsafe(export_name = "_rjem_malloc_conf")] +pub static malloc_conf: &[u8] = + b"background_thread:true,retain:false,dirty_decay_ms:0,muzzy_decay_ms:0\0"; + +use std::collections::BTreeSet; +use std::num::{NonZeroU32, NonZeroU64}; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use tokio_util::sync::CancellationToken; + +use anyhow::Context; +use axum::Json; +use axum::response::Html; +use axum::routing::get; +use base64::Engine; +use knot_atproto::Atproto; +use knot_config::HomepageSource; +use knot_index::Index; +use knot_runtime::{Clock, HttpTransport, OsEntropy, ReqwestHttp, SystemClock}; +use knot_secrets::{MasterKey, SealedStore}; +use knot_types::{ActorId, AuthorName, BranchName, CiLogsAddr, Email, KnotHostname, ObjectCount}; +use knot_xrpc::XrpcState; +use tower_http::services::ServeFile; + +const MAINTENANCE_SHUTDOWN_DRAIN: Duration = Duration::from_secs(30); +const EDGE_SHUTDOWN_DRAIN: Duration = Duration::from_secs(40); + +const DEFAULT_HOMEPAGE: &str = include_str!("homepage.html"); + +struct IndexRepos(Arc); + +impl knot_maintenance::RepoSource for IndexRepos { + fn repos(&self) -> Vec { + self.0.hosted_repos() + } + + fn ready_repos(&self) -> Option> { + match self.0.coverage().registry { + knot_index::Coverage::Ready => Some(self.0.hosted_repos()), + knot_index::Coverage::Warming => None, + } + } +} + +struct AtprotoHandleResolver { + atproto: Arc>, +} + +impl knot_pack::HandleResolver for AtprotoHandleResolver { + fn resolve( + &self, + handle: knot_types::Handle, + ) -> std::pin::Pin< + Box> + Send + '_>, + > { + Box::pin(async move { self.atproto.resolve_handle_to_did(&handle).await.ok() }) + } +} + +fn init_tracing() { + let filter = tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")); + tracing_subscriber::fmt() + .with_env_filter(filter) + .with_writer(std::io::stderr) + .init(); +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + #[cfg(target_os = "linux")] + rustix::process::set_dumpable_behavior(rustix::process::DumpableBehavior::NotDumpable) + .context("disable core dumps and ptrace attachment")?; + + if std::env::args().nth(1).as_deref() == Some("config-template") { + print!("{}", knot_config::template()); + return Ok(()); + } + + init_tracing(); + + tracing::info!("!"); + tracing::info!("!"); + tracing::info!("!"); + tracing::info!("> If knot1 was so good then why isn't there a... ( ˶°ㅁ°)"); + tracing::info!("..."); + tracing::info!("Welcome to knot2!"); + tracing::info!("This code was made with love."); + tracing::info!("Hachapuri is sho tasty, definitely worth a try. Better than pizza tbh."); + tracing::info!("!"); + tracing::info!("!"); + tracing::info!("!"); + + let config_path = std::env::args().nth(1).map(PathBuf::from); + let config = knot_config::load(config_path.as_deref()).context("load configuration")?; + config + .verify_environment() + .context("verify runtime environment")?; + + let resources = knot_resource::init(knot_resource::Ceilings { + max_threads: match config.resources.max_threads { + 0 => None, + n => Some(knot_resource::ThreadCount::new(n as usize)), + }, + max_memory: match config.resources.max_memory_bytes { + 0 => None, + n => Some(knot_resource::MemoryBudget::new(n)), + }, + }); + tracing::info!( + threads = resources.threads.get(), + memory_bytes = resources.memory.map(knot_resource::MemoryBudget::get), + memory_source = ?resources.memory_source, + memory_high_bytes = resources + .memory_high_bytes + .map(knot_resource::MemoryHighBytes::get), + "resource governor initialized" + ); + allocator::install(); + + // I made the hostname stay aa string in the config, + // so that confique can layer env-over-file. + // Here's where it becomes a real type, + // and the DID + the service url stack on this. + let hostname = KnotHostname::new(config.server.hostname.clone()) + .context("server.hostname isn't a valid knot hostname")?; + let knot_did = hostname.knot_did(); + let object_format = config.object_format().context("parse git.object_format")?; + let default_branch = + BranchName::new(config.repo.default_branch.as_str()).context("parse default branch")?; + let layout = knot_git::Layout::new(&config.repo.scan_path) + .with_default_branch(default_branch) + .with_object_format(object_format) + .reserving_meta(&knot_did) + .context("reserve meta-repo path")?; + + let swept = knot_pack::sweep_incoming(&config.repo.scan_path); + if swept > 0 { + tracing::info!(swept, "swept abandoned receive staging directories"); + } + + layout + .bootstrap_meta(&knot_did) + .context("bootstrap meta-repo")?; + let meta_path = layout + .meta_path(&knot_did) + .context("resolve meta-repo path")?; + + let index = Arc::new(Index::new(meta_path.clone(), layout.clone())); + index.rebuild().context("rebuild index from meta-repo")?; + tracing::info!(coverage = ?index.coverage(), "index ready"); + + let warm = Arc::clone(&index); + tokio::task::spawn_blocking(move || warm.warm_collaborators()); + + let http = ReqwestHttp::new(config.http_limits()).context("build outbound HTTP client")?; + let git_http: Arc = Arc::new( + ReqwestHttp::new(config.fork_http_limits()).context("build outbound git fetch client")?, + ); + let atproto = Arc::new(Atproto::new( + http, + SystemClock, + knot_did.clone(), + knot_atproto::PlcDirectory::new(config.atproto.plc_directory.clone()) + .context("atproto.plc_directory isn't a valid PLC base URL")?, + )); + let admins: BTreeSet<_> = config.server.admins.iter().cloned().collect(); + let admission = config.acl.admission; + let service_owner = config + .server + .admins + .first() + .cloned() + .context("at least one admin is configured")?; + + let master_key = MasterKey::new( + base64::engine::general_purpose::STANDARD + .decode( + std::env::var(&config.secrets.master_key_env) + .context("read master key from environment")? + .trim(), + ) + .context("decode master key as base64")?, + ) + .context("master key from environment")?; + let secrets = Arc::new( + SealedStore::open( + &config.secrets.sealed_key_file, + &master_key, + Box::new(OsEntropy), + ) + .context("open sealed key store")?, + ); + let knot_signing_key = secrets + .ensure(&knot_did) + .context("seal knot's own signing key")?; + let knot_actor = ActorId::from_secp256k1(knot_signing_key.as_bytes()); + let appview_endpoint = config.server.appview_endpoint.clone(); + let knot_service_url = knot_types::KnotServiceUrl::new(format!("https://{hostname}")) + .context("server.hostname doesn't form a valid knot service URL")?; + let did_document = + knot_atproto::knot_did_document(&knot_did, &knot_signing_key, &knot_service_url); + + let http_addr = config.server.listen_addr; + let listen_limits = knot_edge::ListenLimits::new( + knot_edge::HeaderTimeout::from_millis( + NonZeroU64::new(config.server.listen_header_timeout_ms) + .context("server.listen_header_timeout_ms must be greater than zero")?, + ), + knot_edge::IdleTimeout::from_millis( + NonZeroU64::new(config.server.listen_idle_timeout_ms) + .context("server.listen_idle_timeout_ms must be greater than zero")?, + ), + NonZeroU32::new(config.server.listen_max_connections) + .context("server.listen_max_connections must be greater than zero")?, + ); + // A header name that doesn't parse will never match, + // `effective_peer` falls back to socket, + // and every request in the world shares + // the proxy's address + its one ratelimit bucket. + // So... better to refuse to start. + let trusted_proxy_header = config + .xrpc + .trusted_proxy_header + .as_deref() + .map(|header| axum::http::HeaderName::from_bytes(header.as_bytes())) + .transpose() + .context("xrpc.trusted_proxy_header isn't a valid HTTP header name")?; + let edge_guards = knot_edge::EdgeGuards::new( + knot_edge::RequestsPerSecond::new( + NonZeroU32::new(config.server.listen_rate_limit_per_second) + .context("server.listen_rate_limit_per_second must be greater than zero")?, + ), + knot_edge::BurstSize::new( + NonZeroU32::new(config.server.listen_rate_limit_burst) + .context("server.listen_rate_limit_burst must be greater than zero")?, + ), + knot_edge::MaxInflightRequests::new( + NonZeroU32::new(config.server.listen_max_inflight_requests) + .context("server.listen_max_inflight_requests must be greater than zero")?, + ), + knot_edge::RequestTimeout::from_millis( + NonZeroU64::new(config.server.listen_request_timeout_ms) + .context("server.listen_request_timeout_ms must be greater than zero")?, + ), + knot_edge::BodyInactivityTimeout::from_millis( + NonZeroU64::new(config.server.listen_body_timeout_ms) + .context("server.listen_body_timeout_ms must be greater than zero")?, + ), + knot_edge::WriteRequestTimeout::from_millis( + NonZeroU64::new(config.server.listen_write_request_timeout_ms) + .context("server.listen_write_request_timeout_ms must be greater than zero")?, + ), + trusted_proxy_header.clone(), + ); + let tls_setup = build_tls_setup(&config, &hostname).context("assemble TLS configuration")?; + if config.tls.http3 && tls_setup.is_none() { + tracing::warn!( + "tls.http3 is set without any TLS certificate, so HTTP/3 won't start. Configure a static cert or ACME to serve h3." + ); + } + if tls_setup.is_none() && config.xrpc.trusted_proxy_header.is_none() { + tracing::warn!( + "running plaintext behind a reverse proxy without xrpc.trusted_proxy_header. Per-IP rate limiting will key on the proxy socket address, throttling all clients as one. Set xrpc.trusted_proxy_header to the header your proxy appends." + ); + } + if config.tls.acme_enabled && http_addr.port() != 443 { + tracing::warn!( + listen_port = http_addr.port(), + "ACME validation over TLS-ALPN-01 needs the certificate authority to reach this host on TCP 443. Map 443 to the listen port if it differs." + ); + } + if config.tls.acme_enabled && config.tls.acme_staging { + tracing::warn!( + "ACME is using the Let's Encrypt staging directory. Its certificates aren't browser-trusted. Unset tls.acme_staging for real certificates." + ); + } + let ssh_addr = config.server.ssh_listen_addr; + let ssh_max_pack_bytes = config.server.ssh_max_pack_bytes as usize; + let pack_limits = knot_pack::PackLimits { + max_objects: ObjectCount::from(config.pack.max_objects), + max_total_bytes: knot_pack::MaxTotalBytes::new(config.pack.max_total_bytes), + ..knot_pack::PackLimits::default() + }; + knot_pack::init_selection_limits(knot_pack::SelectionLimits { + max_objects: ObjectCount::from(config.pack.selection_max_objects), + time_budget: Duration::from_secs(config.pack.selection_time_budget_secs), + }); + let host_key = knot_ssh::load_or_create_host_key(&config.server.ssh_host_key_file) + .context("load or create SSH host key")?; + + let xrpc_limits = knot_xrpc::LimitConfig { + rate: Some(knot_xrpc::RateLimit { + burst: knot_xrpc::Burst::new(config.xrpc.preauth_burst), + refill: knot_xrpc::RefillMicros::new( + config.xrpc.preauth_refill_ms.saturating_mul(1_000), + ), + }), + per_peer_inflight: Some(knot_xrpc::PerPeerInflight::new( + config.xrpc.per_peer_inflight as usize, + )), + global_inflight: Some(knot_xrpc::GlobalInflight::new( + config.xrpc.global_inflight as usize, + )), + }; + let byte_limits = knot_xrpc::ByteLimits { + body: knot_xrpc::BodyLimit::new(config.xrpc.max_body_bytes as usize), + patch: knot_xrpc::PatchLimit::new(config.xrpc.max_patch_bytes as usize), + patch_decompressed: knot_xrpc::PatchDecompressedLimit::new( + config.xrpc.max_patch_decompressed_bytes, + ), + response: knot_xrpc::ResponseLimit::new(config.xrpc.max_response_bytes as usize), + archive: knot_xrpc::ArchiveLimit::new(config.xrpc.max_archive_bytes), + fork_pack: knot_xrpc::ForkPackLimit::new(config.xrpc.fork_max_pack_bytes), + pack: knot_xrpc::MaxWireBytes::new(ssh_max_pack_bytes), + }; + let budgets = knot_xrpc::Budgets { + tree_last_commit: knot_xrpc::TreeReadBudget::new(knot_xrpc::ReadBudget::Within( + Duration::from_millis(config.xrpc.tree_last_commit_budget_ms), + )), + blob_last_commit: knot_xrpc::BlobReadBudget::new(knot_xrpc::ReadBudget::Within( + Duration::from_millis(config.xrpc.blob_last_commit_budget_ms), + )), + languages: knot_xrpc::LanguagesReadBudget::new(knot_xrpc::ReadBudget::Within( + Duration::from_millis(config.xrpc.languages_budget_ms), + )), + languages_push: knot_xrpc::LanguagesPushBudget::new(Duration::from_millis( + config.xrpc.languages_push_budget_ms, + )), + }; + let committer = knot_xrpc::Committer { + name: AuthorName::new(config.git.user_name.clone()), + email: Email::new(config.git.user_email.clone()), + }; + let reservations = Arc::new(knot_xrpc::Reservations::new( + knot_xrpc::ReservationTtl::new(config.xrpc.reservation_ttl_secs as i64), + knot_xrpc::PerActorQuota::new(config.xrpc.per_actor_reservations as usize), + knot_xrpc::GlobalQuota::new(config.xrpc.max_pending_reservations as usize), + )); + let replay_bounds = knot_events::ReplayBounds::new( + knot_events::ReplayEvents::new(config.xrpc.events_replay_buffer as usize) + .context("xrpc.events_replay_buffer must be greater than zero")?, + knot_events::ReplayBytes::new(config.xrpc.events_replay_bytes as usize) + .context("xrpc.events_replay_bytes must be greater than zero")?, + ); + let events = Arc::new(knot_events::EventLog::new(SystemClock, replay_bounds)); + let subscriber_gate = Arc::new(knot_events::SubscriberGate::new( + knot_events::GlobalSubscriberLimit::new(config.xrpc.events_max_subscribers as usize), + knot_events::PerPeerSubscriberLimit::new(config.xrpc.events_max_per_peer as usize), + )); + + let maintenance_enabled = config.maintenance.enabled; + let maintenance_options = knot_maintenance::Options::from_config(&config.maintenance); + let maintenance_interval = Duration::from_secs(config.maintenance.interval_secs); + let maintenance_large_push = + knot_maintenance::PushBytes::new(config.maintenance.large_push_bytes); + + let lfs_handle = config + .lfs + .store_path + .as_ref() + .map(|path| { + knot_lfs::LfsHandle::open( + knot_lfs::LfsStorePath::new(path), + knot_lfs::LfsSize::new(config.lfs.max_object_bytes), + knot_lfs::FreeSpaceFloor::new(config.lfs.free_space_floor_bytes), + ) + .inspect(|_| { + tracing::info!(store = %path.display(), "git-lfs capability enabled"); + }) + }) + .transpose() + .context("open LFS object store")?; + let lfs_max_ssh_transfers = config.lfs.max_ssh_transfers as usize; + let lfs_max_http_downloads = config.lfs.max_http_downloads as usize; + let lfs_gc_grace = knot_maintenance::lfs_grace( + knot_maintenance::GcGrace::from_secs(config.lfs.gc_grace_secs), + knot_maintenance::ReflogRetention::from_secs(config.maintenance.reflog_expire_secs), + ); + let lfs_gc_interval = + knot_maintenance::SweepInterval::new(Duration::from_secs(config.lfs.gc_interval_secs)); + if lfs_handle.is_some() && !maintenance_enabled { + tracing::warn!( + "LFS store configured but maintenance is disabled. Unreferenced LFS objects will accumulate with no garbage collection or orphan sweep." + ); + } + + let pack_cache_config = knot_pack::CacheConfig { + enabled: config.pack_cache.enabled, + ttl: Duration::from_secs(config.pack_cache.ttl_secs), + max_entry_bytes: knot_pack::MaxEntryBytes::new(config.pack_cache.max_entry_bytes as usize), + max_total_bytes: knot_pack::MaxCacheBytes::new(knot_resource::pack_cache_bytes( + config.pack_cache.max_total_bytes, + ) as usize), + }; + + let ci_logs = config + .ci + .logs_addr + .as_deref() + .map(CiLogsAddr::new) + .transpose() + .context("ci.logs_addr must be host:port")?; + + let homepage = config.homepage.source(); + let catalog = Arc::new( + knot_messages::Catalog::parse(&config.messages).context("parse message templates")?, + ); + + knot_config::init(config); + + let (maintenance_handle, maintenance_shutdown, maintenance_task) = if maintenance_enabled { + let (scheduler, handle) = knot_maintenance::Scheduler::new( + layout.clone(), + Arc::new(IndexRepos(Arc::clone(&index))), + SystemClock, + maintenance_options, + maintenance_interval, + maintenance_large_push, + ); + let scheduler = match &lfs_handle { + Some(lfs) => { + scheduler.with_lfs_gc(Arc::clone(&lfs.store), lfs_gc_grace, lfs_gc_interval) + } + None => scheduler, + }; + let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); + let task = tokio::spawn(scheduler.run(shutdown_rx)); + tracing::info!( + interval_secs = maintenance_interval.as_secs(), + "maintenance scheduler running" + ); + (handle, Some(shutdown_tx), Some(task)) + } else { + (knot_maintenance::MaintenanceHandle::disabled(), None, None) + }; + + let slots = knot_resource::Slots::for_machine(); + + let ssh_base = knot_ssh::SshState::new( + layout.clone(), + Arc::clone(&index), + Arc::clone(&atproto), + knot_actor, + Arc::clone(&events), + hostname.clone(), + appview_endpoint.clone(), + admins.clone(), + admission, + byte_limits.pack, + budgets.languages_push, + ci_logs.clone(), + ) + .with_maintenance(maintenance_handle.clone()) + .with_limits(pack_limits) + .with_slots(slots.clone()) + .with_catalog(Arc::clone(&catalog)); + let ssh_state = Arc::new(match &lfs_handle { + Some(handle) => ssh_base.with_lfs(handle.clone(), lfs_max_ssh_transfers), + None => ssh_base, + }); + + let xrpc_state = Arc::new(XrpcState { + layout: layout.clone(), + index: Arc::clone(&index), + atproto: Arc::clone(&atproto), + secrets, + entropy: Arc::new(OsEntropy), + ci_logs, + admins, + admission, + knot_did, + knot_hostname: hostname, + meta_path, + knot_service_url, + limiter: Arc::new(knot_xrpc::PreAuthLimiter::with_config(xrpc_limits)), + cob_locks: Arc::new(knot_xrpc::CobLocks::default()), + reservations, + trusted_proxy_header, + committer, + byte_limits, + budgets, + git_http, + pack_limits, + service_owner, + subscriber_gate, + maintenance: maintenance_handle, + appview: appview_endpoint, + slots: slots.clone(), + events, + lfs: lfs_handle.map(|handle| knot_xrpc::LfsWeb::new(handle, lfs_max_http_downloads)), + catalog: Arc::clone(&catalog), + }); + + let resolver: Arc = { + let index = Arc::clone(&index); + Arc::new(move |target: &knot_pack::RepoTarget| match target { + knot_pack::RepoTarget::Did(did) => { + knot_pack::RepoLookup::from_resolved(index.owner_of(did), |_| did.clone()) + } + knot_pack::RepoTarget::OwnerRkey(owner, rkey) => { + knot_pack::RepoLookup::from_resolved(index.resolve_repo(owner, rkey), |found| found) + } + }) + }; + let receive_advertiser = knot_xrpc::receive_advertiser(Arc::clone(&xrpc_state)); + let handle_resolver: Arc = Arc::new(AtprotoHandleResolver { + atproto: Arc::clone(&atproto), + }); + let (write_routes, early_data_safe) = knot_pack::edge_routes( + layout, + resolver, + Some(receive_advertiser), + Some(handle_resolver), + slots.pack.clone(), + pack_cache_config, + Arc::clone(&catalog), + xrpc_state.knot_hostname.clone(), + Arc::new(SystemClock), + ); + let base_router = write_routes.merge(knot_xrpc::router(xrpc_state)).route( + "/.well-known/did.json", + get(move || { + let document = did_document.clone(); + async move { Json(document) } + }), + ); + let base_router = match homepage { + HomepageSource::Disabled => base_router, + HomepageSource::Default => base_router.route("/", get(|| async { Html(DEFAULT_HOMEPAGE) })), + HomepageSource::File(path) => base_router.route_service("/", ServeFile::new(path)), + }; + let app = knot_edge::RequiresFullHandshake::new(base_router); + let scheme = if tls_setup.is_some() { "https" } else { "http" }; + let edge_config = knot_edge::EdgeConfig { + http_addr: knot_edge::PublicBind::new(http_addr), + limits: listen_limits, + guards: edge_guards, + tls: tls_setup, + }; + tracing::info!("listening on {scheme}://{http_addr} and ssh://{ssh_addr}"); + + let shutdown = CancellationToken::new(); + tokio::spawn(allocator::govern_decay(shutdown.clone())); + let mut edge_task = tokio::spawn(knot_edge::serve( + edge_config, + app, + early_data_safe, + shutdown.clone(), + )); + let mut ssh_task = { + let shutdown = shutdown.clone(); + tokio::spawn(async move { knot_ssh::serve(ssh_addr, host_key, ssh_state, shutdown).await }) + }; + + let exit = tokio::select! { + result = &mut edge_task => FirstExit::Edge(result), + result = &mut ssh_task => FirstExit::Ssh(result), + () = shutdown_signal() => { + tracing::info!("shutdown signal received"); + FirstExit::Signal + } + }; + shutdown.cancel(); + let drain = async { + match &exit { + FirstExit::Edge(_) => { + let _ = (&mut ssh_task).await; + } + FirstExit::Ssh(_) => { + let _ = (&mut edge_task).await; + } + FirstExit::Signal => { + let _ = (&mut edge_task).await; + let _ = (&mut ssh_task).await; + } + } + }; + if tokio::time::timeout(EDGE_SHUTDOWN_DRAIN, drain) + .await + .is_err() + { + tracing::warn!( + timeout_secs = EDGE_SHUTDOWN_DRAIN.as_secs(), + "aborting edge drain after timeout" + ); + } + if let Some(shutdown) = maintenance_shutdown { + let _ = shutdown.send(true); + } + if let Some(task) = maintenance_task + && tokio::time::timeout(MAINTENANCE_SHUTDOWN_DRAIN, task) + .await + .is_err() + { + tracing::warn!( + timeout_secs = MAINTENANCE_SHUTDOWN_DRAIN.as_secs(), + "aborting maintenance drain after timeout :3" + ); + } + match exit { + FirstExit::Edge(result) => result + .context("edge server task panicked")? + .context("serve edge")?, + FirstExit::Ssh(result) => result + .context("ssh server task panicked")? + .context("serve ssh")?, + FirstExit::Signal => {} + } + Ok(()) +} + +enum FirstExit { + Edge(Result, tokio::task::JoinError>), + Ssh(Result, tokio::task::JoinError>), + Signal, +} + +fn build_tls_setup( + config: &knot_config::KnotConfig, + hostname: &knot_types::KnotHostname, +) -> anyhow::Result> { + let tls = &config.tls; + let source = if tls.acme_enabled { + knot_edge::CertSource::Acme(knot_edge::AcmeParams { + domains: vec![hostname.clone()], + contact: knot_edge::AcmeContact::new( + tls.acme_contact + .clone() + .context("tls.acme_contact is required when ACME is enabled")?, + )?, + cache_dir: knot_edge::AcmeCacheDir::new( + tls.acme_cache_dir + .clone() + .context("tls.acme_cache_dir is required when ACME is enabled")?, + ), + production: !tls.acme_staging, + }) + } else { + match (&tls.cert_path, &tls.key_path) { + (Some(cert_path), Some(key_path)) => { + knot_edge::CertSource::Static(knot_edge::StaticCertPaths { + cert_path: knot_edge::CertChainPath::new(cert_path.clone()), + key_path: knot_edge::PrivateKeyPath::new(key_path.clone()), + }) + } + _ => return Ok(None), + } + }; + + let internal = match tls.mtls_enabled { + true => Some(knot_edge::InternalTls { + addr: knot_edge::InternalBind::new(config.server.internal_listen_addr), + client_ca_path: knot_edge::ClientCaPath::new( + tls.mtls_client_ca_path + .clone() + .context("tls.mtls_client_ca_path is required when mTLS is enabled")?, + ), + spki_pin: knot_edge::SpkiPin::from_base64( + tls.mtls_admin_spki_pin + .as_deref() + .context("tls.mtls_admin_spki_pin is required when mTLS is enabled")?, + ) + .context("parse tls.mtls_admin_spki_pin")?, + }), + false => None, + }; + + Ok(Some(knot_edge::TlsSetup { + source, + http3: tls.http3, + internal, + })) +} + +async fn shutdown_signal() { + let interrupt = async { + let _ = tokio::signal::ctrl_c().await; + }; + #[cfg(unix)] + let terminate = async { + match tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) { + Ok(mut stream) => { + stream.recv().await; + } + Err(_) => std::future::pending::<()>().await, + } + }; + #[cfg(not(unix))] + let terminate = std::future::pending::<()>(); + + tokio::select! { + () = interrupt => {} + () = terminate => {} + } +} diff --git a/knot2/crates/knot-server/tests/invariants.rs b/knot2/crates/knot-server/tests/invariants.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-server/tests/invariants.rs @@ -0,0 +1,247 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; + +use walkdir::WalkDir; + +fn knot_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .ancestors() + .nth(2) + .expect("knot root is two levels above server crate") + .to_path_buf() +} + +fn workspace_root() -> PathBuf { + knot_root() + .parent() + .expect("workspace root is the parent of the knot root") + .to_path_buf() +} + +fn crate_src_files() -> impl Iterator { + WalkDir::new(knot_root().join("crates")) + .into_iter() + .filter_map(Result::ok) + .filter(|entry| entry.file_type().is_file()) + .map(|entry| entry.into_path()) + .filter(|path| path.extension().is_some_and(|ext| ext == "rs")) + .filter(|path| { + path.components() + .any(|component| component.as_os_str() == "src") + }) +} + +#[test] +fn no_subprocess_spawning_in_src() { + let offenders: Vec = crate_src_files() + .filter(|path| { + std::fs::read_to_string(path) + .map(|text| text.contains("process::Command")) + .unwrap_or(false) + }) + .map(|path| path.display().to_string()) + .collect(); + assert!( + offenders.is_empty(), + "design pillar: no subprocesses for git or anything else, but these src files spawn one: {offenders:?}" + ); +} + +fn lock_field<'a>(block: &'a str, key: &str) -> Option<&'a str> { + block.lines().find_map(|line| { + line.strip_prefix(key)? + .strip_prefix(" = \"")? + .strip_suffix('"') + }) +} + +type PackageKey<'a> = (&'a str, &'a str); +type LockGraph<'a> = BTreeMap, Vec>>; + +fn lock_graph(lock: &str) -> LockGraph<'_> { + lock.split("[[package]]") + .skip(1) + .filter_map(|block| { + let name = lock_field(block, "name")?; + let version = lock_field(block, "version")?; + Some(((name, version), lock_dependencies(block))) + }) + .collect() +} + +fn lock_dependencies(block: &str) -> Vec> { + block + .split_once("dependencies = [") + .and_then(|(_, rest)| rest.split(']').next()) + .into_iter() + .flat_map(str::lines) + .filter_map(|line| line.trim().strip_prefix('"')) + .filter_map(|entry| entry.split('"').next()) + .map(|entry| { + entry + .split_once(' ') + .map_or(DepRef::Name(entry), |(name, rest)| { + DepRef::Exact(name, rest.split(' ').next().unwrap_or(rest)) + }) + }) + .collect() +} + +#[derive(Debug, Clone, Copy)] +enum DepRef<'a> { + Name(&'a str), + Exact(&'a str, &'a str), +} + +impl<'a> DepRef<'a> { + fn name(&self) -> &'a str { + match self { + DepRef::Name(name) | DepRef::Exact(name, _) => name, + } + } +} + +fn reachable_from<'a>( + graph: &LockGraph<'a>, + by_name: &BTreeMap<&'a str, Vec>>, + package: PackageKey<'a>, + seen: &mut BTreeSet>, +) { + if seen.insert(package) { + graph.get(&package).into_iter().flatten().for_each(|dep| { + let (name, version) = match dep { + DepRef::Name(name) => (*name, None), + DepRef::Exact(name, version) => (*name, Some(*version)), + }; + by_name + .get(name) + .into_iter() + .flatten() + .filter(|(_, held)| version.is_none_or(|version| *held == version)) + .for_each(|key| reachable_from(graph, by_name, *key, seen)); + }); + } +} + +#[test] +fn no_durable_state_or_native_git_crates() { + let lock = std::fs::read_to_string(workspace_root().join("Cargo.lock")) + .expect("workspace Cargo.lock is readable"); + let graph = lock_graph(&lock); + let by_name: BTreeMap<&str, Vec>> = + graph.keys().fold(BTreeMap::new(), |mut names, key| { + names.entry(key.0).or_default().push(*key); + names + }); + let server = by_name + .get("knot-server") + .and_then(|keys| keys.first()) + .copied() + .expect("the lockfile parse must find knot-server"); + assert!( + graph.get(&server).is_some_and(|deps| !deps.is_empty()), + "the lockfile parse must find knot-server's dependency list" + ); + + let mut reachable = BTreeSet::new(); + reachable_from(&graph, &by_name, server, &mut reachable); + assert!( + reachable.iter().any(|(name, _)| *name == "gix"), + "the reachability walk must reach the git engine, so an empty walk is a broken parse" + ); + + let banned = [ + "rusqlite", + "sqlx", + "sled", + "fjall", + "redb", + "git2", + "libgit2-sys", + ]; + let present: Vec<&str> = banned + .into_iter() + .filter(|name| reachable.iter().any(|(held, _)| held == name)) + .collect(); + assert!( + present.is_empty(), + "design pillar: no durable state but git and all git work through gix, but the server's dependency graph includes: {present:?}" + ); +} + +fn dependents_of<'a>(graph: &LockGraph<'a>, package: &str) -> Vec<&'a str> { + graph + .iter() + .filter(|((name, _), _)| *name != package) + .filter(|(_, deps)| deps.iter().any(|dep| dep.name() == package)) + .map(|((name, _), _)| *name) + .collect() +} + +#[test] +fn nothing_depends_on_the_offline_migration_tool() { + let lock = std::fs::read_to_string(workspace_root().join("Cargo.lock")) + .expect("workspace Cargo.lock is readable"); + let graph = lock_graph(&lock); + assert!( + !dependents_of(&graph, "knot-types").is_empty(), + "the shared newtypes have dependents, so an empty answer here is a broken parse" + ); + + let dependents = dependents_of(&graph, "knot-migrate"); + assert!( + dependents.is_empty(), + "knot-migrate is an offline one-shot tool whose rusqlite dependency must never reach the server, but it is depended on by: {dependents:?}" + ); +} + +#[test] +fn the_shared_limit_defaults_match_the_config_defaults() { + use confique::{Config, Layer}; + use knot_xrpc::{Budgets, ByteLimits, ReadBudget}; + + fn ms(budget: ReadBudget) -> u64 { + match budget { + ReadBudget::Within(within) => within.as_millis() as u64, + ReadBudget::Unbounded => u64::MAX, + } + } + + let xrpc = ::Layer::default_values(); + let server = ::Layer::default_values(); + let bytes = ByteLimits::default(); + let budgets = Budgets::default(); + let push_ms = budgets.languages_push.get().as_millis() as u64; + + let configured = [ + ("body", xrpc.max_body_bytes), + ("patch", xrpc.max_patch_bytes), + ("patch_decompressed", xrpc.max_patch_decompressed_bytes), + ("response", xrpc.max_response_bytes), + ("archive", xrpc.max_archive_bytes), + ("fork_pack", xrpc.fork_max_pack_bytes), + ("pack", server.ssh_max_pack_bytes), + ("tree_last_commit", xrpc.tree_last_commit_budget_ms), + ("blob_last_commit", xrpc.blob_last_commit_budget_ms), + ("languages", xrpc.languages_budget_ms), + ("languages_push", xrpc.languages_push_budget_ms), + ] + .map(|(name, value)| (name, value.expect("every limit has a config default"))); + let shared = [ + ("body", bytes.body.get() as u64), + ("patch", bytes.patch.get() as u64), + ("patch_decompressed", bytes.patch_decompressed.get()), + ("response", bytes.response.get() as u64), + ("archive", bytes.archive.get()), + ("fork_pack", bytes.fork_pack.get()), + ("pack", bytes.pack.get() as u64), + ("tree_last_commit", ms(budgets.tree_last_commit.get())), + ("blob_last_commit", ms(budgets.blob_last_commit.get())), + ("languages", ms(budgets.languages.get())), + ("languages_push", push_ms), + ]; + assert_eq!( + configured, shared, + "the config defaults and the in-code defaults must match. update ByteLimits::default and Budgets::default alongside the config defaults" + ); +} diff --git a/knot2/crates/knot-sim/src/harness.rs b/knot2/crates/knot-sim/src/harness.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-sim/src/harness.rs @@ -0,0 +1,861 @@ +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use axum::{Json, Router, routing::get}; +use base64::Engine; +use base64::engine::general_purpose::STANDARD; +use bytes::Bytes; +use serde_json::json; +use tempfile::TempDir; +use url::Url; + +use knot_atproto::{Atproto, knot_did_document}; +use knot_cob::{CobHome, CobStore}; +use knot_cobs::{Grant, Registration, RegistryChange}; +use knot_events::{EventLog, GlobalSubscriberLimit, PerPeerSubscriberLimit, SubscriberGate}; +use knot_git::{ + EntryKind, Identity, Layout, NewCommit, RefUpdate, Repo, StagedAction, StagedChange, +}; +use knot_index::{Index, Resolved}; +use knot_runtime::{ + Clock, Entropy, FakeHttp, HttpRequest, HttpResponse, K256Signer, ManualClock, NetworkError, + PublicKeyBytes, SeededEntropy, Signer, UnixMicros, +}; +use knot_secrets::{MasterKey, SealedStore}; +use knot_types::{ + AccountDid, AdmissionPolicy, AuthorName, BranchName, Email, KnotHostname, KnotId, + KnotServiceUrl, Oid, OwnerDid, RefName, RepoDid, RepoName, RepoRkey, UnixSeconds, +}; +use knot_xrpc::{ + BlobReadBudget, BodyLimit, Budgets, ByteLimits, CobLocks, Committer, GlobalInflight, + GlobalQuota, LanguagesPushBudget, LanguagesReadBudget, LimitConfig, MaxWireBytes, + PerActorQuota, PerPeerInflight, PreAuthLimiter, ReadBudget, ReservationTtl, Reservations, + ResponseLimit, TreeReadBudget, XrpcState, +}; + +use crate::realdata::{ + RealActors, SAMPLE_TAG, SAMPLED_REPOS, STRANGER_POOL, SUBJECT_POOL, admin_signer, owner_signer, +}; +use crate::trace::{RepoCollaborators, RoundNumber, Snapshot}; +use crate::workload::Rng; + +const EMPTY_TREE: &str = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"; +const KNOT_HOST: &str = "knot.nel.pet"; +const ADMIN_HOST: &str = "admin.nel.pet"; +const STRANGER_SEED_BASE: u64 = 1_000; +const PDS_ENDPOINT: &str = "https://pds.nel.pet"; +const START_MICROS: u64 = 1_000_000_000; +const NO_REFLOG_EXPIRY_FLOOR_SECS: i64 = i64::MAX / 4; +const NO_PRUNE_EXPIRY_GRACE_SECS: u64 = (i64::MAX / 4) as u64; +const MAINTAIN_ATTEMPTS: usize = 4; + +pub(crate) const SUBJECT_DIDS: [&str; 8] = [ + "did:plc:limpet", + "did:plc:whelk", + "did:plc:mussel", + "did:plc:conch", + "did:plc:scallop", + "did:plc:cuttle", + "did:plc:periwinkle", + "did:plc:nautilus", +]; + +pub(crate) type Responder = + Box Result + Send + Sync>; + +pub(crate) struct Actor { + pub host: KnotHostname, + pub did: AccountDid, + pub signer: K256Signer, +} + +#[derive(Default)] +pub(crate) struct Faults { + dropped: Mutex>, +} + +impl Faults { + fn is_dropped(&self, host: &KnotHostname) -> bool { + self.dropped.lock().expect("faults lock").contains(host) + } + + pub(crate) fn drop_host(&self, host: &KnotHostname) { + self.dropped + .lock() + .expect("faults lock") + .insert(host.clone()); + } + + pub(crate) fn clear_host(&self, host: &KnotHostname) { + self.dropped.lock().expect("faults lock").remove(host); + } +} + +pub(crate) struct Harness { + _dir: Option, + pub clock: Arc, + pub faults: Arc, + pub layout: Layout, + pub index: Arc, + pub knot_aud: KnotId, + router: Router, + pub admin: Actor, + pub strangers: Vec, + pub subjects: Vec, + pub seed_repo: RepoDid, + options: knot_maintenance::Options, + maintain_lock: Mutex<()>, +} + +impl Harness { + pub(crate) fn build(seed: u64, stranger_pool: usize) -> Self { + let dir = tempfile::tempdir().expect("sim tempdir"); + let knot = KnotId::new(format!("did:web:{KNOT_HOST}")).expect("knot did"); + let layout = Layout::new(dir.path().join("repos")) + .reserving_meta(&knot) + .expect("reserve meta"); + layout.bootstrap_meta(&knot).expect("bootstrap meta"); + let meta_path = layout.meta_path(&knot).expect("meta path"); + + let entropy = Arc::new(SeededEntropy::new(seed ^ 0x5eed_0050)); + let secrets = Arc::new( + SealedStore::open( + dir.path().join("keys.sealed"), + &MasterKey::new([7u8; 32]).unwrap(), + Box::new(SeededEntropy::new(seed ^ 0x5ec0)), + ) + .expect("sealed store"), + ); + let knot_pubkey = secrets.ensure(&knot).expect("seal knot key"); + let knot_service_url = + KnotServiceUrl::new(format!("https://{KNOT_HOST}")).expect("knot service url"); + + let admin = actor(ADMIN_HOST, 1); + let strangers: Vec = (0..stranger_pool) + .map(|index| { + let host = format!("stranger{index}.nel.pet"); + actor(&host, STRANGER_SEED_BASE + index as u64) + }) + .collect(); + let subjects: Vec = SUBJECT_DIDS + .iter() + .map(|did| AccountDid::new(*did).expect("subject did")) + .collect(); + + let seed_repo = RepoDid::new("did:plc:squid").expect("seed repo did"); + seed_on_disk(&layout, &seed_repo); + register_seed_repo(&meta_path, &knot, &secrets, &admin.did, &seed_repo); + + let index = Arc::new(Index::new(meta_path.clone(), layout.clone())); + index.rebuild().expect("rebuild index"); + index.warm_collaborators(); + + let clock = Arc::new(ManualClock::new(UnixMicros::new(START_MICROS))); + let pubkeys = pubkey_map(&admin, &strangers); + let faults = Arc::new(Faults::default()); + let plc = Url::parse("https://plc.directory/").expect("plc url"); + let responder = build_responder(Arc::clone(&faults), pubkeys, HashMap::new(), plc.clone()); + let knot_aud = knot.clone(); + let did_document = knot_did_document(&knot, &knot_pubkey, &knot_service_url); + let router = assemble_router(StateParts { + layout: layout.clone(), + index: Arc::clone(&index), + responder, + secrets, + entropy: entropy as Arc, + admins: BTreeSet::from([admin.did.clone()]), + admission: AdmissionPolicy::Closed, + knot: knot.clone(), + knot_hostname: KnotHostname::new(KNOT_HOST).unwrap(), + meta_path, + knot_service_url, + service_owner: admin.did.clone(), + clock: Arc::clone(&clock), + plc, + did_document, + }); + let options = maintenance_options(); + + Self { + _dir: Some(dir), + clock, + faults, + layout, + index, + knot_aud, + router, + admin, + strangers, + subjects, + seed_repo, + options, + maintain_lock: Mutex::new(()), + } + } + + pub fn open(target: &Path, seed: u64) -> Result<(Self, RealActors), OpenError> { + let config = + knot_config::load(Some(&target.join("config.toml"))).map_err(OpenError::Config)?; + let hostname = + KnotHostname::new(config.server.hostname.clone()).map_err(|_| OpenError::Hostname)?; + let knot = hostname.knot_did(); + let object_format = config.object_format().ok_or(OpenError::ObjectFormat)?; + let default_branch = BranchName::new(config.repo.default_branch.as_str()) + .map_err(|_| OpenError::DefaultBranch)?; + let admin_did = config + .server + .admins + .first() + .cloned() + .ok_or(OpenError::NoAdmin)?; + let admission = config.acl.admission; + let plc = config.atproto.plc_directory.clone(); + + let master_key_env = config.secrets.master_key_env.clone(); + let master_key = MasterKey::new( + STANDARD + .decode( + std::env::var(&master_key_env) + .map_err(|_| OpenError::MasterKeyEnv(master_key_env.clone()))? + .trim(), + ) + .map_err(|_| OpenError::MasterKeyDecode)?, + )?; + + let scratch = materialize_scratch(target, &knot)?; + let root = scratch.path().to_path_buf(); + + let layout = Layout::new(root.join("repos")) + .with_default_branch(default_branch) + .with_object_format(object_format) + .reserving_meta(&knot)?; + let meta_path = layout.meta_path(&knot)?; + let secrets = Arc::new(SealedStore::open( + root.join("sealed-keys"), + &master_key, + Box::new(SeededEntropy::new(seed ^ 0x5ec0)), + )?); + let knot_pubkey = secrets.ensure(&knot)?; + let knot_service_url = + KnotServiceUrl::new(format!("https://{hostname}")).map_err(|_| OpenError::Hostname)?; + + let index = Arc::new(Index::new(meta_path.clone(), layout.clone())); + index.rebuild()?; + index.warm_collaborators(); + + let mut hosted = index.hosted_repos(); + hosted.sort(); + let rng = Rng::new(seed ^ SAMPLE_TAG); + let owner_repos: Vec<(RepoDid, OwnerDid)> = SAMPLED_REPOS + .sample(&hosted, &rng) + .into_iter() + .filter_map(|repo| match index.owner_of(&repo) { + Resolved::Ready(Some(owner)) => Some((repo, owner)), + _ => None, + }) + .collect(); + let subjects = SUBJECT_POOL.sample(&distinct_members(&index), &rng); + + let admin = Actor { + host: hostname.clone(), + did: admin_did.clone(), + signer: admin_signer(seed), + }; + let strangers: Vec = (0..STRANGER_POOL.count()) + .map(|index| { + actor( + &format!("stranger{index}.nel.pet"), + STRANGER_SEED_BASE + index as u64, + ) + }) + .collect(); + + let mut did_overrides: HashMap = HashMap::new(); + did_overrides.insert(admin_did.clone(), admin.signer.public_key()); + owner_repos.iter().for_each(|(_, owner)| { + did_overrides + .entry(AccountDid::from(owner.clone())) + .or_insert_with(|| owner_signer(seed, owner).public_key()); + }); + + let seed_repo = owner_repos + .first() + .map(|(repo, _)| repo.clone()) + .or_else(|| hosted.first().cloned()) + .ok_or(OpenError::NoRepos)?; + + let clock = Arc::new(ManualClock::new(UnixMicros::new(START_MICROS))); + let faults = Arc::new(Faults::default()); + let responder = build_responder( + Arc::clone(&faults), + pubkey_map(&admin, &strangers), + did_overrides, + plc.clone(), + ); + let entropy = Arc::new(SeededEntropy::new(seed ^ 0x5eed_0050)); + let knot_aud = knot.clone(); + let did_document = knot_did_document(&knot, &knot_pubkey, &knot_service_url); + let router = assemble_router(StateParts { + layout: layout.clone(), + index: Arc::clone(&index), + responder, + secrets, + entropy: entropy as Arc, + admins: BTreeSet::from([admin_did.clone()]), + admission, + knot: knot.clone(), + knot_hostname: hostname, + meta_path, + knot_service_url, + service_owner: admin_did.clone(), + clock: Arc::clone(&clock), + plc, + did_document, + }); + + let harness = Self { + _dir: Some(scratch), + clock, + faults, + layout, + index, + knot_aud, + router, + admin, + strangers, + subjects, + seed_repo, + options: maintenance_options(), + maintain_lock: Mutex::new(()), + }; + Ok((harness, RealActors { owner_repos, seed })) + } + + pub(crate) fn router(&self) -> Router { + self.router.clone() + } + + pub(crate) fn now_seconds(&self) -> UnixSeconds { + UnixSeconds::new((self.clock.now_unix_micros().get() / 1_000_000) as i64) + } + + pub(crate) fn advance(&self, delta: std::time::Duration) { + self.clock.advance(delta); + } + + pub(crate) fn maintain(&self, repo: &RepoDid) -> Result<(), String> { + let _serialized = self.maintain_lock.lock().expect("maintenance lock"); + let now = self.now_seconds(); + let attempt = || -> Result<(), knot_maintenance::MaintError> { + let opened = self + .layout + .open(repo) + .map_err(knot_maintenance::MaintError::from)?; + knot_maintenance::run_repo(&opened, now, &self.options).map(|_| ()) + }; + (1..MAINTAIN_ATTEMPTS) + .fold(attempt(), |result, _| result.or_else(|_| attempt())) + .map_err(|error| error.to_string()) + } + + pub(crate) fn populate(&self, repo: &RepoDid) { + let opened = self.layout.open(repo).expect("open created repo"); + write_history(&opened, repo.as_str()); + } + + pub(crate) fn snapshot(&self, round: RoundNumber, repos: &[RepoDid]) -> Snapshot { + let collaborators = repos + .iter() + .map(|repo| { + let _ = self.index.ensure_collaborators(repo); + RepoCollaborators { + repo: repo.clone(), + subjects: sorted(grant_subjects(self.index.collaborator_entries(repo))), + } + }) + .collect(); + let repo_list = { + let mut repos: Vec = self.index.hosted_repos().to_vec(); + repos.sort(); + repos + }; + Snapshot { + round, + clock_micros: self.clock.now_unix_micros(), + members: sorted(grant_subjects(self.index.member_entries())), + blocked: sorted(grant_subjects(self.index.blocked_entries())), + repos: repo_list, + collaborators, + } + } +} + +fn actor(host: &str, seed: u64) -> Actor { + Actor { + host: KnotHostname::new(host).expect("actor hostname"), + did: AccountDid::new(format!("did:web:{host}")).expect("actor did"), + signer: K256Signer::generate(&SeededEntropy::new(seed)), + } +} + +fn pubkey_map(admin: &Actor, strangers: &[Actor]) -> HashMap { + std::iter::once((admin.host.clone(), admin.signer.public_key())) + .chain( + strangers + .iter() + .map(|actor| (actor.host.clone(), actor.signer.public_key())), + ) + .collect() +} + +fn build_responder( + faults: Arc, + pubkeys: HashMap, + did_overrides: HashMap, + plc: Url, +) -> Responder { + let plc_signer = K256Signer::generate(&SeededEntropy::new(7)); + Box::new(move |request: &HttpRequest| { + if request.method == http::Method::POST { + return Ok(ok_body(Bytes::new())); + } + let not_found = || HttpResponse { + status: http::StatusCode::NOT_FOUND, + headers: http::HeaderMap::new(), + body: Bytes::new(), + }; + let Ok(host) = KnotHostname::new(request.url.host_str().unwrap_or_default()) else { + return Ok(not_found()); + }; + if faults.is_dropped(&host) { + return Err(NetworkError::Timeout( + "identity resolution dropped by simulation".to_string(), + )); + } + let is_plc = request.url.host() == plc.host(); + let requested_did = if is_plc { + request + .url + .path_segments() + .and_then(|mut segments| segments.rfind(|segment| !segment.is_empty())) + .and_then(|segment| AccountDid::new(segment).ok()) + } else { + AccountDid::new(format!("did:web:{host}")).ok() + }; + let Some(requested_did) = requested_did else { + return Ok(not_found()); + }; + if let Some(sec1) = did_overrides.get(&requested_did) { + return Ok(ok_body(did_doc(&requested_did, sec1))); + } + if is_plc { + return Ok(ok_body(did_doc(&requested_did, &plc_signer.public_key()))); + } + match pubkeys.get(&host) { + Some(sec1) => Ok(ok_body(did_doc(&requested_did, sec1))), + None => Ok(not_found()), + } + }) +} + +fn did_doc(did: &AccountDid, sec1: &PublicKeyBytes) -> Bytes { + let did = did.as_str(); + let multikey = knot_types::crypto::multikey(0xe7, sec1.as_bytes()); + Bytes::from( + serde_json::to_vec(&json!({ + "id": did, + "alsoKnownAs": [], + "verificationMethod": [{ + "id": format!("{did}#atproto"), + "type": "Multikey", + "controller": did, + "publicKeyMultibase": multikey + }], + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": PDS_ENDPOINT + }] + })) + .expect("did doc serializes"), + ) +} + +fn ok_body(body: Bytes) -> HttpResponse { + HttpResponse { + status: http::StatusCode::OK, + headers: http::HeaderMap::new(), + body, + } +} + +fn seed_on_disk(layout: &Layout, did: &RepoDid) { + let repo = layout.create(did).expect("create seed repo"); + write_history(&repo, "reef"); +} + +fn write_history(repo: &Repo, marker: &str) { + let identity = Identity { + name: AuthorName::new("nel"), + email: Email::new("nel@oyster.cafe"), + time: UnixSeconds::new(1_700_000_000), + offset_seconds: 0, + }; + let main = RefName::new("refs/heads/main").expect("main ref"); + let first_tree = repo + .write_staged_tree( + Oid::from_hex(EMPTY_TREE).expect("empty tree"), + &[StagedChange { + path: knot_types::RepoPath::new("README.md").unwrap(), + action: StagedAction::Put { + content: format!("# {marker}\n").into_bytes(), + kind: EntryKind::Blob, + }, + }], + ) + .expect("first tree"); + let root = repo + .write_commit(&NewCommit { + tree: first_tree, + parents: Vec::new(), + author: identity.clone(), + committer: identity.clone(), + message: "root".to_string(), + extra_headers: Vec::new(), + }) + .expect("root commit"); + let second_tree = repo + .write_staged_tree( + first_tree, + &[StagedChange { + path: knot_types::RepoPath::new("src/main.rs").unwrap(), + action: StagedAction::Put { + content: b"fn main() {}\n".to_vec(), + kind: EntryKind::Blob, + }, + }], + ) + .expect("second tree"); + let tip = repo + .write_commit(&NewCommit { + tree: second_tree, + parents: vec![root], + author: identity.clone(), + committer: identity, + message: "add main".to_string(), + extra_headers: Vec::new(), + }) + .expect("second commit"); + repo.update_ref(&RefUpdate::Create { + name: main.clone(), + new: tip, + }) + .expect("create main"); + repo.set_head(&main).expect("set head"); +} + +fn register_seed_repo( + meta_path: &std::path::Path, + knot: &KnotId, + secrets: &SealedStore, + owner: &AccountDid, + repo: &RepoDid, +) { + let meta = Repo::open(meta_path).expect("open meta"); + let store = CobStore::new(&meta); + let signer = secrets.signer(knot).expect("knot signer"); + store + .create( + &CobHome::from(knot), + &RegistryChange::Register(Registration { + owner: OwnerDid::new(owner.as_str()).expect("owner did"), + rkey: RepoRkey::new("anemone").expect("rkey"), + name: RepoName::new("anemone").expect("name"), + repo: repo.clone(), + created_at: UnixSeconds::new(1), + }), + &signer, + UnixSeconds::new(1), + ) + .expect("register seed repo"); +} + +fn grant_subjects(resolved: Resolved>) -> Vec { + match resolved { + Resolved::Ready(grants) => grants + .into_iter() + .map(|grant| grant.subject.clone()) + .collect(), + Resolved::Warming => Vec::new(), + } +} + +fn sorted(mut values: Vec) -> Vec { + values.sort(); + values +} + +fn distinct_members(index: &Index) -> Vec { + let mut members = grant_subjects(index.member_entries()); + members.sort(); + members.dedup(); + members +} + +fn materialize_scratch(target: &Path, knot: &KnotId) -> Result { + let base = target + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .map(Path::to_path_buf) + .unwrap_or_else(|| PathBuf::from(".")); + let source_repos = target.join("repos"); + let source_meta = Layout::new(source_repos.clone()).meta_path(knot)?; + let scratch = tempfile::tempdir_in(&base).map_err(OpenError::Scratch)?; + let root = scratch.path(); + let scratch_repos = root.join("repos"); + let scratch_meta = Layout::new(scratch_repos.clone()).meta_path(knot)?; + fs::copy(target.join("sealed-keys"), root.join("sealed-keys")).map_err(OpenError::Scratch)?; + hardlink_tree(&source_repos, &scratch_repos).map_err(OpenError::Scratch)?; + fs::remove_dir_all(&scratch_meta).map_err(OpenError::Scratch)?; + copy_tree(&source_meta, &scratch_meta).map_err(OpenError::Scratch)?; + Ok(scratch) +} + +fn hardlink_tree(src: &Path, dst: &Path) -> std::io::Result<()> { + fs::create_dir_all(dst)?; + fs::read_dir(src)?.try_for_each(|entry| { + let entry = entry?; + let from = entry.path(); + let to = dst.join(entry.file_name()); + if entry.file_type()?.is_dir() { + hardlink_tree(&from, &to) + } else { + fs::hard_link(&from, &to).map(|_| ()) + } + }) +} + +fn copy_tree(src: &Path, dst: &Path) -> std::io::Result<()> { + fs::create_dir_all(dst)?; + fs::read_dir(src)?.try_for_each(|entry| { + let entry = entry?; + let from = entry.path(); + let to = dst.join(entry.file_name()); + if entry.file_type()?.is_dir() { + copy_tree(&from, &to) + } else { + fs::copy(&from, &to).map(|_| ()) + } + }) +} + +fn maintenance_options() -> knot_maintenance::Options { + knot_maintenance::Options { + repack_max_objects: knot_maintenance::ObjectCount::new(1_000_000), + geometric_factor: knot_maintenance::GeometricFactor::full_repack(), + prune_grace: knot_maintenance::PruneGrace::from_secs(NO_PRUNE_EXPIRY_GRACE_SECS), + reflog_floor: knot_maintenance::ReflogRetention::from_secs( + NO_REFLOG_EXPIRY_FLOOR_SECS as u64, + ), + commit_graph: true, + multi_pack_index: true, + bitmap: true, + } +} + +struct StateParts { + layout: Layout, + index: Arc, + responder: Responder, + secrets: Arc, + entropy: Arc, + admins: BTreeSet, + admission: AdmissionPolicy, + knot: KnotId, + knot_hostname: KnotHostname, + meta_path: PathBuf, + knot_service_url: KnotServiceUrl, + service_owner: AccountDid, + clock: Arc, + plc: Url, + did_document: serde_json::Value, +} + +fn assemble_router(parts: StateParts) -> Router { + let StateParts { + layout, + index, + responder, + secrets, + entropy, + admins, + admission, + knot, + knot_hostname, + meta_path, + knot_service_url, + service_owner, + clock, + plc, + did_document, + } = parts; + let atproto = Arc::new(Atproto::new( + FakeHttp::new(responder), + Arc::clone(&clock), + knot.clone(), + knot_atproto::PlcDirectory::new(plc).expect("plc directory"), + )); + let state = Arc::new(XrpcState { + layout: layout.clone(), + index: Arc::clone(&index), + atproto, + secrets, + entropy, + ci_logs: None, + admins, + admission, + knot_did: knot, + knot_hostname, + meta_path, + knot_service_url, + limiter: Arc::new(PreAuthLimiter::with_config(LimitConfig { + rate: None, + per_peer_inflight: Some(PerPeerInflight::new(4096)), + global_inflight: Some(GlobalInflight::new(4096)), + })), + cob_locks: Arc::new(CobLocks::default()), + reservations: Arc::new(Reservations::new( + ReservationTtl::new(1_000_000), + PerActorQuota::new(256), + GlobalQuota::new(256), + )), + trusted_proxy_header: None, + committer: Committer { + name: AuthorName::new("knot"), + email: Email::new("knot@nel.pet"), + }, + byte_limits: ByteLimits { + body: BodyLimit::new(256 * 1024), + response: ResponseLimit::new(8 * 1024 * 1024), + pack: MaxWireBytes::new(1024 * 1024 * 1024), + ..ByteLimits::default() + }, + budgets: Budgets { + tree_last_commit: TreeReadBudget::new(ReadBudget::Unbounded), + blob_last_commit: BlobReadBudget::new(ReadBudget::Unbounded), + languages: LanguagesReadBudget::new(ReadBudget::Unbounded), + languages_push: LanguagesPushBudget::new(Duration::from_secs(120)), + }, + git_http: Arc::new(FakeHttp::new(|_request: &HttpRequest| { + Err(NetworkError::Connect( + "simulation serves no git upstream".to_string(), + )) + })), + pack_limits: knot_pack::PackLimits::default(), + service_owner, + events: Arc::new(EventLog::new( + Arc::clone(&clock), + knot_events::ReplayBounds::new( + knot_events::ReplayEvents::new(4096).expect("replay event maximum is nonzero"), + knot_events::ReplayBytes::new(64 << 20).expect("replay byte maximum is nonzero"), + ), + )), + subscriber_gate: Arc::new(SubscriberGate::new( + GlobalSubscriberLimit::new(256), + PerPeerSubscriberLimit::new(64), + )), + maintenance: knot_maintenance::MaintenanceHandle::disabled(), + appview: knot_types::AppviewEndpoint::new("https://tangled.test").unwrap(), + slots: knot_resource::Slots::testing(8), + lfs: None, + catalog: Arc::new(knot_messages::Catalog::defaults()), + }); + let resolver: Arc = { + let index = Arc::clone(&index); + Arc::new(move |target: &knot_pack::RepoTarget| match target { + knot_pack::RepoTarget::Did(did) => match index.owner_of(did) { + Resolved::Ready(Some(_)) => knot_pack::RepoLookup::Hosted(did.clone()), + Resolved::Ready(None) => knot_pack::RepoLookup::Unhosted, + Resolved::Warming => knot_pack::RepoLookup::Unavailable, + }, + knot_pack::RepoTarget::OwnerRkey(owner, rkey) => { + match index.resolve_repo(owner, rkey) { + Resolved::Ready(Some(found)) => knot_pack::RepoLookup::Hosted(found), + Resolved::Ready(None) => knot_pack::RepoLookup::Unhosted, + Resolved::Warming => knot_pack::RepoLookup::Unavailable, + } + } + }) + }; + knot_pack::router(layout, resolver, Arc::clone(&clock) as Arc) + .merge(knot_xrpc::router(Arc::clone(&state))) + .route( + "/.well-known/did.json", + get(move || { + let document = did_document.clone(); + async move { Json(document) } + }), + ) +} + +#[derive(Debug)] +pub enum OpenError { + MasterKeyEnv(String), + MasterKeyDecode, + NoAdmin, + NoRepos, + Scratch(std::io::Error), + Hostname, + ObjectFormat, + DefaultBranch, + Config(knot_config::LoadError), + Git(knot_git::GitError), + Secrets(knot_secrets::SecretsError), + Index(knot_index::IndexError), +} + +impl std::fmt::Display for OpenError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::MasterKeyEnv(name) => write!(f, "{name} is not set"), + Self::MasterKeyDecode => write!(f, "master key is not valid base64"), + Self::NoAdmin => write!(f, "config lists no admin"), + Self::NoRepos => write!(f, "target hosts no repos to sample"), + Self::Scratch(error) => { + write!(f, "materialize disposable scratch copy of target: {error}") + } + Self::Hostname => write!(f, "config hostname is not a valid knot hostname"), + Self::ObjectFormat => { + write!(f, "config git.object_format is not a valid object format") + } + Self::DefaultBranch => write!(f, "config default branch is not a valid branch name"), + Self::Config(error) => write!(f, "read config: {error}"), + Self::Git(error) => write!(f, "open target repos: {error}"), + Self::Secrets(error) => write!(f, "open sealed key store: {error}"), + Self::Index(error) => write!(f, "rebuild index: {error}"), + } + } +} + +impl std::error::Error for OpenError {} + +impl From for OpenError { + fn from(error: knot_git::GitError) -> Self { + Self::Git(error) + } +} + +impl From for OpenError { + fn from(error: knot_secrets::SecretsError) -> Self { + Self::Secrets(error) + } +} + +impl From for OpenError { + fn from(error: knot_index::IndexError) -> Self { + Self::Index(error) + } +} diff --git a/knot2/crates/knot-sim/src/lib.rs b/knot2/crates/knot-sim/src/lib.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-sim/src/lib.rs @@ -0,0 +1,25 @@ +mod harness; +mod realdata; +pub mod trace; +mod workload; + +pub use harness::OpenError; +pub use realdata::run as run_realdata; +pub use trace::{ + OperationIndex, Outcome, Projection, RepoCollaborators, RoundNumber, Snapshot, Step, Trace, +}; + +use std::sync::Arc; + +use harness::Harness; + +pub async fn run(seed: u64, rounds: u32) -> Trace { + let stranger_pool = (rounds as usize) * 2 + 16; + let harness = Arc::new(Harness::build(seed, stranger_pool)); + let plan = workload::plan(seed, rounds, harness.subjects.len()); + workload::execute(harness, seed, plan).await +} + +pub fn predict(seed: u64, rounds: u32) -> Projection { + workload::predict(seed, rounds) +} diff --git a/knot2/crates/knot-sim/src/main.rs b/knot2/crates/knot-sim/src/main.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-sim/src/main.rs @@ -0,0 +1,49 @@ +#[tokio::main(flavor = "multi_thread", worker_threads = 4)] +async fn main() { + let args: Vec = std::env::args().skip(1).collect(); + let dump = args.iter().any(|arg| arg == "dump"); + let target_flag = args.iter().position(|arg| arg == "--target"); + let target = match target_flag { + Some(position) => match args.get(position + 1) { + Some(path) => Some(path.clone()), + None => { + eprintln!("knot-sim: --target needs a path"); + std::process::exit(1); + } + }, + None => None, + }; + let numbers: Vec = args + .iter() + .enumerate() + .filter(|(index, _)| { + !matches!(target_flag, Some(position) if *index == position || *index == position + 1) + }) + .filter_map(|(_, arg)| arg.parse().ok()) + .collect(); + let seed = numbers.first().copied().unwrap_or(1); + let rounds = numbers.get(1).copied().unwrap_or(12) as u32; + let trace = match target { + Some(path) => match knot_sim::run_realdata(std::path::Path::new(&path), seed, rounds).await + { + Ok(trace) => trace, + Err(error) => { + eprintln!("knot-sim: {error}"); + std::process::exit(1); + } + }, + None => knot_sim::run(seed, rounds).await, + }; + if dump { + println!( + "{}", + serde_json::to_string_pretty(&trace).expect("trace serializes") + ); + } + println!( + "knot-sim seed={seed} rounds={rounds} steps={} snapshots={} digest={:016x}", + trace.steps.len(), + trace.snapshots.len(), + trace.digest() + ); +} diff --git a/knot2/crates/knot-sim/src/realdata.rs b/knot2/crates/knot-sim/src/realdata.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-sim/src/realdata.rs @@ -0,0 +1,536 @@ +use std::path::Path; +use std::sync::Arc; + +use axum::body::Bytes; +use futures::future::join_all; +use futures::stream::StreamExt; +use http::Method; +use serde_json::json; + +use knot_runtime::{K256Signer, SeededEntropy}; +use knot_types::{AccountDid, HttpStatus, KnotHostname, OwnerDid, RepoDid}; + +use crate::harness::Harness; +use crate::trace::{OperationIndex, Outcome, RoundNumber, Snapshot, Step, Trace, fnv1a}; +use crate::workload::{ + Request, Rng, body_digest, drive_kill, enc, encode_body, http_call, jwt_window, mint, + repo_did_of, +}; + +pub(crate) const SAMPLE_TAG: u64 = 0x5a3d_e005; +pub(crate) const SAMPLED_REPOS: SampledRepos = SampledRepos(24); +pub(crate) const SUBJECT_POOL: SubjectPool = SubjectPool(16); +pub(crate) const STRANGER_POOL: StrangerPool = StrangerPool(32); +const ADMIN_SIGNER_TAG: u64 = 0x0ad0_0001; +const DRIVE_TAG: u64 = 0x11c7_0005; + +#[derive(Clone, Copy)] +pub(crate) struct SampledRepos(usize); + +#[derive(Clone, Copy)] +pub(crate) struct SubjectPool(usize); + +#[derive(Clone, Copy)] +pub(crate) struct StrangerPool(usize); + +impl SampledRepos { + pub(crate) fn sample(self, source: &[RepoDid], rng: &Rng) -> Vec { + pick(source, self.0, rng) + } +} + +impl SubjectPool { + pub(crate) fn sample(self, source: &[AccountDid], rng: &Rng) -> Vec { + pick(source, self.0, rng) + } +} + +impl StrangerPool { + pub(crate) fn count(self) -> usize { + self.0 + } +} + +pub struct RealActors { + pub(crate) owner_repos: Vec<(RepoDid, OwnerDid)>, + pub(crate) seed: u64, +} + +pub(crate) fn owner_signer(seed: u64, owner: &OwnerDid) -> K256Signer { + K256Signer::generate(&SeededEntropy::new(seed ^ fnv1a(owner.as_str().as_bytes()))) +} + +pub(crate) fn admin_signer(seed: u64) -> K256Signer { + K256Signer::generate(&SeededEntropy::new(seed ^ ADMIN_SIGNER_TAG)) +} + +fn pick(source: &[T], count: usize, rng: &Rng) -> Vec { + if source.is_empty() { + return Vec::new(); + } + (0..count.min(source.len())) + .scan(Vec::::new(), |taken, _| { + let start = rng.below(source.len() as u64) as usize; + let index = (start..source.len()) + .chain(0..start) + .find(|candidate| !taken.contains(candidate)) + .expect("distinct index exists while count <= len"); + taken.push(index); + Some(source[index].clone()) + }) + .collect() +} + +pub async fn run( + target: &Path, + seed: u64, + rounds: u32, +) -> Result { + let (harness, actors) = Harness::open(target, seed)?; + Ok(drive(Arc::new(harness), actors, rounds).await) +} + +struct OpResult { + step: Step, + created: Option, +} + +enum Exec { + Http { + op: &'static str, + actor: String, + fault: &'static str, + request: Request, + killed: bool, + capture_create: bool, + }, + Maintain { + repo: RepoDid, + }, +} + +async fn drive(harness: Arc, actors: RealActors, rounds: u32) -> Trace { + let seed = actors.seed; + let rng = Rng::new(seed ^ DRIVE_TAG); + let harness_ref = &harness; + let actors_ref = &actors; + let rng_ref = &rng; + let base: Vec = actors + .owner_repos + .iter() + .map(|(repo, _)| repo.clone()) + .collect(); + let (_created, steps, snapshots) = futures::stream::iter(0..rounds) + .fold( + ( + Vec::::new(), + Vec::::new(), + Vec::::new(), + ), + |(created, mut steps, mut snapshots), round_index| { + let harness = Arc::clone(harness_ref); + let base = &base; + async move { + let round = RoundNumber::new(round_index); + let working: Vec = + base.iter().chain(created.iter()).cloned().collect(); + let execs = if round_index % 2 == 0 { + mutate(&harness, actors_ref, &working, rng_ref, round) + } else { + reads(&working, rng_ref) + }; + let dropped = arm_drops(&harness, &execs); + + let tasks = execs + .into_iter() + .enumerate() + .map(|(position, exec)| { + let harness = Arc::clone(&harness); + let index = OperationIndex::new(position as u32); + tokio::spawn(async move { run_op(&harness, round, index, exec).await }) + }) + .collect::>(); + let results: Vec = join_all(tasks) + .await + .into_iter() + .map(|joined| joined.expect("real-data op task mustn't panic")) + .collect(); + dropped + .iter() + .for_each(|host| harness.faults.clear_host(host)); + + let fresh: Vec = results + .iter() + .filter_map(|result| result.created.clone()) + .collect(); + fresh.iter().for_each(|did| harness.populate(did)); + steps.extend(results.into_iter().map(|result| result.step)); + + let mut created = created; + created.extend(fresh); + harness.advance(round_advance(rng_ref)); + let snapshot_repos: Vec = + base.iter().chain(created.iter()).cloned().collect(); + snapshots.push(harness.snapshot(round, &snapshot_repos)); + (created, steps, snapshots) + } + }, + ) + .await; + Trace { + seed, + steps, + snapshots, + } +} + +fn round_advance(rng: &Rng) -> std::time::Duration { + std::time::Duration::from_micros(rng.below(3_000_000)) +} + +fn arm_drops(harness: &Harness, execs: &[Exec]) -> Vec { + let hosts: Vec = execs + .iter() + .filter_map(|exec| match exec { + Exec::Http { + op: "probe", + fault: "drop_identity", + actor, + .. + } => KnotHostname::new(actor).ok(), + _ => None, + }) + .collect(); + hosts.iter().for_each(|host| harness.faults.drop_host(host)); + hosts +} + +fn member_verb(draw: u64) -> (&'static str, &'static str) { + match draw { + 0 => ("sh.tangled.knot.addMember", "addMember"), + 1 => ("sh.tangled.knot.removeMember", "removeMember"), + 2 => ("sh.tangled.knot.ban", "ban"), + _ => ("sh.tangled.knot.unban", "unban"), + } +} + +struct Caller<'a> { + signer: &'a K256Signer, + did: &'a AccountDid, +} + +fn signed_post( + harness: &Harness, + caller: Caller<'_>, + nsid: &'static str, + body: serde_json::Value, + skew: bool, + round: RoundNumber, + slot: usize, +) -> Request { + let token = mint( + caller.signer, + caller.did, + &harness.knot_aud, + nsid, + jwt_window(harness, skew), + round, + OperationIndex::new(slot as u32), + ); + Request { + method: Method::POST, + uri: format!("/xrpc/{nsid}"), + token: Some(token), + body: encode_body(body), + actor: String::new(), + } +} + +fn mutate( + harness: &Harness, + actors: &RealActors, + working: &[RepoDid], + rng: &Rng, + round: RoundNumber, +) -> Vec { + let subjects = &harness.subjects; + let mut execs: Vec = Vec::new(); + + subjects + .iter() + .filter(|_| rng.chance(2, 3)) + .for_each(|subject| { + let (nsid, short) = member_verb(rng.below(4)); + let skew = rng.chance(1, 5); + let request = signed_post( + harness, + Caller { + signer: &harness.admin.signer, + did: &harness.admin.did, + }, + nsid, + json!({ "subject": subject.as_str() }), + skew, + round, + execs.len(), + ); + execs.push(Exec::Http { + op: short, + actor: format!("admin:{short}"), + fault: skew_fault(skew), + request, + killed: false, + capture_create: false, + }); + }); + + let collaborated: Vec = if subjects.is_empty() { + Vec::new() + } else { + actors + .owner_repos + .iter() + .filter(|_| rng.chance(1, 2)) + .map(|(repo, owner)| { + let subject = &subjects[rng.below(subjects.len() as u64) as usize]; + let skew = rng.chance(1, 6); + let request = signed_post( + harness, + Caller { + signer: &owner_signer(actors.seed, owner), + did: &AccountDid::from(owner.clone()), + }, + "sh.tangled.repo.addCollaborator", + json!({ "repo": repo.as_str(), "subject": subject.as_str() }), + skew, + round, + execs.len(), + ); + execs.push(Exec::Http { + op: "addCollaborator", + actor: format!("owner:{}", owner.as_str()), + fault: skew_fault(skew), + request, + killed: false, + capture_create: false, + }); + repo.clone() + }) + .collect() + }; + + working + .iter() + .filter(|repo| !collaborated.contains(*repo)) + .filter(|_| rng.chance(1, 3)) + .for_each(|repo| execs.push(Exec::Maintain { repo: repo.clone() })); + + (0..rng.below(3)).for_each(|key| { + let name = format!("sim-repo-{}-{}", round.get(), key); + let skew = rng.chance(1, 8); + let request = signed_post( + harness, + Caller { + signer: &harness.admin.signer, + did: &harness.admin.did, + }, + "sh.tangled.repo.create", + json!({ "rkey": name, "name": name }), + skew, + round, + execs.len(), + ); + execs.push(Exec::Http { + op: "createRepo", + actor: "admin:create".to_string(), + fault: skew_fault(skew), + request, + killed: false, + capture_create: !skew, + }); + }); + + if !subjects.is_empty() { + let probes = rng.below(3) as usize; + let slots: Vec = (0..harness.strangers.len()).collect(); + pick(&slots, probes, rng).into_iter().for_each(|slot| { + let stranger = &harness.strangers[slot]; + let drop = rng.chance(1, 2); + let request = signed_post( + harness, + Caller { + signer: &stranger.signer, + did: &stranger.did, + }, + "sh.tangled.knot.addMember", + json!({ "subject": subjects[0].as_str() }), + false, + round, + execs.len(), + ); + execs.push(Exec::Http { + op: "probe", + actor: stranger.host.to_string(), + fault: if drop { "drop_identity" } else { "none" }, + request, + killed: false, + capture_create: false, + }); + }); + } + + execs +} + +fn skew_fault(skew: bool) -> &'static str { + if skew { "clock_skew" } else { "none" } +} + +fn reads(working: &[RepoDid], rng: &Rng) -> Vec { + let mut execs: Vec = [ + ("version", "/xrpc/sh.tangled.knot.version"), + ("owner", "/xrpc/sh.tangled.owner"), + ("didJson", "/.well-known/did.json"), + ] + .into_iter() + .map(|(op, uri)| anon_read(op, uri, rng.chance(1, 5))) + .collect(); + + working + .iter() + .filter(|_| rng.chance(2, 3)) + .for_each(|repo| { + let killed = rng.chance(1, 5); + let exec = match rng.below(7) { + 0 => repo_read("branches", "repo", repo, killed), + 1 => repo_read("log", "repo", repo, killed), + 2 => repo_read("describeRepo", "repoDid", repo, killed), + 3 => anon_read( + "infoRefs", + &format!("/{}/info/refs?service=git-upload-pack", repo.as_str()), + killed, + ), + 4 => repo_read("tree", "repo", repo, killed), + 5 => blob_read(repo, killed), + _ => repo_read("languages", "repo", repo, killed), + }; + execs.push(exec); + }); + execs +} + +fn anon_read(op: &'static str, uri: &str, killed: bool) -> Exec { + Exec::Http { + op, + actor: "anon".to_string(), + fault: if killed { "killed" } else { "none" }, + request: Request { + method: Method::GET, + uri: uri.to_string(), + token: None, + body: Bytes::new(), + actor: "anon".to_string(), + }, + killed, + capture_create: false, + } +} + +fn repo_read(op: &'static str, param: &str, repo: &RepoDid, killed: bool) -> Exec { + anon_read( + op, + &format!("/xrpc/sh.tangled.repo.{op}?{param}={}", enc(repo.as_str())), + killed, + ) +} + +fn blob_read(repo: &RepoDid, killed: bool) -> Exec { + anon_read( + "blob", + &format!( + "/xrpc/sh.tangled.repo.blob?repo={}&path=README.md", + enc(repo.as_str()) + ), + killed, + ) +} + +async fn run_op( + harness: &Harness, + round: RoundNumber, + index: OperationIndex, + exec: Exec, +) -> OpResult { + match exec { + Exec::Maintain { repo } => { + let outcome = match harness.maintain(&repo) { + Ok(()) => Outcome::Answered { + status: HttpStatus::new(200), + body: 0, + }, + Err(message) => Outcome::Answered { + status: HttpStatus::new(500), + body: fnv1a(message.as_bytes()), + }, + }; + OpResult { + step: Step { + round, + index, + op: "maintain", + actor: "knot".to_string(), + fault: "none", + outcome, + }, + created: None, + } + } + Exec::Http { + op, + actor, + fault, + request, + killed, + capture_create, + } => { + if killed { + drive_kill(harness.router(), request.method, &request.uri, request.body).await; + return OpResult { + step: Step { + round, + index, + op, + actor, + fault: "killed", + outcome: Outcome::Killed, + }, + created: None, + }; + } + let (status, body) = http_call( + harness.router(), + request.method, + &request.uri, + request.token.as_deref(), + request.body, + ) + .await; + let created = (capture_create && status == HttpStatus::new(200)) + .then(|| repo_did_of(&body).expect("200 createRepo response carries a repoDid")); + OpResult { + step: Step { + round, + index, + op, + actor, + fault, + outcome: Outcome::Answered { + status, + body: body_digest(&body), + }, + }, + created, + } + } + } +} diff --git a/knot2/crates/knot-sim/src/trace.rs b/knot2/crates/knot-sim/src/trace.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-sim/src/trace.rs @@ -0,0 +1,92 @@ +use knot_runtime::UnixMicros; +use knot_types::{AccountDid, HttpStatus, RepoDid}; +use serde::Serialize; + +const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325; +const FNV_PRIME: u64 = 0x0000_0100_0000_01b3; + +pub(crate) fn fnv1a(bytes: &[u8]) -> u64 { + bytes.iter().fold(FNV_OFFSET, |hash, byte| { + (hash ^ u64::from(*byte)).wrapping_mul(FNV_PRIME) + }) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize)] +#[serde(transparent)] +pub struct RoundNumber(u32); + +impl RoundNumber { + pub const fn new(value: u32) -> Self { + Self(value) + } + + pub const fn get(self) -> u32 { + self.0 + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize)] +#[serde(transparent)] +pub struct OperationIndex(u32); + +impl OperationIndex { + pub const fn new(value: u32) -> Self { + Self(value) + } + + pub const fn get(self) -> u32 { + self.0 + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum Outcome { + Answered { status: HttpStatus, body: u64 }, + Killed, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Step { + pub round: RoundNumber, + pub index: OperationIndex, + pub op: &'static str, + pub actor: String, + pub fault: &'static str, + pub outcome: Outcome, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct RepoCollaborators { + pub repo: RepoDid, + pub subjects: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Projection { + pub members: Vec, + pub blocked: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Snapshot { + pub round: RoundNumber, + pub clock_micros: UnixMicros, + pub members: Vec, + pub blocked: Vec, + pub repos: Vec, + pub collaborators: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Trace { + pub seed: u64, + pub steps: Vec, + pub snapshots: Vec, +} + +impl Trace { + pub fn digest(&self) -> u64 { + fnv1a(&serde_json::to_vec(self).expect("trace is always serializable")) + } +} diff --git a/knot2/crates/knot-sim/src/workload.rs b/knot2/crates/knot-sim/src/workload.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-sim/src/workload.rs @@ -0,0 +1,836 @@ +use axum::body::Bytes; +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use futures::future::join_all; +use futures::stream::StreamExt; +use http::Method; +use http::header::AUTHORIZATION; +use serde_json::{Value, json}; +use std::collections::BTreeSet; +use std::net::SocketAddr; +use std::sync::Arc; +use tower::ServiceExt; + +use knot_runtime::{Entropy, K256Signer, SeededEntropy, Signer}; +use knot_types::{AccountDid, HttpStatus, KnotHostname, KnotId, RepoDid, UnixSeconds}; + +use crate::harness::{Harness, SUBJECT_DIDS}; +use crate::trace::{OperationIndex, Outcome, Projection, RoundNumber, Step, Trace, fnv1a}; + +const SKEW_BACKDATE_SECS: i64 = 600; +const SKEW_LIFETIME_SECS: i64 = 60; + +#[derive(Clone, Copy)] +struct RepoIndex(usize); + +#[derive(Clone, Copy)] +struct SubjectIndex(usize); + +#[derive(Clone, Copy)] +struct StrangerIndex(usize); + +#[derive(Clone, Copy)] +enum ReadOp { + Version, + Owner, + ListMembers, + DidJson, + InfoRefs(RepoIndex), + Branches(RepoIndex), + Log(RepoIndex), + DescribeRepo(RepoIndex), + Tree(RepoIndex), + Blob(RepoIndex), + Languages(RepoIndex), +} + +impl ReadOp { + fn name(self) -> &'static str { + match self { + ReadOp::Version => "version", + ReadOp::Owner => "owner", + ReadOp::ListMembers => "listMembers", + ReadOp::DidJson => "didJson", + ReadOp::InfoRefs(_) => "infoRefs", + ReadOp::Branches(_) => "branches", + ReadOp::Log(_) => "log", + ReadOp::DescribeRepo(_) => "describeRepo", + ReadOp::Tree(_) => "tree", + ReadOp::Blob(_) => "blob", + ReadOp::Languages(_) => "languages", + } + } +} + +#[derive(Clone, Copy)] +enum AdminOp { + AddMember(SubjectIndex), + RemoveMember(SubjectIndex), + Ban(SubjectIndex), + Unban(SubjectIndex), + CreateRepo(u32), + AddCollaborator(RepoIndex, SubjectIndex), +} + +impl AdminOp { + fn name(self) -> &'static str { + match self { + AdminOp::AddMember(_) => "addMember", + AdminOp::RemoveMember(_) => "removeMember", + AdminOp::Ban(_) => "ban", + AdminOp::Unban(_) => "unban", + AdminOp::CreateRepo(_) => "createRepo", + AdminOp::AddCollaborator(_, _) => "addCollaborator", + } + } +} + +#[derive(Clone, Copy)] +enum Planned { + Read { op: ReadOp, killed: bool }, + Admin { op: AdminOp, skew: bool }, + Probe { stranger: StrangerIndex, drop: bool }, + Maintain { repo: RepoIndex }, +} + +pub(crate) struct Round { + ops: Vec, + advance: std::time::Duration, +} + +pub(crate) struct Rng(SeededEntropy); + +impl Rng { + pub(crate) fn new(seed: u64) -> Self { + Self(SeededEntropy::new(seed ^ 0x57ee_d000)) + } + + pub(crate) fn below(&self, n: u64) -> u64 { + self.0.next_u64() % n.max(1) + } + + pub(crate) fn chance(&self, num: u64, den: u64) -> bool { + assert!(num <= den, "chance numerator exceeds denominator"); + self.below(den) < num + } +} + +pub(crate) fn plan(seed: u64, rounds: u32, subjects: usize) -> Vec { + let rng = Rng::new(seed); + let mut available: u32 = 1; + let mut rkey: u32 = 0; + let mut stranger: usize = 0; + (0..rounds) + .map(|round| { + let (ops, fresh_repos) = if round % 2 == 0 { + mutate_round(&rng, subjects, available, &mut rkey, &mut stranger) + } else { + (read_round(&rng, available), 0) + }; + let advance = std::time::Duration::from_micros(rng.below(3_000_000)); + available += fresh_repos; + Round { ops, advance } + }) + .collect() +} + +pub(crate) fn predict(seed: u64, rounds: u32) -> Projection { + let (members, blocked) = plan(seed, rounds, SUBJECT_DIDS.len()) + .iter() + .flat_map(|round| round.ops.iter()) + .fold( + (BTreeSet::::new(), BTreeSet::::new()), + |(mut members, mut blocked), planned| { + let subject_did = |subject: &SubjectIndex| { + AccountDid::new(SUBJECT_DIDS[subject.0]).expect("subject did") + }; + if let Planned::Admin { op, skew: false } = planned { + match op { + AdminOp::AddMember(subject) => { + members.insert(subject_did(subject)); + } + AdminOp::RemoveMember(subject) => { + members.remove(&subject_did(subject)); + } + AdminOp::Ban(subject) => { + blocked.insert(subject_did(subject)); + } + AdminOp::Unban(subject) => { + blocked.remove(&subject_did(subject)); + } + AdminOp::CreateRepo(_) | AdminOp::AddCollaborator(_, _) => {} + } + } + (members, blocked) + }, + ); + Projection { + members: members.into_iter().collect(), + blocked: blocked.into_iter().collect(), + } +} + +fn mutate_round( + rng: &Rng, + subjects: usize, + available: u32, + rkey: &mut u32, + stranger: &mut usize, +) -> (Vec, u32) { + let mut ops: Vec = (0..subjects) + .filter(|_| rng.chance(2, 3)) + .map(|subject| { + let subject = SubjectIndex(subject); + let op = match rng.below(4) { + 0 => AdminOp::AddMember(subject), + 1 => AdminOp::RemoveMember(subject), + 2 => AdminOp::Ban(subject), + _ => AdminOp::Unban(subject), + }; + Planned::Admin { + op, + skew: rng.chance(1, 5), + } + }) + .collect(); + + (0..available) + .filter(|_| rng.chance(1, 2)) + .for_each(|repo| { + let planned = if rng.chance(1, 2) { + let subject = SubjectIndex(rng.below(subjects as u64) as usize); + Planned::Admin { + op: AdminOp::AddCollaborator(RepoIndex(repo as usize), subject), + skew: rng.chance(1, 6), + } + } else { + Planned::Maintain { + repo: RepoIndex(repo as usize), + } + }; + ops.push(planned); + }); + + let mut fresh_repos = 0; + (0..rng.below(3)).for_each(|_| { + let key = *rkey; + *rkey += 1; + let skew = rng.chance(1, 8); + if !skew { + fresh_repos += 1; + } + ops.push(Planned::Admin { + op: AdminOp::CreateRepo(key), + skew, + }); + }); + + (0..rng.below(3)).for_each(|_| { + let stranger_index = StrangerIndex(*stranger); + *stranger += 1; + ops.push(Planned::Probe { + stranger: stranger_index, + drop: rng.chance(1, 2), + }); + }); + + (ops, fresh_repos) +} + +fn read_round(rng: &Rng, available: u32) -> Vec { + let mut ops: Vec = [ + ReadOp::Version, + ReadOp::Owner, + ReadOp::ListMembers, + ReadOp::DidJson, + ] + .into_iter() + .map(|op| Planned::Read { + op, + killed: rng.chance(1, 5), + }) + .collect(); + (0..available) + .filter(|_| rng.chance(2, 3)) + .for_each(|repo| { + let repo = RepoIndex(repo as usize); + let op = match rng.below(7) { + 0 => ReadOp::Branches(repo), + 1 => ReadOp::Log(repo), + 2 => ReadOp::DescribeRepo(repo), + 3 => ReadOp::InfoRefs(repo), + 4 => ReadOp::Tree(repo), + 5 => ReadOp::Blob(repo), + _ => ReadOp::Languages(repo), + }; + ops.push(Planned::Read { + op, + killed: rng.chance(1, 5), + }); + }); + ops +} + +struct OpResult { + step: Step, + created: Option, +} + +pub(crate) async fn execute(harness: Arc, seed: u64, plan: Vec) -> Trace { + let initial = ( + vec![harness.seed_repo.clone()], + Vec::::new(), + Vec::new(), + ); + let harness = &harness; + let (repos, steps, snapshots) = futures::stream::iter(plan.into_iter().enumerate()) + .fold( + initial, + |(repos, mut steps, mut snapshots), (round_index, round)| { + let harness = Arc::clone(harness); + async move { + let round_no = RoundNumber::new(round_index as u32); + let drops = arm_drops(&harness, &round.ops); + let repos = Arc::new(repos); + let tasks = round + .ops + .iter() + .enumerate() + .map(|(index, planned)| { + let harness = Arc::clone(&harness); + let repos = Arc::clone(&repos); + let planned = *planned; + tokio::spawn(async move { + run_op( + &harness, + &repos, + round_no, + OperationIndex::new(index as u32), + planned, + ) + .await + }) + }) + .collect::>(); + let results: Vec = join_all(tasks) + .await + .into_iter() + .map(|joined| joined.expect("sim op task mustn't panic")) + .collect(); + drops + .iter() + .for_each(|host| harness.faults.clear_host(host)); + + let created: Vec = results + .iter() + .filter_map(|result| result.created.clone()) + .collect(); + let planned_creates = round + .ops + .iter() + .filter(|planned| { + matches!( + planned, + Planned::Admin { + op: AdminOp::CreateRepo(_), + skew: false, + } + ) + }) + .count(); + assert_eq!( + planned_creates, + created.len(), + "round {}: {planned_creates} non-skew creates planned but \ + {} materialized, so plan/execute repo indices have drifted apart", + round_no.get(), + created.len() + ); + created.iter().for_each(|did| harness.populate(did)); + steps.extend(results.into_iter().map(|result| result.step)); + let mut repos = Arc::into_inner(repos) + .expect("all op tasks released the round repo snapshot"); + repos.extend(created); + + harness.advance(round.advance); + snapshots.push(harness.snapshot(round_no, &repos)); + (repos, steps, snapshots) + } + }, + ) + .await; + let no_fault_creates = steps + .iter() + .filter(|step| step.op == "createRepo" && step.fault == "none") + .count(); + let materialized = repos.len() - 1; + assert_eq!( + no_fault_creates, materialized, + "no-fault createRepo count {no_fault_creates} doesn't match the {materialized} repos \ + materialized: a planned create silently failed and repo_at would have masked the drift" + ); + Trace { + seed, + steps, + snapshots, + } +} + +fn arm_drops(harness: &Harness, ops: &[Planned]) -> Vec { + let hosts: Vec = ops + .iter() + .filter_map(|planned| match planned { + Planned::Probe { + stranger, + drop: true, + } => Some(harness.strangers[stranger.0].host.clone()), + _ => None, + }) + .collect(); + hosts.iter().for_each(|host| harness.faults.drop_host(host)); + hosts +} + +async fn run_op( + harness: &Harness, + repos: &[RepoDid], + round: RoundNumber, + index: OperationIndex, + planned: Planned, +) -> OpResult { + let make = |op: &'static str, actor: String, fault: &'static str, outcome: Outcome| Step { + round, + index, + op, + actor, + fault, + outcome, + }; + + match planned { + Planned::Maintain { repo } => { + let repo = repo_at(repos, repo); + let outcome = match harness.maintain(repo) { + Ok(()) => Outcome::Answered { + status: HttpStatus::new(200), + body: 0, + }, + Err(message) => Outcome::Answered { + status: HttpStatus::new(500), + body: fnv1a(message.as_bytes()), + }, + }; + OpResult { + step: make("maintain", "knot".to_string(), "none", outcome), + created: None, + } + } + Planned::Read { op, killed } => { + let request = read_request(repos, op); + if killed { + drive_kill(harness.router(), request.method, &request.uri, request.body).await; + return OpResult { + step: make(op.name(), request.actor, "killed", Outcome::Killed), + created: None, + }; + } + let (status, body) = http_call( + harness.router(), + request.method, + &request.uri, + None, + request.body, + ) + .await; + OpResult { + step: make( + op.name(), + request.actor, + "none", + Outcome::Answered { + status, + body: body_digest(&body), + }, + ), + created: None, + } + } + Planned::Admin { op, skew } => { + let request = admin_request(harness, repos, op, skew, round, index); + let (status, body) = http_call( + harness.router(), + request.method, + &request.uri, + request.token.as_deref(), + request.body, + ) + .await; + let created = match op { + AdminOp::CreateRepo(_) if status == HttpStatus::new(200) => repo_did_of(&body), + _ => None, + }; + OpResult { + step: make( + op.name(), + request.actor, + if skew { "clock_skew" } else { "none" }, + Outcome::Answered { + status, + body: body_digest(&body), + }, + ), + created, + } + } + Planned::Probe { stranger, drop } => { + let request = probe_request(harness, stranger, round, index); + let (status, body) = http_call( + harness.router(), + request.method, + &request.uri, + request.token.as_deref(), + request.body, + ) + .await; + OpResult { + step: make( + "probe", + request.actor, + if drop { "drop_identity" } else { "none" }, + Outcome::Answered { + status, + body: body_digest(&body), + }, + ), + created: None, + } + } + } +} + +pub(crate) struct Request { + pub(crate) method: Method, + pub(crate) uri: String, + pub(crate) token: Option, + pub(crate) body: Bytes, + pub(crate) actor: String, +} + +fn read_request(repos: &[RepoDid], op: ReadOp) -> Request { + match op { + ReadOp::Version => get("/xrpc/sh.tangled.knot.version"), + ReadOp::Owner => get("/xrpc/sh.tangled.owner"), + ReadOp::ListMembers => { + get("/xrpc/sh.tangled.knot.listMembers?subject=did:web:knot.nel.pet") + } + ReadOp::DidJson => get("/.well-known/did.json"), + ReadOp::InfoRefs(repo) => Request { + method: Method::GET, + uri: format!( + "/{}/info/refs?service=git-upload-pack", + repo_at(repos, repo).as_str() + ), + token: None, + body: Bytes::new(), + actor: "anon".to_string(), + }, + ReadOp::Branches(repo) => repo_get("branches", "repo", repo_at(repos, repo)), + ReadOp::Log(repo) => repo_get("log", "repo", repo_at(repos, repo)), + ReadOp::DescribeRepo(repo) => repo_get("describeRepo", "repoDid", repo_at(repos, repo)), + ReadOp::Tree(repo) => repo_get("tree", "repo", repo_at(repos, repo)), + ReadOp::Languages(repo) => repo_get("languages", "repo", repo_at(repos, repo)), + ReadOp::Blob(repo) => Request { + method: Method::GET, + uri: format!( + "/xrpc/sh.tangled.repo.blob?repo={}&path=README.md", + enc(repo_at(repos, repo).as_str()) + ), + token: None, + body: Bytes::new(), + actor: "anon".to_string(), + }, + } +} + +fn admin_request( + harness: &Harness, + repos: &[RepoDid], + op: AdminOp, + skew: bool, + round: RoundNumber, + index: OperationIndex, +) -> Request { + let subjects = &harness.subjects; + match op { + AdminOp::AddMember(subject) => admin_post( + harness, + "addMember", + "sh.tangled.knot.addMember", + json!({ "subject": subjects[subject.0].as_str() }), + skew, + round, + index, + ), + AdminOp::RemoveMember(subject) => admin_post( + harness, + "removeMember", + "sh.tangled.knot.removeMember", + json!({ "subject": subjects[subject.0].as_str() }), + skew, + round, + index, + ), + AdminOp::Ban(subject) => admin_post( + harness, + "ban", + "sh.tangled.knot.ban", + json!({ "subject": subjects[subject.0].as_str() }), + skew, + round, + index, + ), + AdminOp::Unban(subject) => admin_post( + harness, + "unban", + "sh.tangled.knot.unban", + json!({ "subject": subjects[subject.0].as_str() }), + skew, + round, + index, + ), + AdminOp::CreateRepo(key) => { + let name = format!("repo{key}"); + admin_post( + harness, + "create", + "sh.tangled.repo.create", + json!({ "rkey": name, "name": name }), + skew, + round, + index, + ) + } + AdminOp::AddCollaborator(repo, subject) => admin_post( + harness, + "addCollaborator", + "sh.tangled.repo.addCollaborator", + json!({ + "repo": repo_at(repos, repo).as_str(), + "subject": subjects[subject.0].as_str(), + }), + skew, + round, + index, + ), + } +} + +fn probe_request( + harness: &Harness, + stranger: StrangerIndex, + round: RoundNumber, + index: OperationIndex, +) -> Request { + let actor = &harness.strangers[stranger.0]; + let token = mint( + &actor.signer, + &actor.did, + &harness.knot_aud, + "sh.tangled.knot.addMember", + jwt_window(harness, false), + round, + index, + ); + Request { + method: Method::POST, + uri: "/xrpc/sh.tangled.knot.addMember".to_string(), + token: Some(token), + body: encode_body(json!({ "subject": harness.subjects[0].as_str() })), + actor: actor.host.to_string(), + } +} + +fn get(path: &str) -> Request { + Request { + method: Method::GET, + uri: path.to_string(), + token: None, + body: Bytes::new(), + actor: "anon".to_string(), + } +} + +fn repo_get(method: &str, param: &str, repo: &RepoDid) -> Request { + Request { + method: Method::GET, + uri: format!( + "/xrpc/sh.tangled.repo.{method}?{param}={}", + enc(repo.as_str()) + ), + token: None, + body: Bytes::new(), + actor: "anon".to_string(), + } +} + +fn admin_post( + harness: &Harness, + method_short: &str, + nsid: &'static str, + body: Value, + skew: bool, + round: RoundNumber, + index: OperationIndex, +) -> Request { + let admin = &harness.admin; + let token = mint( + &admin.signer, + &admin.did, + &harness.knot_aud, + nsid, + jwt_window(harness, skew), + round, + index, + ); + Request { + method: Method::POST, + uri: format!("/xrpc/{nsid}"), + token: Some(token), + body: encode_body(body), + actor: format!("admin:{method_short}"), + } +} + +pub(crate) fn jwt_window(harness: &Harness, skew: bool) -> (UnixSeconds, UnixSeconds) { + let now = harness.now_seconds(); + if skew { + ( + now.saturating_sub_secs(SKEW_BACKDATE_SECS + SKEW_LIFETIME_SECS), + now.saturating_sub_secs(SKEW_BACKDATE_SECS), + ) + } else { + (now, now.saturating_add_secs(60)) + } +} + +pub(crate) fn mint( + signer: &K256Signer, + issuer: &AccountDid, + aud: &KnotId, + nsid: &str, + window: (UnixSeconds, UnixSeconds), + round: RoundNumber, + index: OperationIndex, +) -> String { + let header = URL_SAFE_NO_PAD.encode(br#"{"alg":"ES256K","typ":"JWT"}"#); + let payload = URL_SAFE_NO_PAD.encode( + serde_json::to_vec(&json!({ + "iss": issuer.as_str(), + "aud": aud.as_str(), + "iat": window.0.get(), + "exp": window.1.get(), + "jti": format!("sim-{}-{}", round.get(), index.get()), + "lxm": nsid, + })) + .expect("claims serialize"), + ); + let signing_input = format!("{header}.{payload}"); + let signature = signer.sign(signing_input.as_bytes()); + format!( + "{signing_input}.{}", + URL_SAFE_NO_PAD.encode(signature.as_bytes()) + ) +} + +pub(crate) async fn http_call( + router: axum::Router, + method: Method, + uri: &str, + token: Option<&str>, + body: Bytes, +) -> (HttpStatus, Bytes) { + let mut request = http::Request::builder() + .method(method) + .uri(uri) + .body(axum::body::Body::from(body)) + .expect("request builds"); + if let Some(token) = token { + request.headers_mut().insert( + AUTHORIZATION, + http::HeaderValue::from_str(&format!("Bearer {token}")).expect("bearer header"), + ); + } + request + .extensions_mut() + .insert(axum::extract::ConnectInfo(SocketAddr::from(( + [127, 0, 0, 1], + 4242, + )))); + let response = router.oneshot(request).await.expect("router answers"); + let status = HttpStatus::from(response.status()); + let bytes = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .expect("response body"); + (status, bytes) +} + +pub(crate) async fn drive_kill(router: axum::Router, method: Method, uri: &str, body: Bytes) { + let call = http_call(router, method, uri, None, body); + futures::pin_mut!(call); + tokio::select! { + biased; + _ = &mut call => {} + _ = tokio::task::yield_now() => {} + } +} + +pub(crate) fn repo_did_of(body: &Bytes) -> Option { + serde_json::from_slice::(body) + .ok() + .and_then(|value| { + value + .get("repoDid") + .and_then(Value::as_str) + .map(str::to_string) + }) + .and_then(|did| RepoDid::new(did).ok()) +} + +pub(crate) fn body_digest(body: &Bytes) -> u64 { + match serde_json::from_slice::(body) { + Ok(mut value) => { + canonicalize(&mut value); + fnv1a(&serde_json::to_vec(&value).expect("canonical body serializes")) + } + Err(_) => fnv1a(body), + } +} + +fn canonicalize(value: &mut Value) { + match value { + Value::Array(items) => { + items.iter_mut().for_each(canonicalize); + items.sort_by_cached_key(|item| serde_json::to_string(item).expect("array item")); + } + Value::Object(map) => map.values_mut().for_each(canonicalize), + _ => {} + } +} + +pub(crate) fn encode_body(value: Value) -> Bytes { + Bytes::from(serde_json::to_vec(&value).expect("request body serializes")) +} + +pub(crate) fn enc(did: &str) -> String { + did.replace(':', "%3A") +} + +fn repo_at(repos: &[RepoDid], index: RepoIndex) -> &RepoDid { + repos.get(index.0).unwrap_or_else(|| { + panic!( + "plan/execute repo drift: index {} exceeds {} repos created so far", + index.0, + repos.len() + ) + }) +} diff --git a/knot2/crates/knot-sim/tests/h3.rs b/knot2/crates/knot-sim/tests/h3.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-sim/tests/h3.rs @@ -0,0 +1,245 @@ +mod common; + +use std::collections::BTreeSet; +use std::io::Write; +use std::path::Path; +use std::process::Stdio; +use std::sync::Arc; + +use bytes::Bytes; +use common::Edge; +use http::Method; +use knot_edge::RequiresFullHandshake; +use knot_git::Layout; +use knot_pack::{CacheConfig, RepoLookup, RepoResolver, RepoTarget}; +use knot_types::{ObjectFormat, RepoDid}; + +const PINNED_DATE: &str = "2026-06-20T12:00:00+00:00"; + +fn git(cwd: &Path, args: &[&str]) -> String { + let out = knot_fixtures::command(cwd) + .args(args) + .env("GIT_AUTHOR_DATE", PINNED_DATE) + .env("GIT_COMMITTER_DATE", PINNED_DATE) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8(out.stdout) + .expect("git stdout is utf-8") + .trim() + .to_string() +} + +fn object_set(dir: &Path) -> BTreeSet { + git( + dir, + &[ + "cat-file", + "--batch-all-objects", + "--batch-check=%(objectname)", + ], + ) + .lines() + .map(|line| line.trim().to_string()) + .filter(|line| !line.is_empty()) + .collect() +} + +fn seed_bare(work: &Path, bare: &Path, format: ObjectFormat) -> String { + let fmt = format!("--object-format={}", format.capability()); + std::fs::create_dir_all(work).unwrap(); + git(work, &["init", &fmt, "-q", "-b", "main"]); + std::fs::write(work.join("README.md"), "h3 over the simulated quic edge\n").unwrap(); + git(work, &["add", "-A"]); + git(work, &["commit", "-q", "-m", "c1"]); + std::fs::write(work.join("extra.txt"), "a second object\n").unwrap(); + git(work, &["add", "-A"]); + git(work, &["commit", "-q", "-m", "c2"]); + git(work, &["push", "-q", bare.to_str().unwrap(), "main"]); + git(bare, &["symbolic-ref", "HEAD", "refs/heads/main"]); + git(work, &["rev-parse", "HEAD"]) +} + +fn init(dir: &Path, format: ObjectFormat) { + std::fs::create_dir_all(dir).unwrap(); + let fmt = format!("--object-format={}", format.capability()); + git(dir, &["init", &fmt, "-q", dir.to_str().unwrap()]); +} + +fn index_pack(repo: &Path, pack: &[u8]) { + let mut child = knot_fixtures::command(repo) + .args(["index-pack", "--stdin", "--fix-thin"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn index-pack"); + child.stdin.take().unwrap().write_all(pack).unwrap(); + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "index-pack failed: {}", + String::from_utf8_lossy(&output.stderr) + ); +} + +fn fetch_body(tip: &str) -> Bytes { + let mut body = Vec::new(); + body.extend_from_slice(&common::pkt(b"command=fetch\n")); + body.extend_from_slice(&common::pkt(b"agent=knot/0\n")); + body.extend_from_slice(b"0001"); + body.extend_from_slice(&common::pkt(b"no-progress\n")); + body.extend_from_slice(&common::pkt(b"ofs-delta\n")); + body.extend_from_slice(&common::pkt(format!("want {tip}\n").as_bytes())); + body.extend_from_slice(&common::pkt(b"done\n")); + body.extend_from_slice(b"0000"); + Bytes::from(body) +} + +fn extract_pack(response: &[u8]) -> Vec { + let mut channel = Vec::new(); + let mut pos = 0usize; + while pos + 4 <= response.len() { + let len = std::str::from_utf8(&response[pos..pos + 4]) + .ok() + .and_then(|hex| usize::from_str_radix(hex, 16).ok()) + .unwrap_or(0); + pos += 4; + if len < 4 { + continue; + } + let end = (pos + len - 4).min(response.len()); + let payload = &response[pos..end]; + pos = end; + if payload.first() == Some(&1) { + channel.extend_from_slice(&payload[1..]); + } + } + match channel.windows(4).position(|window| window == b"PACK") { + Some(start) => channel.split_off(start), + None => channel, + } +} + +fn serve_dids() -> Arc { + Arc::new(|target: &RepoTarget| match target { + RepoTarget::Did(did) => RepoLookup::Hosted(did.clone()), + RepoTarget::OwnerRkey(_, _) => RepoLookup::Unhosted, + }) +} + +async fn clone_over_h3(edge: &Edge, did: &str, tip: &str) -> Vec { + let connection = edge + .client + .connect(edge.addr, "localhost") + .unwrap() + .await + .unwrap(); + let quic = connection.clone(); + let (mut driver, mut sender) = h3::client::new(h3_quinn::Connection::new(connection)) + .await + .unwrap(); + let drive = tokio::spawn(async move { + let _ = std::future::poll_fn(|cx| driver.poll_close(cx)).await; + }); + + let warmup = http::Request::get(format!( + "https://localhost/{did}/info/refs?service=git-upload-pack" + )) + .header("git-protocol", "version=2") + .body(()) + .unwrap(); + let mut warm = sender.send_request(warmup).await.unwrap(); + common::finish_request(&mut warm).await; + assert!( + warm.recv_response().await.unwrap().status().is_success(), + "h3 info/refs advertisement must serve over QUIC" + ); + common::drain(&mut warm).await; + + let request = http::Request::builder() + .method(Method::POST) + .uri(format!("https://localhost/{did}/git-upload-pack")) + .header("content-type", "application/x-git-upload-pack-request") + .header("git-protocol", "version=2") + .body(()) + .unwrap(); + let mut stream = sender.send_request(request).await.unwrap(); + stream.send_data(fetch_body(tip)).await.unwrap(); + common::finish_request(&mut stream).await; + let response = stream.recv_response().await.unwrap(); + assert!( + response.status().is_success(), + "h3 upload-pack returned {}", + response.status() + ); + let out = common::drain(&mut stream).await; + quic.close(0u32.into(), b"done"); + drive.abort(); + out +} + +async fn cloned_set( + layout: &Layout, + did: &str, + tip: &str, + format: ObjectFormat, +) -> BTreeSet { + let certdir = tempfile::tempdir().unwrap(); + let clonedir = tempfile::tempdir().unwrap(); + let edge = common::serve_edge(certdir.path(), || { + let (write_routes, advertisement) = knot_pack::edge_routes( + layout.clone(), + serve_dids(), + None, + None, + knot_resource::PackSlots::new(4), + CacheConfig::default(), + Arc::new(knot_messages::Catalog::defaults()), + knot_pack::default_hostname().clone(), + Arc::new(knot_runtime::SystemClock), + ); + (RequiresFullHandshake::new(write_routes), advertisement) + }) + .await; + let pack = extract_pack(&clone_over_h3(&edge, did, tip).await); + edge.shutdown.cancel(); + edge.task.abort(); + init(clonedir.path(), format); + index_pack(clonedir.path(), &pack); + object_set(clonedir.path()) +} + +async fn the_live_h3_transport_is_logically_reproducible(format: ObjectFormat, did_str: &str) { + let scan = tempfile::tempdir().unwrap(); + let scratch = tempfile::tempdir().unwrap(); + + let did = RepoDid::new(did_str).unwrap(); + let layout = Layout::new(scan.path()).with_object_format(format); + layout.create(&did).unwrap(); + let bare = layout.repo_path(&did).unwrap(); + let tip = seed_bare(&scratch.path().join("work"), &bare, format); + let truth = object_set(&bare); + + let first = cloned_set(&layout, did_str, &tip, format).await; + let second = cloned_set(&layout, did_str, &tip, format).await; + + assert_eq!( + first, second, + "{format:?}: two independent live h3 connections must deliver the same object set" + ); + assert_eq!( + first, truth, + "{format:?}: the h3 clone's object set must equal the bare's full object set" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn the_new_h3_transport_replays_to_the_same_object_set_off_the_recorded_trace() { + the_live_h3_transport_is_logically_reproducible(ObjectFormat::SHA1, "did:plc:squid").await; + the_live_h3_transport_is_logically_reproducible(ObjectFormat::SHA256, "did:plc:cuttle").await; +} diff --git a/knot2/crates/knot-sim/tests/lfs_chaos.rs b/knot2/crates/knot-sim/tests/lfs_chaos.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-sim/tests/lfs_chaos.rs @@ -0,0 +1,227 @@ +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant, SystemTime}; + +use knot_cob::{CobHome, CobStore}; +use knot_cobs::{Registration, RegistryChange, RepoRef, RepoRegistryCob, deregister_repo}; +use knot_git::{Layout, Repo}; +use knot_lfs::{ClaimedSize, DiskStore, LfsOid, LfsStore, LfsStorePath}; +use knot_runtime::OsEntropy; +use knot_secrets::{MasterKey, SealedStore}; +use knot_types::{KnotId, OwnerDid, RepoDid, RepoName, RepoRkey, UnixSeconds}; +use sha2::{Digest, Sha256}; + +const REPO_DID: &str = "did:plc:squid"; +const REPO_NAME: &str = "anemone"; +const OWNER_DID: &str = "did:plc:nel"; +const KNOT_DID: &str = "did:web:nel.pet"; +const MEDIA: &[u8] = b"\xff\x00media that mustn't outlive its repo"; +const GRACE: Duration = Duration::from_secs(86_400); +const BACKDATE: Duration = Duration::from_secs(60 * 86_400); + +fn did() -> RepoDid { + RepoDid::new(REPO_DID).unwrap() +} + +fn knot() -> KnotId { + KnotId::new(KNOT_DID).unwrap() +} + +fn master() -> MasterKey { + MasterKey::new([7u8; 32]).unwrap() +} + +fn media_oid() -> LfsOid { + LfsOid::from_digest(Sha256::digest(MEDIA).into()) +} + +struct Paths { + meta: PathBuf, + scan: PathBuf, + store: PathBuf, + keys: PathBuf, +} + +impl Paths { + fn under(root: &Path) -> Self { + Self { + meta: root.join("meta"), + scan: root.join("repos"), + store: root.join("lfs"), + keys: root.join("keys.sealed"), + } + } +} + +fn build_fixture(root: &Path) -> Paths { + let paths = Paths::under(root); + Repo::create(&paths.meta).unwrap(); + Layout::new(&paths.scan).create(&did()).unwrap(); + + let secrets = SealedStore::open(&paths.keys, &master(), Box::new(OsEntropy)).unwrap(); + secrets.ensure(&knot()).unwrap(); + let signer = secrets.signer(&knot()).unwrap(); + let meta = Repo::open(&paths.meta).unwrap(); + CobStore::new(&meta) + .create( + &CobHome::from(&knot()), + &RegistryChange::Register(Registration { + owner: OwnerDid::new(OWNER_DID).unwrap(), + rkey: RepoRkey::new(REPO_NAME).unwrap(), + name: RepoName::new(REPO_NAME).unwrap(), + repo: did(), + created_at: UnixSeconds::new(1), + }), + &signer, + UnixSeconds::new(1), + ) + .unwrap(); + + std::fs::create_dir_all(&paths.store).unwrap(); + let store = DiskStore::open(LfsStorePath::new(&paths.store)).unwrap(); + store + .put( + &did(), + &media_oid(), + ClaimedSize::new(MEDIA.len() as u64), + &mut &MEDIA[..], + ) + .unwrap(); + let object_path = store.object_file(&did(), &media_oid()).unwrap().unwrap().1; + std::fs::OpenOptions::new() + .write(true) + .open(object_path) + .unwrap() + .set_modified(SystemTime::now() - BACKDATE) + .unwrap(); + paths +} + +fn spawn_worker(paths: &Paths) -> std::process::Child { + Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "chaos_delete_worker", "--nocapture"]) + .env("KNOT_CHAOS_ROLE", "delete") + .env("KNOT_CHAOS_META", &paths.meta) + .env("KNOT_CHAOS_SCAN", &paths.scan) + .env("KNOT_CHAOS_STORE", &paths.store) + .env("KNOT_CHAOS_KEYS", &paths.keys) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn chaos worker") +} + +#[test] +fn chaos_delete_worker() { + if std::env::var("KNOT_CHAOS_ROLE").as_deref() != Ok("delete") { + return; + } + let meta = Repo::open(std::env::var("KNOT_CHAOS_META").unwrap()).unwrap(); + let store = CobStore::new(&meta); + let secrets = SealedStore::open( + std::env::var("KNOT_CHAOS_KEYS").unwrap(), + &master(), + Box::new(OsEntropy), + ) + .unwrap(); + let signer = secrets.signer(&knot()).unwrap(); + let object = store.list::().unwrap()[0]; + deregister_repo( + &store, + &CobHome::from(&knot()), + object, + RepoRef { + owner: OwnerDid::new(OWNER_DID).unwrap(), + rkey: RepoRkey::new(REPO_NAME).unwrap(), + }, + did(), + &signer, + UnixSeconds::new(2), + ) + .unwrap(); + Layout::new(std::env::var("KNOT_CHAOS_SCAN").unwrap()) + .remove(&did()) + .unwrap(); + DiskStore::open(LfsStorePath::new( + std::env::var("KNOT_CHAOS_STORE").unwrap(), + )) + .unwrap() + .remove_repo(&did()) + .unwrap(); +} + +#[test] +fn kill9_between_delete_steps_never_strands_the_store_prefix() { + let scratch = tempfile::tempdir().unwrap(); + + let warm = build_fixture(&scratch.path().join("warm")); + let started = Instant::now(); + spawn_worker(&warm).wait().unwrap(); + let full = started.elapsed(); + let warm_store = DiskStore::open(LfsStorePath::new(&warm.store)).unwrap(); + assert_eq!( + warm_store.probe(&did(), &media_oid()).unwrap(), + None, + "an uninterrupted delete removes the store prefix itself" + ); + + let fractions = [0.20, 0.35, 0.45, 0.55, 0.65, 0.75, 0.85, 0.95]; + let delays: Vec = std::iter::once(Duration::from_millis(1)) + .chain(std::iter::once(Duration::from_millis(3))) + .chain(fractions.iter().map(|fraction| full.mul_f64(*fraction))) + .chain(std::iter::once(full.mul_f64(2.0))) + .collect(); + + let outcomes: Vec = delays + .iter() + .enumerate() + .map(|(trial, delay)| { + let paths = build_fixture(&scratch.path().join(format!("trial-{trial}"))); + let mut child = spawn_worker(&paths); + std::thread::sleep(*delay); + let _ = child.kill(); + child.wait().unwrap(); + + let index = knot_index::Index::new(paths.meta.clone(), Layout::new(&paths.scan)); + index.rebuild().unwrap_or_else(|error| { + panic!("trial {trial}: registry must rebuild after a killed delete: {error}") + }); + assert_eq!( + index.coverage().registry, + knot_index::Coverage::Ready, + "trial {trial}: a rebuilt projection is ready" + ); + let hosted: HashSet = index.hosted_repos().into_iter().collect(); + let registered = hosted.contains(&did()); + + let store = DiskStore::open(LfsStorePath::new(&paths.store)).unwrap(); + store + .sweep_orphans(&hosted, GRACE, SystemTime::now()) + .unwrap_or_else(|error| { + panic!("trial {trial}: orphan sweep must run after a killed delete: {error}") + }); + let present = store.probe(&did(), &media_oid()).unwrap().is_some(); + match registered { + true => assert!( + present, + "trial {trial}: a still-registered repo's store prefix is never condemned" + ), + false => assert!( + !present, + "trial {trial}: the orphan sweep reclaims the prefix the killed delete left behind" + ), + } + registered + }) + .collect(); + + assert!( + outcomes.iter().any(|registered| *registered), + "some trial must die before the deregister lands, or the chaos window never opened" + ); + assert!( + outcomes.iter().any(|registered| !registered), + "some trial must land the deregister, or the kill delays are all too short" + ); +} diff --git a/knot2/crates/knot-sim/tests/lfs_roundtrip.rs b/knot2/crates/knot-sim/tests/lfs_roundtrip.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-sim/tests/lfs_roundtrip.rs @@ -0,0 +1,1465 @@ +mod common; + +use std::collections::BTreeSet; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::Arc; +use std::time::Duration; + +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use bytes::Bytes; +use http::Method; +use knot_atproto::Atproto; +use knot_cob::{CobHome, CobStore}; +use knot_cobs::{Registration, RegistryChange}; +use knot_edge::RequiresFullHandshake; +use knot_git::{Layout, Repo}; +use knot_lfs::{FreeSpaceFloor, LfsHandle, LfsOid, LfsSize, LfsStore, LfsStorePath}; +use knot_runtime::{ + FakeHttp, HttpResponse, K256Signer, ManualClock, OsEntropy, Signer, UnixMicros, +}; +use knot_secrets::{MasterKey, SealedStore}; +use knot_types::{ + AccountDid, AdmissionPolicy, AuthorName, Email, KnotHostname, KnotId, OwnerDid, RepoDid, + RepoName, RepoRkey, UnixSeconds, +}; +use sha2::{Digest, Sha256}; +use tempfile::TempDir; +use tokio::net::TcpListener; +use tower::ServiceExt; +use url::Url; + +const REPO_DID: &str = "did:plc:squid"; +const REPO_NAME: &str = "anemone"; +const FORK_NAME: &str = "anemone-fork"; +const OWNER_DID: &str = "did:plc:nel"; +const PDS_HOST: &str = "pds.oyster.cafe"; +const KNOT_DID: &str = "did:web:nel.pet"; +const PINNED_DATE: &str = "2026-07-07T12:00:00+00:00"; + +fn require_git_lfs() -> bool { + let available = Command::new("git-lfs") + .arg("version") + .output() + .map(|out| out.status.success()) + .unwrap_or(false); + match (available, std::env::var("KNOT_LFS_ROUNDTRIP").as_deref()) { + (true, _) => true, + (false, Ok("skip")) => { + eprintln!( + "skipping lfs round trip gate: git-lfs unavailable and KNOT_LFS_ROUNDTRIP=skip" + ); + false + } + (false, _) => panic!( + "the lfs round trip gate found no working git-lfs on PATH. \ + Install git-lfs or set KNOT_LFS_ROUNDTRIP=skip to skip the gate." + ), + } +} + +fn media_bytes() -> Vec { + (0..1_048_576u32) + .map(|n| (n.wrapping_mul(31) % 251) as u8) + .collect() +} + +fn second_media_bytes() -> Vec { + (0..524_288u32) + .map(|n| (n.wrapping_mul(97).wrapping_add(13) % 253) as u8) + .collect() +} + +fn require_scutiger() -> bool { + let available = Command::new("git-lfs-transfer") + .arg("--help") + .output() + .map(|out| out.status.success()) + .unwrap_or(false); + match (available, std::env::var("KNOT_LFS_CONFORMANCE").as_deref()) { + (true, _) => true, + (false, Ok("skip")) => { + eprintln!( + "skipping lfs conformance gate: git-lfs-transfer unavailable and \ + KNOT_LFS_CONFORMANCE=skip" + ); + false + } + (false, _) => panic!( + "the lfs conformance gate found no scutiger git-lfs-transfer on PATH. \ + Install it or set KNOT_LFS_CONFORMANCE=skip to skip the gate." + ), + } +} + +fn git(cwd: &Path, env: &[(String, String)], args: &[&str]) -> (bool, String) { + let mut command = knot_fixtures::command_at(cwd, PINNED_DATE); + command.args(args); + env.iter().for_each(|(key, value)| { + command.env(key, value); + }); + let out = command.output().expect("git runs"); + ( + out.status.success(), + format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ), + ) +} + +fn keygen(dir: &Path) -> (String, String) { + let path = dir.join("client"); + let out = Command::new("ssh-keygen") + .args([ + "-t", + "ed25519", + "-N", + "", + "-C", + "nel@oyster.cafe", + "-f", + path.to_str().unwrap(), + ]) + .output() + .expect("ssh-keygen runs"); + assert!(out.status.success()); + let public_line = std::fs::read_to_string(dir.join("client.pub")) + .unwrap() + .trim() + .to_string(); + (path.to_str().unwrap().to_string(), public_line) +} + +fn actor_signer() -> K256Signer { + K256Signer::from_slice(&[9u8; 32]).unwrap() +} + +fn did_document(did: &str) -> Vec { + let multikey = knot_types::crypto::multikey(0xe7, actor_signer().public_key().as_bytes()); + serde_json::to_vec(&serde_json::json!({ + "id": did, + "alsoKnownAs": ["at://nel.pet"], + "verificationMethod": [{ + "id": format!("{did}#atproto"), + "type": "Multikey", + "controller": did, + "publicKeyMultibase": multikey + }], + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": format!("https://{PDS_HOST}") + }] + })) + .unwrap() +} + +fn list_records_body(public_line: &str) -> Vec { + serde_json::to_vec(&serde_json::json!({ + "records": [{ + "uri": format!("at://{OWNER_DID}/sh.tangled.publicKey/1"), + "value": { + "$type": "sh.tangled.publicKey", + "key": public_line, + "name": "laptop", + "createdAt": "2026-07-01T00:00:00Z" + } + }] + })) + .unwrap() +} + +fn fake_http( + published_line: String, +) -> FakeHttp< + impl Fn(&knot_runtime::HttpRequest) -> Result + + Send + + Sync, +> { + FakeHttp::new(move |request: &knot_runtime::HttpRequest| { + let host = request.url.host_str().unwrap_or_default().to_string(); + let path = request.url.path().to_string(); + let body = if host == PDS_HOST { + list_records_body(&published_line) + } else if host == "plc.directory" && request.method == http::Method::POST { + b"{}".to_vec() + } else if host == "plc.directory" && path.starts_with("/did:") { + did_document(path.trim_start_matches('/')) + } else { + return Ok(HttpResponse { + status: http::StatusCode::NOT_FOUND, + headers: http::HeaderMap::new(), + body: bytes::Bytes::new(), + }); + }; + Ok(HttpResponse { + status: http::StatusCode::OK, + headers: http::HeaderMap::new(), + body: bytes::Bytes::from(body), + }) + }) +} + +fn service_jwt(nsid: &str, jti: &str) -> String { + service_jwt_as(OWNER_DID, nsid, jti) +} + +fn service_jwt_as(iss: &str, nsid: &str, jti: &str) -> String { + let header = URL_SAFE_NO_PAD.encode(br#"{"alg":"ES256K","typ":"JWT"}"#); + let claims = serde_json::json!({ + "iss": iss, + "aud": KNOT_DID, + "exp": 1_001, + "iat": 999, + "jti": jti, + "lxm": nsid, + }); + let payload = URL_SAFE_NO_PAD.encode(serde_json::to_vec(&claims).unwrap()); + let signing_input = format!("{header}.{payload}"); + let signature = actor_signer().sign(signing_input.as_bytes()); + format!( + "{signing_input}.{}", + URL_SAFE_NO_PAD.encode(signature.as_bytes()) + ) +} + +struct World { + _scan: TempDir, + lfs: LfsHandle, + ssh_port: u16, + http_base: String, + router: axum::Router, + layout: Layout, + h3: Option, + _certdir: Option, +} + +async fn spawn_world(published_line: String) -> World { + spawn(published_line, false).await +} + +async fn spawn(published_line: String, with_h3: bool) -> World { + let scan = tempfile::tempdir().unwrap(); + let meta_path = scan.path().join("meta"); + Repo::create(&meta_path).unwrap(); + let layout = Layout::new(scan.path().join("repos")); + let repo_did = RepoDid::new(REPO_DID).unwrap(); + layout.create(&repo_did).unwrap(); + + let knot = KnotId::new(KNOT_DID).unwrap(); + let secrets = Arc::new( + SealedStore::open( + scan.path().join("keys.sealed"), + &MasterKey::new([7u8; 32]).unwrap(), + Box::new(OsEntropy), + ) + .unwrap(), + ); + secrets.ensure(&knot).unwrap(); + let knot_signer = secrets.signer(&knot).unwrap(); + + let meta = Repo::open(&meta_path).unwrap(); + CobStore::new(&meta) + .create( + &CobHome::from(&knot), + &RegistryChange::Register(Registration { + owner: OwnerDid::new(OWNER_DID).unwrap(), + rkey: RepoRkey::new(REPO_NAME).unwrap(), + name: RepoName::new(REPO_NAME).unwrap(), + repo: repo_did.clone(), + created_at: UnixSeconds::new(1), + }), + &knot_signer, + UnixSeconds::new(1), + ) + .unwrap(); + + let index = Arc::new(knot_index::Index::new(meta_path.clone(), layout.clone())); + index.rebuild().unwrap(); + + let atproto = Arc::new(Atproto::new( + fake_http(published_line), + ManualClock::new(UnixMicros::new(1_000_000_000)), + knot.clone(), + knot_atproto::PlcDirectory::new(Url::parse("https://plc.directory/").unwrap()).unwrap(), + )); + + let lfs_store_dir = scan.path().join("lfs"); + std::fs::create_dir_all(&lfs_store_dir).unwrap(); + let lfs = LfsHandle::open( + LfsStorePath::new(&lfs_store_dir), + LfsSize::new(64 * 1024 * 1024), + FreeSpaceFloor::new(0), + ) + .unwrap(); + + let key_dir = scan.path().join("hostkey"); + std::fs::create_dir_all(&key_dir).unwrap(); + let host_key = knot_ssh::load_or_create_host_key(&key_dir.join("host")).unwrap(); + let events = Arc::new(knot_events::EventLog::new( + ManualClock::new(UnixMicros::new(1_000_000_000)), + knot_events::ReplayBounds::new( + knot_events::ReplayEvents::new(64).unwrap(), + knot_events::ReplayBytes::new(16 << 20).unwrap(), + ), + )); + let ssh_state = Arc::new( + knot_ssh::SshState::new( + layout.clone(), + Arc::clone(&index), + Arc::clone(&atproto), + knot_types::ActorId::from_secp256k1(actor_signer().public_key().as_bytes()), + Arc::clone(&events), + KnotHostname::new("nel.pet").unwrap(), + knot_types::AppviewEndpoint::new("https://tangled.test").unwrap(), + BTreeSet::from([AccountDid::new(OWNER_DID).unwrap()]), + AdmissionPolicy::Closed, + knot_xrpc::MaxWireBytes::new(1 << 30), + knot_xrpc::LanguagesPushBudget::new(Duration::from_secs(2)), + None, + ) + .with_lfs(lfs.clone(), 16), + ); + let ssh_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let ssh_port = ssh_listener.local_addr().unwrap().port(); + tokio::spawn(async move { + let _ = knot_ssh::serve_on_socket(ssh_listener, host_key, ssh_state).await; + }); + + let http_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let http_base = format!( + "http://127.0.0.1:{}", + http_listener.local_addr().unwrap().port() + ); + + let xrpc_state = Arc::new(knot_xrpc::XrpcState { + ci_logs: None, + layout: layout.clone(), + index: Arc::clone(&index), + atproto, + secrets, + entropy: Arc::new(OsEntropy), + admins: BTreeSet::from([AccountDid::new(OWNER_DID).unwrap()]), + admission: AdmissionPolicy::Closed, + knot_did: knot, + knot_hostname: KnotHostname::new("nel.pet").unwrap(), + meta_path, + knot_service_url: knot_types::KnotServiceUrl::new(http_base.clone()).unwrap(), + limiter: Arc::new(knot_xrpc::PreAuthLimiter::default()), + cob_locks: Arc::new(knot_xrpc::CobLocks::default()), + reservations: Arc::new(knot_xrpc::Reservations::new( + knot_xrpc::ReservationTtl::new(1_000_000), + knot_xrpc::PerActorQuota::new(16), + knot_xrpc::GlobalQuota::new(16), + )), + trusted_proxy_header: None, + committer: knot_xrpc::Committer { + name: AuthorName::new("Tangled"), + email: Email::new("noreply@tangled.sh"), + }, + byte_limits: knot_xrpc::ByteLimits { + pack: knot_xrpc::MaxWireBytes::new(1 << 30), + ..knot_xrpc::ByteLimits::default() + }, + budgets: knot_xrpc::Budgets::default(), + git_http: Arc::new(FakeHttp::new(|_request: &knot_runtime::HttpRequest| { + Err(knot_runtime::NetworkError::Connect( + "no remote upstream is served in this gate".to_string(), + )) + })), + pack_limits: knot_pack::PackLimits::default(), + service_owner: AccountDid::new(OWNER_DID).unwrap(), + events, + subscriber_gate: Arc::new(knot_events::SubscriberGate::new( + knot_events::GlobalSubscriberLimit::new(16), + knot_events::PerPeerSubscriberLimit::new(4), + )), + maintenance: knot_maintenance::MaintenanceHandle::disabled(), + appview: knot_types::AppviewEndpoint::new("https://tangled.test").unwrap(), + slots: knot_resource::Slots::testing(8), + lfs: Some(knot_xrpc::LfsWeb::new(lfs.clone(), 8)), + catalog: Arc::new(knot_messages::Catalog::defaults()), + }); + + let resolver: Arc = { + let index = Arc::clone(&index); + Arc::new(move |target: &knot_pack::RepoTarget| match target { + knot_pack::RepoTarget::Did(did) => match index.owner_of(did) { + knot_index::Resolved::Ready(Some(_)) => knot_pack::RepoLookup::Hosted(did.clone()), + knot_index::Resolved::Ready(None) => knot_pack::RepoLookup::Unhosted, + knot_index::Resolved::Warming => knot_pack::RepoLookup::Unavailable, + }, + knot_pack::RepoTarget::OwnerRkey(owner, rkey) => { + match index.resolve_repo(owner, rkey) { + knot_index::Resolved::Ready(Some(found)) => { + knot_pack::RepoLookup::Hosted(found) + } + knot_index::Resolved::Ready(None) => knot_pack::RepoLookup::Unhosted, + knot_index::Resolved::Warming => knot_pack::RepoLookup::Unavailable, + } + } + }) + }; + let advertiser = knot_xrpc::receive_advertiser(Arc::clone(&xrpc_state)); + let (write_routes, advertisement) = knot_pack::edge_routes( + layout.clone(), + Arc::clone(&resolver), + Some(Arc::clone(&advertiser)), + None, + knot_resource::PackSlots::new(4), + knot_pack::CacheConfig::default(), + Arc::new(knot_messages::Catalog::defaults()), + knot_pack::default_hostname().clone(), + Arc::new(knot_runtime::SystemClock), + ); + let router = write_routes + .merge(advertisement.into_router()) + .merge(knot_xrpc::router(Arc::clone(&xrpc_state))); + let served = router.clone(); + tokio::spawn(async move { + let _ = axum::serve(http_listener, served).await; + }); + + let (h3, certdir) = match with_h3 { + true => { + let certdir = tempfile::tempdir().unwrap(); + let edge = common::serve_edge(certdir.path(), || { + let (write_routes, advertisement) = knot_pack::edge_routes( + layout.clone(), + Arc::clone(&resolver), + Some(Arc::clone(&advertiser)), + None, + knot_resource::PackSlots::new(4), + knot_pack::CacheConfig::default(), + Arc::new(knot_messages::Catalog::defaults()), + knot_pack::default_hostname().clone(), + Arc::new(knot_runtime::SystemClock), + ); + let app = RequiresFullHandshake::new( + write_routes.merge(knot_xrpc::router(Arc::clone(&xrpc_state))), + ); + (app, advertisement) + }) + .await; + (Some(edge), Some(certdir)) + } + false => (None, None), + }; + + World { + _scan: scan, + lfs, + ssh_port, + http_base, + router, + layout, + h3, + _certdir: certdir, + } +} + +async fn in_git_blocking(task: impl FnOnce() -> T + Send + 'static) -> T { + tokio::task::spawn_blocking(task).await.unwrap() +} + +fn seed_lfs_work(work: &Path, env: &[(String, String)], media: &[u8]) { + std::fs::create_dir_all(work).unwrap(); + let steps: [&[&str]; 2] = [ + &["init", "-q", "-b", "main"], + &["lfs", "install", "--local"], + ]; + steps.iter().for_each(|args| { + let (ok, out) = git(work, env, args); + assert!(ok, "{args:?} failed:\n{out}"); + }); + let (ok, out) = git(work, env, &["lfs", "track", "*.bin"]); + assert!(ok, "lfs track failed:\n{out}"); + let (ok, out) = git(work, env, &["config", "lfs.locksverify", "false"]); + assert!(ok, "config failed:\n{out}"); + std::fs::write(work.join("media.bin"), media).unwrap(); + std::fs::write(work.join("README.md"), "media lives in lfs\n").unwrap(); + let commit: [&[&str]; 2] = [&["add", "-A"], &["commit", "-q", "-m", "media"]]; + commit.iter().for_each(|args| { + let (ok, out) = git(work, env, args); + assert!(ok, "{args:?} failed:\n{out}"); + }); +} + +fn clone_and_pull(base: &Path, url: &str, name: &str, env: &[(String, String)]) -> PathBuf { + let skip_smudge: Vec<(String, String)> = env + .iter() + .cloned() + .chain([("GIT_LFS_SKIP_SMUDGE".to_string(), "1".to_string())]) + .collect(); + let (ok, out) = git(base, &skip_smudge, &["clone", "-q", url, name]); + assert!(ok, "anonymous clone of {url} failed:\n{out}"); + let dst = base.join(name); + let pointer = std::fs::read_to_string(dst.join("media.bin")).unwrap(); + assert!( + pointer.contains("git-lfs.github.com/spec/v1"), + "clone must land the pointer before lfs pull, got:\n{pointer}" + ); + let (ok, out) = git(&dst, env, &["lfs", "install", "--local"]); + assert!(ok, "lfs install in {name} failed:\n{out}"); + let (ok, out) = git(&dst, env, &["lfs", "pull"]); + assert!(ok, "git lfs pull in {name} failed:\n{out}"); + dst +} + +async fn create_fork(world: &World, jti: &str) -> (http::StatusCode, serde_json::Value) { + let token = service_jwt("sh.tangled.repo.create", jti); + let body = serde_json::json!({ + "rkey": FORK_NAME, + "name": FORK_NAME, + "source": format!("{}/{OWNER_DID}/{REPO_NAME}", world.http_base), + }); + let request = http::Request::builder() + .method("POST") + .uri("/xrpc/sh.tangled.repo.create") + .header(http::header::CONTENT_TYPE, "application/json") + .header(http::header::AUTHORIZATION, format!("Bearer {token}")) + .body(axum::body::Body::from(serde_json::to_vec(&body).unwrap())) + .unwrap(); + let response = world.router.clone().oneshot(request).await.unwrap(); + let status = response.status(); + let bytes = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + let value = serde_json::from_slice(&bytes).unwrap_or(serde_json::Value::Null); + (status, value) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_lfs_round_trip_gate_holds_over_both_transports_and_the_fork() { + if !require_git_lfs() { + return; + } + let scratch = tempfile::tempdir().unwrap(); + let (key_path, public_line) = keygen(scratch.path()); + let world = spawn_world(public_line).await; + + let media = media_bytes(); + let media_oid = LfsOid::from_digest(Sha256::digest(&media).into()); + let ssh = format!( + "ssh -i {key_path} -o IdentitiesOnly=yes -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null -o PreferredAuthentications=publickey -o BatchMode=yes" + ); + let path_env = std::env::var("PATH").unwrap_or_default(); + let home = scratch.path().to_str().unwrap().to_string(); + let env: Vec<(String, String)> = [ + ("GIT_SSH_COMMAND", &ssh), + ("PATH", &path_env), + ("HOME", &home), + ] + .map(|(key, value)| (key.to_string(), value.clone())) + .to_vec(); + + let work = scratch.path().join("work"); + seed_lfs_work(&work, &env, &media); + + let push_url = format!( + "ssh://git@127.0.0.1:{}/{OWNER_DID}/{REPO_NAME}", + world.ssh_port + ); + let (ok, out) = { + let work = work.clone(); + let env = env.clone(); + in_git_blocking(move || git(&work, &env, &["push", "-q", &push_url, "main"])).await + }; + assert!(ok, "lfs push over ssh failed:\n{out}"); + + let source_repo = RepoDid::new(REPO_DID).unwrap(); + assert_eq!( + world.lfs.store.probe(&source_repo, &media_oid).unwrap(), + Some(LfsSize::new(media.len() as u64)), + "pushed media must be durable in the store" + ); + + let clone_url = format!("{}/{OWNER_DID}/{REPO_NAME}", world.http_base); + let dst = { + let base = scratch.path().to_path_buf(); + let env = env.clone(); + in_git_blocking(move || clone_and_pull(&base, &clone_url, "reader", &env)).await + }; + assert_eq!( + std::fs::read(dst.join("media.bin")).unwrap(), + media, + "anonymous http reader must see byte-identical media" + ); + + let (status, created) = create_fork(&world, "gate-fork-1").await; + assert_eq!( + status, + http::StatusCode::OK, + "fork create failed: {created}" + ); + assert!( + created.get("lfsMissing").is_none(), + "local fork must copy every object, got {created}" + ); + let fork_did = RepoDid::new(created["repoDid"].as_str().unwrap()).unwrap(); + assert_eq!( + world.lfs.store.probe(&fork_did, &media_oid).unwrap(), + Some(LfsSize::new(media.len() as u64)), + "fork prefix must hold its own copy of the media" + ); + + let fork_url = format!("{}/{OWNER_DID}/{FORK_NAME}", world.http_base); + let fork_dst = { + let base = scratch.path().to_path_buf(); + let env = env.clone(); + in_git_blocking(move || clone_and_pull(&base, &fork_url, "fork-reader", &env)).await + }; + assert_eq!( + std::fs::read(fork_dst.join("media.bin")).unwrap(), + media, + "anonymous clone of the fork must see byte-identical media" + ); +} + +fn seed_many_lfs(work: &Path, env: &[(String, String)], count: usize) { + std::fs::create_dir_all(work).unwrap(); + let steps: [&[&str]; 2] = [ + &["init", "-q", "-b", "main"], + &["lfs", "install", "--local"], + ]; + steps.iter().for_each(|args| { + let (ok, out) = git(work, env, args); + assert!(ok, "{args:?} failed:\n{out}"); + }); + let (ok, out) = git(work, env, &["lfs", "track", "*.bin"]); + assert!(ok, "lfs track failed:\n{out}"); + let (ok, out) = git(work, env, &["config", "lfs.locksverify", "false"]); + assert!(ok, "config failed:\n{out}"); + (0..count).for_each(|index| { + let size = 200 + index * 7; + let bytes: Vec = (0..size) + .map(|n| (n.wrapping_mul(31).wrapping_add(index) % 251) as u8) + .collect(); + std::fs::write(work.join(format!("object-{index}.bin")), bytes).unwrap(); + }); + let commit: [&[&str]; 2] = [&["add", "-A"], &["commit", "-q", "-m", "many media"]]; + commit.iter().for_each(|args| { + let (ok, out) = git(work, env, args); + assert!(ok, "{args:?} failed:\n{out}"); + }); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn many_objects_ride_default_git_lfs_concurrency_over_both_transports() { + if !require_git_lfs() { + return; + } + let scratch = tempfile::tempdir().unwrap(); + let (key_path, public_line) = keygen(scratch.path()); + let world = spawn_world(public_line).await; + + let ssh = format!( + "ssh -i {key_path} -o IdentitiesOnly=yes -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null -o PreferredAuthentications=publickey -o BatchMode=yes" + ); + let path_env = std::env::var("PATH").unwrap_or_default(); + let home = scratch.path().to_str().unwrap().to_string(); + let env: Vec<(String, String)> = [ + ("GIT_SSH_COMMAND", &ssh), + ("PATH", &path_env), + ("HOME", &home), + ] + .map(|(key, value)| (key.to_string(), value.clone())) + .to_vec(); + + let count = 25usize; + let work = scratch.path().join("work"); + seed_many_lfs(&work, &env, count); + + let push_url = format!( + "ssh://git@127.0.0.1:{}/{OWNER_DID}/{REPO_NAME}", + world.ssh_port + ); + let (ok, out) = { + let work = work.clone(); + let env = env.clone(); + in_git_blocking(move || git(&work, &env, &["push", "-q", &push_url, "main"])).await + }; + assert!( + ok, + "git-lfs at its default concurrency must push {count} objects over ssh without tripping \ + the per-peer connection limit:\n{out}" + ); + + let clone_url = format!("{}/{OWNER_DID}/{REPO_NAME}", world.http_base); + let dst = { + let base = scratch.path().to_path_buf(); + let env = env.clone(); + in_git_blocking(move || { + let skip_smudge: Vec<(String, String)> = env + .iter() + .cloned() + .chain([("GIT_LFS_SKIP_SMUDGE".to_string(), "1".to_string())]) + .collect(); + let (ok, out) = git(&base, &skip_smudge, &["clone", "-q", &clone_url, "reader"]); + assert!(ok, "anonymous clone failed:\n{out}"); + let dst = base.join("reader"); + let (ok, out) = git(&dst, &env, &["lfs", "install", "--local"]); + assert!(ok, "lfs install failed:\n{out}"); + let (ok, out) = git(&dst, &env, &["lfs", "pull"]); + assert!( + ok, + "anonymous http pull of {count} objects mustn't be throttled by the xrpc \ + pre-auth limiter:\n{out}" + ); + dst + }) + .await + }; + (0..count).for_each(|index| { + assert_eq!( + std::fs::read(work.join(format!("object-{index}.bin"))).unwrap(), + std::fs::read(dst.join(format!("object-{index}.bin"))).unwrap(), + "object-{index}.bin must be byte-identical over anonymous http" + ); + }); +} + +fn write_shim(dir: &Path) -> String { + let shim = dir.join("local-ssh.sh"); + std::fs::write( + &shim, + "#!/bin/sh\n\ + while [ \"$#\" -gt 0 ]; do\n\ + case \"$1\" in\n\ + -o|-p) shift 2 ;;\n\ + -*) shift ;;\n\ + *) break ;;\n\ + esac\n\ + done\n\ + shift\n\ + eval exec \"$@\"\n", + ) + .unwrap(); + let mut permissions = std::fs::metadata(&shim).unwrap().permissions(); + std::os::unix::fs::PermissionsExt::set_mode(&mut permissions, 0o755); + std::fs::set_permissions(&shim, permissions).unwrap(); + shim.to_str().unwrap().to_string() +} + +fn hex_object_files(root: &Path) -> Vec<(String, u64, PathBuf)> { + let entries = match std::fs::read_dir(root) { + Ok(entries) => entries, + Err(_) => return Vec::new(), + }; + entries + .filter_map(Result::ok) + .flat_map(|entry| { + let path = entry.path(); + if path.is_dir() { + return hex_object_files(&path); + } + path.file_name() + .and_then(|name| name.to_str()) + .filter(|name| LfsOid::new(*name).is_ok()) + .map(|name| { + let size = std::fs::metadata(&path).map(|meta| meta.len()).unwrap_or(0); + vec![(name.to_string(), size, path.clone())] + }) + .unwrap_or_default() + }) + .collect() +} + +fn pull_verdict(base: &Path, url: &str, name: &str, env: &[(String, String)]) -> (bool, String) { + let skip_smudge: Vec<(String, String)> = env + .iter() + .cloned() + .chain([("GIT_LFS_SKIP_SMUDGE".to_string(), "1".to_string())]) + .collect(); + let (ok, out) = git(base, &skip_smudge, &["clone", "-q", url, name]); + assert!(ok, "clone of {url} failed:\n{out}"); + let dst = base.join(name); + let (ok, out) = git(&dst, env, &["lfs", "install", "--local"]); + assert!(ok, "lfs install in {name} failed:\n{out}"); + git(&dst, env, &["lfs", "pull"]) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_lfs_stack_is_conformant_with_the_reference_server_and_client() { + if !require_git_lfs() || !require_scutiger() { + return; + } + let scratch = tempfile::tempdir().unwrap(); + let (key_path, public_line) = keygen(scratch.path()); + let world = spawn_world(public_line).await; + + let media = media_bytes(); + let second = second_media_bytes(); + let media_oid = LfsOid::from_digest(Sha256::digest(&media).into()); + let second_oid = LfsOid::from_digest(Sha256::digest(&second).into()); + let expected: std::collections::BTreeSet<(String, u64)> = [ + (media_oid.as_str().to_string(), media.len() as u64), + (second_oid.as_str().to_string(), second.len() as u64), + ] + .into(); + + let ssh = format!( + "ssh -i {key_path} -o IdentitiesOnly=yes -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null -o PreferredAuthentications=publickey -o BatchMode=yes" + ); + let path_env = std::env::var("PATH").unwrap_or_default(); + let home = scratch.path().to_str().unwrap().to_string(); + let knot_env: Vec<(String, String)> = [ + ("GIT_SSH_COMMAND", &ssh), + ("PATH", &path_env), + ("HOME", &home), + ] + .map(|(key, value)| (key.to_string(), value.clone())) + .to_vec(); + let shim = write_shim(scratch.path()); + let reference_env: Vec<(String, String)> = [ + ("GIT_SSH_COMMAND", &shim), + ("PATH", &path_env), + ("HOME", &home), + ] + .map(|(key, value)| (key.to_string(), value.clone())) + .to_vec(); + + let upstream = scratch.path().join("reference-upstream.git"); + let (ok, out) = git( + scratch.path(), + &reference_env, + &[ + "init", + "-q", + "--bare", + "-b", + "main", + upstream.to_str().unwrap(), + ], + ); + assert!(ok, "reference upstream init failed:\n{out}"); + + let work = scratch.path().join("work"); + seed_lfs_work(&work, &knot_env, &media); + std::fs::write(work.join("extra.bin"), &second).unwrap(); + let commit: [&[&str]; 2] = [&["add", "-A"], &["commit", "-q", "-m", "extra media"]]; + commit.iter().for_each(|args| { + let (ok, out) = git(&work, &knot_env, args); + assert!(ok, "{args:?} failed:\n{out}"); + }); + + let knot_push_url = format!( + "ssh://git@127.0.0.1:{}/{OWNER_DID}/{REPO_NAME}", + world.ssh_port + ); + let reference_push_url = format!("ssh://ref@localhost{}", upstream.display()); + let pushes = { + let work = work.clone(); + let knot_env = knot_env.clone(); + let reference_env = reference_env.clone(); + let knot_push_url = knot_push_url.clone(); + let reference_push_url = reference_push_url.clone(); + in_git_blocking(move || { + [ + git(&work, &knot_env, &["push", "-q", &knot_push_url, "main"]), + git( + &work, + &reference_env, + &["push", "-q", &reference_push_url, "main"], + ), + ] + }) + .await + }; + pushes.iter().for_each(|(ok, out)| { + assert!(ok, "push failed:\n{out}"); + }); + + let source_repo = RepoDid::new(REPO_DID).unwrap(); + let knot_objects: std::collections::BTreeSet<(String, u64)> = world + .lfs + .store + .enumerate(&source_repo) + .unwrap() + .into_iter() + .map(|object| (object.oid.as_str().to_string(), object.size.get())) + .collect(); + let reference_objects: std::collections::BTreeSet<(String, u64)> = hex_object_files(&upstream) + .into_iter() + .map(|(name, size, _)| (name, size)) + .collect(); + assert_eq!( + knot_objects, expected, + "the knot store holds exactly the pushed object set" + ); + assert_eq!( + knot_objects, reference_objects, + "both servers hold identical object sets after the same push" + ); + + let knot_clone_url = format!("{}/{OWNER_DID}/{REPO_NAME}", world.http_base); + let (knot_dst, reference_dst) = { + let base = scratch.path().to_path_buf(); + let knot_env = knot_env.clone(); + let reference_env = reference_env.clone(); + let knot_clone_url = knot_clone_url.clone(); + let reference_push_url = reference_push_url.clone(); + in_git_blocking(move || { + ( + clone_and_pull(&base, &knot_clone_url, "knot-reader", &knot_env), + clone_and_pull( + &base, + &reference_push_url, + "reference-reader", + &reference_env, + ), + ) + }) + .await + }; + ["media.bin", "extra.bin"].iter().for_each(|file| { + assert_eq!( + std::fs::read(knot_dst.join(file)).unwrap(), + std::fs::read(reference_dst.join(file)).unwrap(), + "{file}: both servers must check out identical media" + ); + }); + assert_eq!(std::fs::read(knot_dst.join("media.bin")).unwrap(), media); + assert_eq!(std::fs::read(knot_dst.join("extra.bin")).unwrap(), second); + + let knot_removed = world + .lfs + .store + .object_file(&source_repo, &second_oid) + .unwrap() + .unwrap() + .1; + std::fs::remove_file(knot_removed).unwrap(); + let removed = hex_object_files(&upstream) + .into_iter() + .filter(|(name, _, _)| name == second_oid.as_str()) + .map(|(_, _, path)| std::fs::remove_file(path).unwrap()) + .count(); + assert!( + removed > 0, + "the reference server holds the object to remove" + ); + + let verdicts = { + let base = scratch.path().to_path_buf(); + let knot_env = knot_env.clone(); + let reference_env = reference_env.clone(); + let knot_push_url = knot_push_url.clone(); + in_git_blocking(move || { + [ + pull_verdict(&base, &knot_clone_url, "knot-missing-http", &knot_env), + pull_verdict(&base, &knot_push_url, "knot-missing-ssh", &knot_env), + pull_verdict( + &base, + &reference_push_url, + "reference-missing", + &reference_env, + ), + ] + }) + .await + }; + let [(http_ok, http_out), (ssh_ok, ssh_out), (reference_ok, _)] = verdicts; + assert!( + !http_ok, + "an http pull of a missing object must fail loudly, never succeed silently:\n{http_out}" + ); + assert!( + !ssh_ok, + "an ssh pull of a missing object must fail loudly, never succeed silently:\n{ssh_out}" + ); + assert!( + reference_ok, + "scutiger 0.3.0 answers noop for a missing download and the client silently \ + succeeds. This pin is the recorded reason knot answers download instead, so its \ + get-object 404 turns the pull into a loud failure. If the reference starts failing \ + loudly too, the divergence note can be retired." + ); +} + +async fn lfs_batch( + world: &World, + auth: Option<&str>, + op: &str, + oid: &LfsOid, + size: u64, +) -> (http::StatusCode, serde_json::Value) { + let body = serde_json::json!({ + "operation": op, + "transfers": ["basic"], + "objects": [{ "oid": oid.as_str(), "size": size }], + "hash_algo": "sha256", + }); + let mut builder = http::Request::builder() + .method("POST") + .uri(format!("/{OWNER_DID}/{REPO_NAME}/info/lfs/objects/batch")) + .header(http::header::CONTENT_TYPE, "application/vnd.git-lfs+json"); + if let Some(auth) = auth { + builder = builder.header(http::header::AUTHORIZATION, auth); + } + let request = builder + .body(axum::body::Body::from(serde_json::to_vec(&body).unwrap())) + .unwrap(); + let response = world.router.clone().oneshot(request).await.unwrap(); + let status = response.status(); + let bytes = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + ( + status, + serde_json::from_slice(&bytes).unwrap_or(serde_json::Value::Null), + ) +} + +async fn lfs_put( + world: &World, + auth: Option<&str>, + oid: &LfsOid, + bytes: Vec, +) -> http::StatusCode { + let mut builder = http::Request::builder() + .method("PUT") + .uri(format!( + "/{OWNER_DID}/{REPO_NAME}/info/lfs/objects/{}", + oid.as_str() + )) + .header(http::header::CONTENT_LENGTH, bytes.len()); + if let Some(auth) = auth { + builder = builder.header(http::header::AUTHORIZATION, auth); + } + let request = builder.body(axum::body::Body::from(bytes)).unwrap(); + world + .router + .clone() + .oneshot(request) + .await + .unwrap() + .status() +} + +fn basic_auth(token: &str) -> String { + let raw = base64::engine::general_purpose::STANDARD.encode(format!("x-tangled-token:{token}")); + format!("Basic {raw}") +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn lfs_http_push_stores_an_object_with_a_push_token() { + let scratch = tempfile::tempdir().unwrap(); + let (_key_path, public_line) = keygen(scratch.path()); + let world = spawn_world(public_line).await; + let repo = RepoDid::new(REPO_DID).unwrap(); + + let payload: Vec = (0..4096u32) + .map(|n| (n.wrapping_mul(17) % 251) as u8) + .collect(); + let oid = LfsOid::from_digest(Sha256::digest(&payload).into()); + let size = payload.len() as u64; + + let bearer = format!( + "Bearer {}", + service_jwt("sh.tangled.repo.push", "lfs-http-push-1") + ); + let (status, body) = lfs_batch(&world, Some(&bearer), "upload", &oid, size).await; + assert_eq!( + status, + http::StatusCode::OK, + "authenticated upload batch: {body}" + ); + let href = body["objects"][0]["actions"]["upload"]["href"] + .as_str() + .unwrap_or_else(|| panic!("expected an upload action, got {body}")); + assert_eq!( + href, + format!( + "{}/{OWNER_DID}/{REPO_NAME}/info/lfs/objects/{}", + world.http_base, + oid.as_str() + ), + "upload href points at the object route on this knot" + ); + assert_ne!( + body["objects"][0]["authenticated"], + serde_json::Value::Bool(true), + "upload objects mustn't claim authenticated=true, else git-lfs sends the object put with no auth and loops on 401: {body}" + ); + assert!( + world.lfs.store.probe(&repo, &oid).unwrap().is_none(), + "object must be absent before the put" + ); + + let put = lfs_put(&world, Some(&bearer), &oid, payload.clone()).await; + assert_eq!( + put, + http::StatusCode::OK, + "the same push token must authorize both the batch and the object put" + ); + assert_eq!( + world.lfs.store.probe(&repo, &oid).unwrap(), + Some(LfsSize::new(size)), + "the put object must be durable in the store" + ); + + let get = http::Request::builder() + .method("GET") + .uri(format!( + "/{OWNER_DID}/{REPO_NAME}/info/lfs/objects/{}", + oid.as_str() + )) + .body(axum::body::Body::empty()) + .unwrap(); + let response = world.router.clone().oneshot(get).await.unwrap(); + assert_eq!( + response.status(), + http::StatusCode::OK, + "anonymous download" + ); + let served = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + assert_eq!( + served.as_ref(), + payload.as_slice(), + "an anonymous reader sees the byte-identical object a push stored" + ); + + let payload2: Vec = (0..2048u32) + .map(|n| (n.wrapping_mul(29) % 251) as u8) + .collect(); + let oid2 = LfsOid::from_digest(Sha256::digest(&payload2).into()); + let basic = basic_auth(&service_jwt("sh.tangled.repo.push", "lfs-http-push-2")); + let (status, body) = + lfs_batch(&world, Some(&basic), "upload", &oid2, payload2.len() as u64).await; + assert_eq!( + status, + http::StatusCode::OK, + "basic-auth upload batch: {body}" + ); + let put = lfs_put(&world, Some(&basic), &oid2, payload2.clone()).await; + assert_eq!( + put, + http::StatusCode::OK, + "a push token presented as the http basic password must authenticate the put" + ); + assert_eq!( + world.lfs.store.probe(&repo, &oid2).unwrap(), + Some(LfsSize::new(payload2.len() as u64)), + "the basic-authenticated object must be durable too" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn lfs_http_push_rejects_missing_and_mismatched_credentials() { + let scratch = tempfile::tempdir().unwrap(); + let (_key_path, public_line) = keygen(scratch.path()); + let world = spawn_world(public_line).await; + + let payload: Vec = (0..1024u32) + .map(|n| (n.wrapping_mul(13) % 251) as u8) + .collect(); + let oid = LfsOid::from_digest(Sha256::digest(&payload).into()); + let size = payload.len() as u64; + + let (status, _) = lfs_batch(&world, None, "upload", &oid, size).await; + assert_eq!( + status, + http::StatusCode::UNAUTHORIZED, + "an unauthenticated upload batch is challenged" + ); + + let wrong_method = format!( + "Bearer {}", + service_jwt("sh.tangled.repo.create", "lfs-http-neg-method") + ); + let (status, _) = lfs_batch(&world, Some(&wrong_method), "upload", &oid, size).await; + assert_eq!( + status, + http::StatusCode::UNAUTHORIZED, + "a token bound to another method cannot authorize a push" + ); + + let stranger = format!( + "Bearer {}", + service_jwt_as(REPO_DID, "sh.tangled.repo.push", "lfs-http-neg-acl") + ); + let (status, _) = lfs_batch(&world, Some(&stranger), "upload", &oid, size).await; + assert_eq!( + status, + http::StatusCode::FORBIDDEN, + "a valid push token from a did that cannot push is refused by the acl" + ); + + let put = lfs_put(&world, None, &oid, payload).await; + assert_eq!( + put, + http::StatusCode::UNAUTHORIZED, + "an unauthenticated object put is challenged" + ); + assert!( + world + .lfs + .store + .probe(&RepoDid::new(REPO_DID).unwrap(), &oid) + .unwrap() + .is_none(), + "no rejected request may leave an object behind" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn git_push_over_http_authenticates_and_lands_the_ref() { + let scratch = tempfile::tempdir().unwrap(); + let (_key_path, public_line) = keygen(scratch.path()); + let world = spawn_world(public_line).await; + + let path_env = std::env::var("PATH").unwrap_or_default(); + let home = scratch.path().to_str().unwrap().to_string(); + let env: Vec<(String, String)> = [("PATH", &path_env), ("HOME", &home)] + .map(|(key, value)| (key.to_string(), value.clone())) + .to_vec(); + + let work = scratch.path().join("work"); + std::fs::create_dir_all(&work).unwrap(); + let (ok, out) = git(&work, &env, &["init", "-q", "-b", "main"]); + assert!(ok, "init failed:\n{out}"); + std::fs::write(work.join("README.md"), "hello over http\n").unwrap(); + let (ok, out) = git(&work, &env, &["add", "-A"]); + assert!(ok, "add failed:\n{out}"); + let (ok, out) = git(&work, &env, &["commit", "-q", "-m", "init over http"]); + assert!(ok, "commit failed:\n{out}"); + + let url = format!("{}/{OWNER_DID}/{REPO_NAME}", world.http_base); + + let (ok, out) = { + let work = work.clone(); + let env = env.clone(); + let url = url.clone(); + in_git_blocking(move || git(&work, &env, &["push", "-q", &url, "main"])).await + }; + assert!( + !ok, + "an unauthenticated http push must be refused, git reported success:\n{out}" + ); + + let token = service_jwt("sh.tangled.repo.push", "git-http-push-1"); + let header = format!( + "http.extraHeader=Authorization: Basic {}", + base64::engine::general_purpose::STANDARD.encode(format!("x-tangled-token:{token}")) + ); + let (ok, out) = { + let work = work.clone(); + let env = env.clone(); + let url = url.clone(); + let header = header.clone(); + in_git_blocking(move || git(&work, &env, &["-c", &header, "push", "-q", &url, "main"])) + .await + }; + assert!(ok, "authenticated http push failed:\n{out}"); + + let (ok, refs) = { + let scratch = scratch.path().to_path_buf(); + let env = env.clone(); + let url = url.clone(); + in_git_blocking(move || git(&scratch, &env, &["ls-remote", &url])).await + }; + assert!( + ok && refs.contains("refs/heads/main"), + "the pushed ref must be advertised to an anonymous reader:\n{refs}" + ); + + let wrong = format!( + "http.extraHeader=Authorization: Basic {}", + base64::engine::general_purpose::STANDARD.encode(format!( + "x-tangled-token:{}", + service_jwt("sh.tangled.repo.create", "git-http-push-neg") + )) + ); + std::fs::write(work.join("README.md"), "second write\n").unwrap(); + let (ok, out) = git(&work, &env, &["commit", "-q", "-am", "second"]); + assert!(ok, "second commit failed:\n{out}"); + let (ok, out) = { + let work = work.clone(); + let env = env.clone(); + let url = url.clone(); + let wrong = wrong.clone(); + in_git_blocking(move || git(&work, &env, &["-c", &wrong, "push", "-q", &url, "main"])).await + }; + assert!( + !ok, + "a token bound to another method mustn't authorize a push:\n{out}" + ); +} + +fn build_pack(work: &Path, env: &[(String, String)], tip: &str) -> Vec { + let mut command = knot_fixtures::command(work); + command + .args(["pack-objects", "--revs", "--stdout", "--delta-base-offset"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + env.iter().for_each(|(key, value)| { + command.env(key, value); + }); + let mut child = command.spawn().expect("git pack-objects spawns"); + child + .stdin + .take() + .unwrap() + .write_all(format!("{tip}\n").as_bytes()) + .unwrap(); + let out = child.wait_with_output().unwrap(); + assert!( + out.status.success(), + "git pack-objects failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + out.stdout +} + +fn build_receive_body(tip: &str, pack: &[u8]) -> Bytes { + let mut command = format!("{} {tip} refs/heads/main", "0".repeat(tip.len())).into_bytes(); + command.push(0); + command.extend_from_slice(b"report-status side-band-64k agent=knot-h3-test/0"); + command.push(b'\n'); + let mut body = common::pkt(&command); + body.extend_from_slice(b"0000"); + body.extend_from_slice(pack); + Bytes::from(body) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn git_push_over_h3_authenticates_and_lands_the_ref() { + let scratch = tempfile::tempdir().unwrap(); + let (_key_path, public_line) = keygen(scratch.path()); + let world = spawn(public_line, true).await; + let edge = world.h3.as_ref().expect("the h3 edge is stood up"); + + let path_env = std::env::var("PATH").unwrap_or_default(); + let home = scratch.path().to_str().unwrap().to_string(); + let env: Vec<(String, String)> = [("PATH", &path_env), ("HOME", &home)] + .map(|(key, value)| (key.to_string(), value.clone())) + .to_vec(); + + let work = scratch.path().join("work"); + std::fs::create_dir_all(&work).unwrap(); + let (ok, out) = git(&work, &env, &["init", "-q", "-b", "main"]); + assert!(ok, "init failed:\n{out}"); + std::fs::write(work.join("README.md"), "hello over http3\n").unwrap(); + let (ok, out) = git(&work, &env, &["add", "-A"]); + assert!(ok, "add failed:\n{out}"); + let (ok, out) = git(&work, &env, &["commit", "-q", "-m", "init over http3"]); + assert!(ok, "commit failed:\n{out}"); + let (ok, tip) = git(&work, &env, &["rev-parse", "HEAD"]); + assert!(ok, "rev-parse failed:\n{tip}"); + let tip = tip.trim().to_string(); + let pack = build_pack(&work, &env, &tip); + + let advert_uri = + format!("https://localhost/{OWNER_DID}/{REPO_NAME}/info/refs?service=git-receive-pack"); + let receive_uri = format!("https://localhost/{OWNER_DID}/{REPO_NAME}/git-receive-pack"); + let warmup = + format!("https://localhost/{OWNER_DID}/{REPO_NAME}/info/refs?service=git-upload-pack"); + const RECEIVE_CT: &str = "application/x-git-receive-pack-request"; + + let (status, _) = + common::h3_request(edge, Method::GET, advert_uri.clone(), &[], None, None).await; + assert_eq!( + status, + http::StatusCode::UNAUTHORIZED, + "an unauthenticated receive advertisement must be challenged over h3" + ); + + let token = basic_auth(&service_jwt("sh.tangled.repo.push", "git-h3-adv-1")); + let (status, advert) = common::h3_request( + edge, + Method::GET, + advert_uri, + &[("authorization", token.as_str())], + None, + None, + ) + .await; + assert_eq!( + status, + http::StatusCode::OK, + "an authenticated receive advertisement is served over h3" + ); + assert!( + String::from_utf8_lossy(&advert).contains("# service=git-receive-pack"), + "the h3 receive advertisement includes the service banner" + ); + + let body = build_receive_body(&tip, &pack); + let (status, _) = common::h3_request( + edge, + Method::POST, + receive_uri.clone(), + &[("content-type", RECEIVE_CT)], + Some(body.clone()), + Some(warmup.as_str()), + ) + .await; + assert_eq!( + status, + http::StatusCode::UNAUTHORIZED, + "an unauthenticated receive-pack post must be challenged over h3" + ); + + let wrong = basic_auth(&service_jwt("sh.tangled.repo.create", "git-h3-neg")); + let (status, _) = common::h3_request( + edge, + Method::POST, + receive_uri.clone(), + &[ + ("content-type", RECEIVE_CT), + ("authorization", wrong.as_str()), + ], + Some(body.clone()), + Some(warmup.as_str()), + ) + .await; + assert_eq!( + status, + http::StatusCode::UNAUTHORIZED, + "a token bound to another method cannot authorize a receive-pack over h3" + ); + + let good = basic_auth(&service_jwt("sh.tangled.repo.push", "git-h3-push-1")); + let (status, report) = common::h3_request( + edge, + Method::POST, + receive_uri, + &[ + ("content-type", RECEIVE_CT), + ("authorization", good.as_str()), + ], + Some(body), + Some(warmup.as_str()), + ) + .await; + assert_eq!(status, http::StatusCode::OK, "the authenticated h3 push"); + let report = String::from_utf8_lossy(&report); + assert!( + report.contains("unpack ok"), + "the pack must unpack cleanly over h3:\n{report}" + ); + assert!( + report.contains("ok refs/heads/main"), + "the ref update must be accepted over h3:\n{report}" + ); + + let repo = world.layout.open(&RepoDid::new(REPO_DID).unwrap()).unwrap(); + let landed = repo.references().unwrap().into_iter().any(|record| { + record.name.as_str() == "refs/heads/main" && record.target.to_string() == tip + }); + assert!( + landed, + "the ref pushed over h3 must be durable in the bare repo" + ); +} diff --git a/knot2/crates/knot-sim/tests/reproducible.rs b/knot2/crates/knot-sim/tests/reproducible.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-sim/tests/reproducible.rs @@ -0,0 +1,238 @@ +use std::collections::BTreeSet; + +use futures::future::join_all; +use futures::stream::StreamExt; +use knot_sim::{Outcome, Step, Trace}; + +fn status_of(step: &Step) -> Option { + match step.outcome { + Outcome::Answered { status, .. } => Some(status.get()), + Outcome::Killed => None, + } +} + +fn union_has(traces: &[Trace], predicate: impl Fn(&Step) -> bool) -> bool { + traces + .iter() + .any(|trace| trace.steps.iter().any(&predicate)) +} + +const READ_OPS: [&str; 11] = [ + "version", + "owner", + "listMembers", + "didJson", + "infoRefs", + "branches", + "log", + "describeRepo", + "tree", + "blob", + "languages", +]; + +fn every_step_with<'a>( + traces: &'a [Trace], + fault: &'a str, + holds: impl Fn(&Step) -> bool + 'a, +) -> (bool, usize) { + let matching: Vec<&Step> = traces + .iter() + .flat_map(|trace| trace.steps.iter()) + .filter(|step| step.fault == fault) + .collect(); + (matching.iter().all(|step| holds(step)), matching.len()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_same_seed_produces_an_identical_whole_system_trace() { + futures::stream::iter([1u64, 7, 42, 100, 2026]) + .for_each(|seed| async move { + let first = knot_sim::run(seed, 16).await; + let second = knot_sim::run(seed, 16).await; + assert_eq!( + first, second, + "seed {seed} must replay to an identical trace" + ); + assert_eq!( + first.digest(), + second.digest(), + "seed {seed} digest is stable" + ); + }) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn concurrent_execution_stays_stable_across_fifty_repeats() { + let baseline = knot_sim::run(7, 18).await.digest(); + let digests: Vec = futures::stream::iter(0..50) + .then(|_| knot_sim::run(7, 18)) + .map(|trace| trace.digest()) + .collect() + .await; + assert!( + digests.iter().all(|digest| *digest == baseline), + "4-thread schedule leaked into trace: {} of 50 replays diverged", + digests.iter().filter(|digest| **digest != baseline).count() + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn distinct_seeds_produce_distinct_traces() { + let digests: Vec = join_all((0u64..8).map(|seed| knot_sim::run(seed, 16))) + .await + .iter() + .map(Trace::digest) + .collect(); + let unique: BTreeSet = digests.iter().copied().collect(); + assert_eq!( + unique.len(), + digests.len(), + "every seed must drive different whole-system trace: {digests:?}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_round_count_extends_the_same_prefix() { + let short = knot_sim::run(7, 10).await; + let long = knot_sim::run(7, 20).await; + assert_ne!(short.digest(), long.digest()); + assert_eq!( + short.steps, + long.steps[..short.steps.len()], + "longer run extends the same prefix the shorter run produced" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_injected_failures_are_correlated_with_their_observable_outcomes() { + let traces = join_all([1u64, 7, 13, 42, 99, 2026].map(|seed| knot_sim::run(seed, 16))).await; + + let (skew_all_401, skew_count) = + every_step_with(&traces, "clock_skew", |step| status_of(step) == Some(401)); + assert!(skew_count > 0, "clock-skew fault must actually fire"); + assert!( + skew_all_401, + "every clock-skewed token must produce 401 expiry rejection, not just one example" + ); + + let (drop_all_503, drop_count) = every_step_with(&traces, "drop_identity", |step| { + status_of(step) == Some(503) + }); + assert!(drop_count > 0, "drop-identity fault must actually fire"); + assert!( + drop_all_503, + "every dropped identity resolution must surface as 503 upstream-unavailable failure" + ); + + let (killed_all_killed, killed_count) = every_step_with(&traces, "killed", |step| { + matches!(step.outcome, Outcome::Killed) && READ_OPS.contains(&step.op) + }); + assert!(killed_count > 0, "kill fault must actually fire"); + assert!( + killed_all_killed, + "killed connection must record a Killed outcome and is only injected on read paths" + ); + + assert!( + union_has(&traces, |step| step.op == "probe" + && step.fault == "none" + && status_of(step) == Some(403)), + "resolved stranger with no fault must be denied 403 by access-control layer" + ); + + [ + "addMember", + "addCollaborator", + "createRepo", + "maintain", + "describeRepo", + "listMembers", + "infoRefs", + "didJson", + "branches", + "log", + "tree", + "blob", + "languages", + ] + .iter() + .for_each(|op| { + assert!( + union_has(&traces, |step| step.op == *op + && status_of(step) == Some(200)), + "{op} path must succeed against the doubles in at least one seed" + ); + }); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_final_projection_matches_an_independent_model() { + let seeds = [1u64, 7, 13, 42, 99, 2026]; + let predicted: Vec = seeds + .iter() + .map(|seed| knot_sim::predict(*seed, 18)) + .collect(); + let runs = join_all(seeds.map(|seed| knot_sim::run(seed, 18))).await; + + seeds + .iter() + .zip(predicted.iter()) + .zip(runs.iter()) + .for_each(|((seed, model), trace)| { + let last = trace.snapshots.last().expect("at least one round"); + assert_eq!( + last.members, model.members, + "seed {seed}: executed member projection must equal the independent model" + ); + assert_eq!( + last.blocked, model.blocked, + "seed {seed}: executed blocklist projection must equal the independent model" + ); + }); + + assert!( + predicted.iter().any(|model| !model.members.is_empty()), + "oracle is vacuous unless at least one seed predicts a non-empty member set" + ); + assert!( + predicted.iter().any(|model| !model.blocked.is_empty()), + "oracle is vacuous unless at least one seed predicts a non-empty blocklist" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn maintenance_never_reports_a_failure() { + let traces = join_all([1u64, 7, 13, 42, 99, 2026].map(|seed| knot_sim::run(seed, 16))).await; + let failures = traces + .iter() + .flat_map(|trace| trace.steps.iter()) + .filter(|step| step.op == "maintain" && status_of(step) == Some(500)) + .count(); + assert_eq!( + failures, 0, + "maintenance must succeed on every repo simulation runs it against" + ); + assert!( + union_has(&traces, |step| step.op == "maintain" + && status_of(step) == Some(200)), + "maintenance path must actually run against a populated repo in at least one seed" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_final_projection_state_is_seed_stable() { + let first = knot_sim::run(2026, 16).await; + let second = knot_sim::run(2026, 16).await; + assert_eq!( + first.snapshots, second.snapshots, + "order-independent COB projections must converge to the same logical state" + ); + let last = first.snapshots.last().expect("at least one round"); + assert!( + last.repos.len() > 1, + "simulation must have minted repos beyond the seed repo: {:?}", + last.repos + ); +} diff --git a/knot2/crates/knot-sim/tests/ssh.rs b/knot2/crates/knot-sim/tests/ssh.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-sim/tests/ssh.rs @@ -0,0 +1,322 @@ +use std::path::Path; +use std::process::Command; +use std::sync::Arc; + +use futures::stream::StreamExt; +use knot_atproto::Atproto; +use knot_cob::{CobHome, CobStore}; +use knot_cobs::{Registration, RegistryChange}; +use knot_git::{Layout, Repo}; +use knot_runtime::{ + FakeHttp, HttpResponse, K256Signer, ManualClock, SeededEntropy, Signer, UnixMicros, +}; +use knot_types::{KnotId, OwnerDid, RepoDid, RepoName, RepoRkey, UnixSeconds}; +use tempfile::TempDir; +use tokio::net::TcpListener; +use url::Url; + +const REPO_DID: &str = "did:plc:squid"; +const REPO_NAME: &str = "anemone"; +const OWNER_DID: &str = "did:plc:nel"; +const PDS_HOST: &str = "pds.oyster.cafe"; +const KNOT_DID: &str = "did:web:nel.pet"; +const PINNED_DATE: &str = "2026-06-20T12:00:00+00:00"; + +fn git(cwd: &Path, env: &[(&str, &str)], args: &[&str]) -> (bool, String) { + let mut command = knot_fixtures::command_at(cwd, PINNED_DATE); + command.args(args); + env.iter().for_each(|(key, value)| { + command.env(key, value); + }); + let out = command.output().expect("git runs"); + ( + out.status.success(), + format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ), + ) +} + +fn keygen(dir: &Path) -> (String, String) { + let path = dir.join("client"); + let out = Command::new("ssh-keygen") + .args([ + "-t", + "ed25519", + "-N", + "", + "-C", + "nel@oyster.cafe", + "-f", + path.to_str().unwrap(), + ]) + .output() + .expect("ssh-keygen runs"); + assert!(out.status.success()); + let public_line = std::fs::read_to_string(dir.join("client.pub")) + .unwrap() + .trim() + .to_string(); + (path.to_str().unwrap().to_string(), public_line) +} + +fn did_document(signer: &K256Signer, did: &str) -> Vec { + let multikey = knot_types::crypto::multikey(0xe7, signer.public_key().as_bytes()); + serde_json::to_vec(&serde_json::json!({ + "id": did, + "alsoKnownAs": ["at://nel.pet"], + "verificationMethod": [{ + "id": format!("{did}#atproto"), + "type": "Multikey", + "controller": did, + "publicKeyMultibase": multikey + }], + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": format!("https://{PDS_HOST}") + }] + })) + .unwrap() +} + +fn list_records_body(public_line: &str) -> Vec { + serde_json::to_vec(&serde_json::json!({ + "records": [{ + "uri": format!("at://{OWNER_DID}/sh.tangled.publicKey/1"), + "value": { + "$type": "sh.tangled.publicKey", + "key": public_line, + "name": "laptop", + "createdAt": "2026-06-08T00:00:00Z" + } + }] + })) + .unwrap() +} + +fn fake_http(published_line: String) -> impl knot_runtime::HttpTransport { + let signer = K256Signer::generate(&SeededEntropy::new(1)); + FakeHttp::new(move |request| { + let host = request.url.host_str().unwrap_or_default().to_string(); + let path = request.url.path().to_string(); + let body = if host == PDS_HOST { + list_records_body(&published_line) + } else if path.ends_with(OWNER_DID) { + did_document(&signer, OWNER_DID) + } else if path.ends_with(REPO_DID) { + did_document(&signer, REPO_DID) + } else { + return Ok(HttpResponse { + status: http::StatusCode::NOT_FOUND, + headers: http::HeaderMap::new(), + body: bytes::Bytes::new(), + }); + }; + Ok(HttpResponse { + status: http::StatusCode::OK, + headers: http::HeaderMap::new(), + body: bytes::Bytes::from(body), + }) + }) +} + +fn actor_for_seed(seed: u64) -> knot_types::ActorId { + knot_types::ActorId::from_secp256k1( + K256Signer::generate(&SeededEntropy::new(seed)) + .public_key() + .as_bytes(), + ) +} + +struct Server { + _scan: TempDir, + layout: Layout, + repo_did: RepoDid, + port: u16, + events: Arc>, +} + +async fn spawn(published_line: String) -> Server { + let scan = tempfile::tempdir().unwrap(); + let meta_path = scan.path().join("meta"); + Repo::create(&meta_path).unwrap(); + let layout = Layout::new(scan.path().join("repos")); + let repo_did = RepoDid::new(REPO_DID).unwrap(); + layout.create(&repo_did).unwrap(); + + let signer = K256Signer::generate(&SeededEntropy::new(2)); + let meta = Repo::open(&meta_path).unwrap(); + CobStore::new(&meta) + .create( + &CobHome::from(&KnotId::new(KNOT_DID).unwrap()), + &RegistryChange::Register(Registration { + owner: OwnerDid::new(OWNER_DID).unwrap(), + rkey: RepoRkey::new(REPO_NAME).unwrap(), + name: RepoName::new(REPO_NAME).unwrap(), + repo: repo_did.clone(), + created_at: UnixSeconds::new(1), + }), + &signer, + UnixSeconds::new(1), + ) + .unwrap(); + + let index = Arc::new(knot_index::Index::new(meta_path, layout.clone())); + index.rebuild().unwrap(); + + let atproto = Arc::new(Atproto::new( + fake_http(published_line), + ManualClock::new(UnixMicros::new(1_000_000_000)), + KnotId::new(KNOT_DID).unwrap(), + knot_atproto::PlcDirectory::new(Url::parse("https://plc.directory/").unwrap()).unwrap(), + )); + let key_dir = scan.path().join("hostkey"); + std::fs::create_dir_all(&key_dir).unwrap(); + let host_key = knot_ssh::load_or_create_host_key(&key_dir.join("host")).unwrap(); + let events = Arc::new(knot_events::EventLog::new( + ManualClock::new(UnixMicros::new(1_000_000_000)), + knot_events::ReplayBounds::new( + knot_events::ReplayEvents::new(64).unwrap(), + knot_events::ReplayBytes::new(16 << 20).unwrap(), + ), + )); + let state = Arc::new(knot_ssh::SshState::new( + layout.clone(), + index, + atproto, + actor_for_seed(1), + Arc::clone(&events), + knot_types::KnotHostname::new("knot.test").unwrap(), + knot_types::AppviewEndpoint::new("https://tangled.test").unwrap(), + std::collections::BTreeSet::new(), + knot_types::AdmissionPolicy::Closed, + knot_xrpc::MaxWireBytes::new(1 << 30), + knot_xrpc::LanguagesPushBudget::new(std::time::Duration::from_secs(2)), + None, + )); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + let _ = knot_ssh::serve_on_socket(listener, host_key, state).await; + }); + Server { + _scan: scan, + layout, + repo_did, + port, + events, + } +} + +fn ssh_command(key_path: &str) -> String { + format!( + "ssh -i {key_path} -o IdentitiesOnly=yes -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null -o PreferredAuthentications=publickey -o BatchMode=yes" + ) +} + +fn seed_work(work: &Path) -> String { + std::fs::create_dir_all(work).unwrap(); + git(work, &[], &["init", "-q", "-b", "main"]); + std::fs::write(work.join("README.md"), "hello over the simulated ssh\n").unwrap(); + git(work, &[], &["add", "-A"]); + git(work, &[], &["commit", "-q", "-m", "initial"]); + let (ok, head) = git(work, &[], &["rev-parse", "HEAD"]); + assert!(ok); + head.trim().to_string() +} + +async fn push_once(scratch: &Path) -> (String, Option, serde_json::Value) { + let (key_path, public_line) = keygen(scratch); + let server = spawn(public_line).await; + let url = format!( + "ssh://git@127.0.0.1:{}/{OWNER_DID}/{REPO_NAME}", + server.port + ); + let ssh = ssh_command(&key_path); + let work = scratch.join("work"); + let head = seed_work(&work); + + let (ok, out) = tokio::task::spawn_blocking(move || { + git( + &work, + &[("GIT_SSH_COMMAND", &ssh)], + &["push", "-q", &url, "main"], + ) + }) + .await + .unwrap(); + assert!(ok, "simulated ssh server must accept push:\n{out}"); + + let stored = server + .layout + .open(&server.repo_did) + .unwrap() + .find_ref(&knot_types::RefName::new("refs/heads/main").unwrap()) + .unwrap(); + let event = poll_for_event(&server.events).await; + drop(server); + (head, stored, event) +} + +fn replay_bounds() -> knot_events::ReplayBounds { + knot_events::ReplayBounds::new( + knot_events::ReplayEvents::new(32).unwrap(), + knot_events::ReplayBytes::new(16 << 20).unwrap(), + ) +} + +async fn poll_for_event(events: &knot_events::EventLog) -> serde_json::Value { + futures::stream::iter(0..100) + .then(|_| async { + let hit = events + .replay(knot_events::EventCursor::START, replay_bounds()) + .events + .into_iter() + .find(|event| event.nsid == "sh.tangled.git.refUpdate") + .map(|event| serde_json::to_value(&*event).unwrap()["event"].clone()); + if hit.is_none() { + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + hit + }) + .filter_map(|hit| async move { hit }) + .boxed() + .next() + .await + .expect("refUpdate event must be published within polling window") +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_simulated_ssh_write_path_lands_a_seed_deterministic_tip() { + let first_dir = tempfile::tempdir().unwrap(); + let (first_head, first_stored, first_event) = push_once(first_dir.path()).await; + + let second_dir = tempfile::tempdir().unwrap(); + let (second_head, second_stored, second_event) = push_once(second_dir.path()).await; + + let tip = knot_types::Oid::from_hex(&first_head).unwrap(); + assert_eq!( + first_stored, + Some(tip), + "pushed commit must be the repository's main tip" + ); + assert_eq!( + first_head, second_head, + "two independent runs of the simulated ssh push must produce same commit oid" + ); + assert_eq!( + first_stored, second_stored, + "assembled-against-doubles ssh write path is logically reproducible" + ); + + assert_eq!(first_event["ref"], "refs/heads/main"); + assert_eq!(first_event["newSha"], first_head); + assert_eq!( + first_event["newSha"], second_event["newSha"], + "ref-update event the push emits is seed-stable too" + ); +} diff --git a/knot2/crates/knot-ssh/examples/ephemeral_knot.rs b/knot2/crates/knot-ssh/examples/ephemeral_knot.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-ssh/examples/ephemeral_knot.rs @@ -0,0 +1,242 @@ +use std::collections::BTreeSet; +use std::sync::Arc; +use std::time::Duration; + +use knot_atproto::Atproto; +use knot_cob::{CobHome, CobStore}; +use knot_cobs::{Registration, RegistryChange}; +use knot_git::{Layout, Repo}; +use knot_index::Index; +use knot_runtime::{ + FakeHttp, HttpResponse, K256Signer, ManualClock, SeededEntropy, Signer, UnixMicros, +}; +use knot_types::{ + AdmissionPolicy, KnotHostname, KnotId, OwnerDid, RepoDid, RepoName, RepoRkey, UnixSeconds, +}; +use tokio::net::TcpListener; +use url::Url; + +#[global_allocator] +static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +const REPO_DID: &str = "did:plc:squid"; +const REPO_NAME: &str = "anemone"; +const OWNER_DID: &str = "did:plc:nel"; +const PDS_HOST: &str = "pds.oyster.cafe"; + +fn apply_decay(ms: isize) { + unsafe { + let _ = tikv_jemalloc_ctl::raw::write(b"arenas.dirty_decay_ms\0", ms); + if let Ok(narenas) = tikv_jemalloc_ctl::raw::read::(b"arenas.narenas\0") { + (0..narenas).for_each(|arena| { + let name = format!("arena.{arena}.dirty_decay_ms\0"); + let _ = tikv_jemalloc_ctl::raw::write(name.as_bytes(), ms); + }); + } + } +} + +async fn govern_decay() { + unsafe { + let _ = tikv_jemalloc_ctl::raw::write(b"background_thread\0", true); + } + let mut ticker = tokio::time::interval(Duration::from_secs(1)); + let mut applied = knot_resource::target_decay(); + apply_decay(applied.ms()); + loop { + ticker.tick().await; + let target = knot_resource::target_decay(); + if target != applied { + apply_decay(target.ms()); + applied = target; + } + } +} + +fn vm_hwm_mib() -> u64 { + std::fs::read_to_string("/proc/self/status") + .unwrap_or_default() + .lines() + .find_map(|line| line.strip_prefix("VmHWM:")) + .and_then(|rest| rest.split_whitespace().next()) + .and_then(|kb| kb.parse::().ok()) + .map(|kb| kb / 1024) + .unwrap_or(0) +} + +fn did_document(signer: &K256Signer, did: &str, pds: &str) -> Vec { + let multikey = knot_types::crypto::multikey(0xe7, signer.public_key().as_bytes()); + serde_json::to_vec(&serde_json::json!({ + "id": did, + "alsoKnownAs": ["at://nel.pet"], + "verificationMethod": [{ + "id": format!("{did}#atproto"), + "type": "Multikey", + "controller": did, + "publicKeyMultibase": multikey + }], + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": pds + }] + })) + .unwrap() +} + +fn list_records_body(published_line: &str) -> Vec { + serde_json::to_vec(&serde_json::json!({ + "records": [{ + "value": { + "$type": "sh.tangled.publicKey", + "key": published_line, + "name": "laptop", + "createdAt": "2026-06-08T00:00:00Z" + } + }] + })) + .unwrap() +} + +fn ok_body(body: Vec) -> HttpResponse { + HttpResponse { + status: http::StatusCode::OK, + headers: http::HeaderMap::new(), + body: bytes::Bytes::from(body), + } +} + +fn not_found() -> HttpResponse { + HttpResponse { + status: http::StatusCode::NOT_FOUND, + headers: http::HeaderMap::new(), + body: bytes::Bytes::new(), + } +} + +fn fake_http(published_line: String) -> impl knot_runtime::HttpTransport { + let signer = K256Signer::generate(&SeededEntropy::new(1)); + let pds = format!("https://{PDS_HOST}"); + FakeHttp::new(move |request| { + let host = request.url.host_str().unwrap_or_default().to_string(); + let path = request.url.path().to_string(); + let body = if host == PDS_HOST { + list_records_body(&published_line) + } else if path.ends_with(REPO_DID) { + did_document(&signer, REPO_DID, &pds) + } else if path.ends_with(OWNER_DID) { + did_document(&signer, OWNER_DID, &pds) + } else { + return Ok(not_found()); + }; + Ok(ok_body(body)) + }) +} + +#[tokio::main(flavor = "multi_thread")] +async fn main() { + let mut args = std::env::args().skip(1); + let port: u16 = args + .next() + .expect("usage: ephemeral_knot ") + .parse() + .expect("port"); + let pubkey_path = args.next().expect("client-pubkey-file"); + let published_line = std::fs::read_to_string(&pubkey_path) + .expect("read client pubkey") + .trim() + .to_string(); + + let max_threads = std::env::var("KNOT_MAX_THREADS") + .ok() + .and_then(|value| value.parse::().ok()); + knot_resource::init(knot_resource::Ceilings { + max_threads: max_threads.map(knot_resource::ThreadCount::new), + max_memory: None, + }); + + let scan = tempfile::tempdir().expect("scan tempdir"); + let meta_path = scan.path().join("meta"); + Repo::create(&meta_path).unwrap(); + let layout = Layout::new(scan.path().join("repos")); + let repo_did = RepoDid::new(REPO_DID).unwrap(); + layout.create(&repo_did).unwrap(); + + let signer = K256Signer::generate(&SeededEntropy::new(2)); + let meta = Repo::open(&meta_path).unwrap(); + CobStore::new(&meta) + .create( + &CobHome::from(&KnotId::new("did:web:nel.pet").unwrap()), + &RegistryChange::Register(Registration { + owner: OwnerDid::new(OWNER_DID).unwrap(), + rkey: RepoRkey::new(REPO_NAME).unwrap(), + name: RepoName::new(REPO_NAME).unwrap(), + repo: repo_did.clone(), + created_at: UnixSeconds::new(1), + }), + &signer, + UnixSeconds::new(1), + ) + .unwrap(); + + let index = Arc::new(Index::new(meta_path, layout.clone())); + index.rebuild().unwrap(); + + let atproto = Arc::new(Atproto::new( + fake_http(published_line), + ManualClock::new(UnixMicros::new(1_000_000_000)), + KnotId::new("did:web:nel.pet").unwrap(), + knot_atproto::PlcDirectory::new(Url::parse("https://plc.directory/").unwrap()).unwrap(), + )); + + let key_dir = scan.path().join("hostkey"); + std::fs::create_dir_all(&key_dir).unwrap(); + let host_key = knot_ssh::load_or_create_host_key(&key_dir.join("host")).unwrap(); + + let events = Arc::new(knot_events::EventLog::new( + ManualClock::new(UnixMicros::new(1_000_000_000)), + knot_events::ReplayBounds::new( + knot_events::ReplayEvents::new(64).unwrap(), + knot_events::ReplayBytes::new(16 << 20).unwrap(), + ), + )); + let actor = knot_types::ActorId::from_secp256k1( + K256Signer::generate(&SeededEntropy::new(1)) + .public_key() + .as_bytes(), + ); + let state = Arc::new(knot_ssh::SshState::new( + layout, + index, + atproto, + actor, + events, + KnotHostname::new("knot.test").unwrap(), + knot_types::AppviewEndpoint::new("https://tangled.test").unwrap(), + BTreeSet::new(), + AdmissionPolicy::Closed, + knot_pack::MaxWireBytes::new(1 << 34), + knot_postreceive::LanguagesPushBudget::new(Duration::from_secs(2)), + None, + )); + + let listener = TcpListener::bind(("127.0.0.1", port)).await.unwrap(); + let bound = listener.local_addr().unwrap().port(); + tokio::spawn(govern_decay()); + tokio::spawn(async move { + let _ = knot_ssh::serve_on_socket(listener, host_key, state).await; + }); + + println!("READY pid={} port={}", std::process::id(), bound); + let mut reporter = tokio::time::interval(Duration::from_secs(1)); + tokio::select! { + _ = tokio::signal::ctrl_c() => {} + () = async { + loop { + reporter.tick().await; + println!("VmHWM {} MiB", vm_hwm_mib()); + } + } => {} + } + drop(scan); +} diff --git a/knot2/crates/knot-ssh/src/exec.rs b/knot2/crates/knot-ssh/src/exec.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-ssh/src/exec.rs @@ -0,0 +1,965 @@ +use std::net::IpAddr; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use futures::StreamExt; +use knot_acl::{KnotAcl, can_push}; +use knot_index::Resolved; +use knot_lfs::TransferOp; +use knot_pack::{PackError, PackLimits, RepoLookup}; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{AccountDid, ObjectFormat, OfferedKey, OwnerDid, RepoDid, RepoRkey}; +use russh::Channel; +use russh::server::Msg; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; +use tokio::runtime::Handle; +use tokio::sync::mpsc; + +use crate::SshState; + +const READ_CHUNK: usize = 64 * 1024; +const MAX_UPLOAD_REQUEST: usize = 16 * 1024 * 1024; +const RECEIVE_BODY_DEADLINE: Duration = Duration::from_secs(1800); +const ARCHIVE_REQUEST_DEADLINE: Duration = Duration::from_secs(60); +const LFS_PROGRESS_GRACE: Duration = Duration::from_secs(60); +const LFS_PROGRESS_FLOOR_BYTES_PER_SEC: u64 = 1024; +const LFS_STALL_TIMEOUT: Duration = Duration::from_secs(120); + +fn lfs_within_progress_budget(waited: Duration, moved_bytes: u64) -> bool { + waited + <= LFS_PROGRESS_GRACE + Duration::from_secs(moved_bytes / LFS_PROGRESS_FLOOR_BYTES_PER_SEC) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Service { + Upload, + UploadArchive, + Receive, + Lfs(TransferOp), +} + +enum ReadError { + Io(std::io::Error), + Pack(PackError), + TooLarge, + Truncated, + Deadline, +} + +enum RepoRef { + Did(RepoDid), + OwnerRkey(OwnerDid, Vec), + HandleRkey(knot_types::Handle, Vec), +} + +enum ResolvedRef { + Did(RepoDid), + OwnerRkey(OwnerDid, Vec), +} + +fn parse_exec(command: &[u8]) -> Option<(Service, RepoRef)> { + let text = std::str::from_utf8(command).ok()?.trim(); + if let Some(rest) = text.strip_prefix("git-lfs-transfer ") { + let (path, op_token) = rest.trim().rsplit_once(' ')?; + let op = TransferOp::parse(op_token.trim())?; + return Some((Service::Lfs(op), parse_repo_path(path)?)); + } + let (service, rest) = [ + ("git-upload-pack ", Service::Upload), + ("git upload-pack ", Service::Upload), + ("git-upload-archive ", Service::UploadArchive), + ("git upload-archive ", Service::UploadArchive), + ("git-receive-pack ", Service::Receive), + ("git receive-pack ", Service::Receive), + ] + .into_iter() + .find_map(|(prefix, service)| text.strip_prefix(prefix).map(|rest| (service, rest)))?; + Some((service, parse_repo_path(rest)?)) +} + +fn parse_repo_path(raw: &str) -> Option { + let path = raw + .trim() + .trim_matches('\'') + .trim_matches('"') + .trim_start_matches('/'); + match path.split_once('/') { + Some((owner, name)) => { + let candidates: Vec = RepoRkey::clone_path_candidates(name).collect(); + if candidates.is_empty() { + return None; + } + match knot_types::OwnerRef::parse(owner)? { + knot_types::OwnerRef::Did(owner) => Some(RepoRef::OwnerRkey(owner, candidates)), + knot_types::OwnerRef::Handle(handle) => { + Some(RepoRef::HandleRkey(handle, candidates)) + } + } + } + None => Some(RepoRef::Did(RepoDid::new(path).ok()?)), + } +} + +fn resolve_repo_ref( + state: &Arc>, + repo_ref: ResolvedRef, +) -> RepoLookup { + let candidate = match repo_ref { + ResolvedRef::Did(did) => RepoLookup::Hosted(did), + ResolvedRef::OwnerRkey(owner, candidates) => RepoLookup::first(candidates, |rkey| { + RepoLookup::from_resolved(state.index.resolve_repo(&owner, &rkey), |found| found) + }), + }; + match candidate { + RepoLookup::Hosted(did) => { + RepoLookup::from_resolved(state.index.owner_of(&did), |_| did.clone()) + } + undecided => undecided, + } +} + +pub(crate) async fn run_exec( + state: Arc>, + key: Option, + channel: Channel, + command: &[u8], + protocol_v2: bool, + peer: Option, +) { + let Some((service, repo_ref)) = parse_exec(command) else { + fail(channel, &state.catalog.ssh.unsupported_command.text()).await; + return; + }; + let peer_limiter = match &service { + Service::Lfs(_) => state + .lfs + .as_ref() + .map_or(&state.peer_slots, |lfs| &lfs.peer_slots), + _ => &state.peer_slots, + }; + let _peer_guard = match peer_limiter.admit(peer, state.atproto.now()) { + Ok(guard) => guard, + Err(refusal) => { + let reason = match refusal { + knot_resource::Refusal::RateLimited => "peer request rate exceeded", + knot_resource::Refusal::Saturated => "peer concurrency limit reached", + }; + tracing::warn!(?peer, reason, "ssh exec rejected"); + return fail(channel, &state.catalog.ssh.too_many_operations.text()).await; + } + }; + let resolved_ref = match repo_ref { + RepoRef::Did(did) => ResolvedRef::Did(did), + RepoRef::OwnerRkey(owner, candidates) => ResolvedRef::OwnerRkey(owner, candidates), + RepoRef::HandleRkey(owner_handle, candidates) => { + match state + .atproto + .resolve_handle_to_did(&owner_handle) + .await + .ok() + { + Some(did) => ResolvedRef::OwnerRkey(did.into(), candidates), + None => { + fail(channel, &state.catalog.ssh.repo_not_found.text()).await; + return; + } + } + } + }; + let repo_did = match resolve_repo_ref(&state, resolved_ref) { + RepoLookup::Hosted(did) => did, + RepoLookup::Unhosted => { + fail(channel, &state.catalog.ssh.repo_not_found.text()).await; + return; + } + RepoLookup::Unavailable => { + fail(channel, &state.catalog.ssh.index_warming.text()).await; + return; + } + }; + let layout = state.layout.clone(); + let did = repo_did.clone(); + let opened = tokio::task::spawn_blocking(move || layout.open(&did).is_ok()) + .await + .unwrap_or(false); + if !opened { + fail(channel, &state.catalog.ssh.repo_not_found.text()).await; + return; + } + match service { + Service::Upload => serve_upload(state, channel, repo_did, protocol_v2).await, + Service::UploadArchive => serve_upload_archive(state, channel, repo_did).await, + Service::Receive => serve_receive(state, key, channel, repo_did).await, + Service::Lfs(op) => serve_lfs(state, key, channel, repo_did, op).await, + } +} + +async fn serve_lfs( + state: Arc>, + key: Option, + mut channel: Channel, + repo_did: RepoDid, + op: TransferOp, +) { + let Some(lfs) = state.lfs.clone() else { + return fail(channel, &state.catalog.ssh.lfs_disabled.text()).await; + }; + if op == TransferOp::Upload { + let pusher = resolve_pusher(&state, key.as_ref(), &repo_did).await; + let allowed = pusher.as_ref().is_some_and(|did| { + let acl = KnotAcl::new(&state.admins, state.admission, &state.index); + can_push(&acl, did, &repo_did).is_allowed() + }); + if !allowed { + tracing::warn!( + repo = repo_did.as_str(), + registered = pusher.is_some(), + "ssh lfs upload denied" + ); + let message = match pusher { + None => state.catalog.ssh.key_not_registered.text(), + Some(_) => state.catalog.ssh.push_denied.text(), + }; + return fail(channel, &message).await; + } + } + let permit = match Arc::clone(&lfs.slots).acquire_owned().await { + Ok(permit) => permit, + Err(_) => return fail(channel, &state.catalog.ssh.shutting_down.text()).await, + }; + let started = std::time::Instant::now(); + + let (tx, rx) = mpsc::channel::>(8); + let writer = Box::pin(channel.make_writer()); + let handle = lfs.handle.clone(); + let runtime = Handle::current(); + let did = repo_did.clone(); + let catalog = Arc::clone(&state.catalog); + let mut engine = tokio::task::spawn_blocking(move || { + let _permit = permit; + let output = std::io::BufWriter::new(MeteredWrite::new(runtime.clone(), writer)); + knot_lfs::serve_transfer( + handle.store.as_ref(), + handle.admission.as_ref(), + &did, + op, + &catalog.lfs, + MpscRead::new(runtime, rx), + output, + ) + }); + let joined = { + let reader = channel.make_reader(); + tokio::select! { + joined = &mut engine => joined, + () = pump_input(reader, tx) => engine.await, + } + }; + let status = match joined { + Ok(Ok(())) => { + tracing::info!( + repo = repo_did.as_str(), + op = match op { + TransferOp::Upload => "upload", + TransferOp::Download => "download", + }, + duration_ms = started.elapsed().as_millis() as u64, + "ssh lfs transfer finished" + ); + 0 + } + Ok(Err(fault)) => { + tracing::warn!(repo = repo_did.as_str(), %fault, "ssh lfs transfer failed"); + 1 + } + Err(join) => { + tracing::error!(repo = repo_did.as_str(), %join, "ssh lfs transfer task panicked"); + 1 + } + }; + finish(channel, status).await; +} + +async fn pump_input(reader: R, tx: mpsc::Sender>) { + use futures::TryStreamExt; + let _ = tokio_util::io::ReaderStream::with_capacity(reader, READ_CHUNK) + .map_err(|_| ()) + .try_for_each(|chunk| { + let tx = &tx; + async move { + match chunk.is_empty() { + true => Ok(()), + false => tx.send(chunk.to_vec()).await.map_err(|_| ()), + } + } + }) + .await; +} + +fn stalled(direction: &'static str) -> std::io::Error { + std::io::Error::other(format!("lfs {direction} stalled past the idle timeout")) +} + +struct MpscRead { + runtime: Handle, + rx: mpsc::Receiver>, + buffer: Vec, + offset: usize, + waited: Duration, + received: u64, +} + +impl MpscRead { + fn new(runtime: Handle, rx: mpsc::Receiver>) -> Self { + Self { + runtime, + rx, + buffer: Vec::new(), + offset: 0, + waited: Duration::ZERO, + received: 0, + } + } +} + +impl std::io::Read for MpscRead { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + if self.offset >= self.buffer.len() { + let started = std::time::Instant::now(); + let rx = &mut self.rx; + let received = self + .runtime + // I know I know, but these aren't runtime workers here + .block_on(async { tokio::time::timeout(LFS_STALL_TIMEOUT, rx.recv()).await }); + match received { + Ok(Some(chunk)) => { + self.waited += started.elapsed(); + self.received += chunk.len() as u64; + if !lfs_within_progress_budget(self.waited, self.received) { + return Err(std::io::Error::other( + "lfs input trickles below the progress floor", + )); + } + self.buffer = chunk; + self.offset = 0; + } + Ok(None) => return Ok(0), + Err(_) => return Err(stalled("input")), + } + } + let take = out.len().min(self.buffer.len() - self.offset); + out[..take].copy_from_slice(&self.buffer[self.offset..self.offset + take]); + self.offset += take; + Ok(take) + } +} + +struct MeteredWrite { + runtime: Handle, + inner: W, + waited: Duration, + written: u64, +} + +impl MeteredWrite { + fn new(runtime: Handle, inner: W) -> Self { + Self { + runtime, + inner, + waited: Duration::ZERO, + written: 0, + } + } + + fn charge(&mut self, started: std::time::Instant) -> std::io::Result<()> { + self.waited += started.elapsed(); + match lfs_within_progress_budget(self.waited, self.written) { + true => Ok(()), + false => Err(std::io::Error::other( + "lfs output trickles below the progress floor", + )), + } + } +} + +impl std::io::Write for MeteredWrite { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let started = std::time::Instant::now(); + let inner = &mut self.inner; + let wrote = self + .runtime + .block_on(async { tokio::time::timeout(LFS_STALL_TIMEOUT, inner.write(buf)).await }) + .map_err(|_| stalled("output"))??; + self.written += wrote as u64; + self.charge(started).map(|()| wrote) + } + + fn flush(&mut self) -> std::io::Result<()> { + let started = std::time::Instant::now(); + let inner = &mut self.inner; + self.runtime + .block_on(async { tokio::time::timeout(LFS_STALL_TIMEOUT, inner.flush()).await }) + .map_err(|_| stalled("output"))??; + self.charge(started) + } +} + +async fn serve_upload_archive( + state: Arc>, + mut channel: Channel, + repo_did: RepoDid, +) { + let request = { + let mut reader = channel.make_reader(); + tokio::time::timeout(ARCHIVE_REQUEST_DEADLINE, read_archive_request(&mut reader)).await + }; + let request = match request { + Ok(Ok(request)) => request, + Ok(Err(())) => return fail(channel, &state.catalog.ssh.archive_malformed.text()).await, + Err(_) => return fail(channel, &state.catalog.ssh.archive_timeout.text()).await, + }; + + let permit = state.slots.pack.acquire().await; + let (tx, mut rx) = mpsc::channel::>(16); + let layout = state.layout.clone(); + let did = repo_did.clone(); + let handle = tokio::task::spawn_blocking(move || -> Result<(), PackError> { + let _permit = permit; + let repo = layout.open(&did)?; + let mut sink = |chunk: &[u8]| -> std::io::Result<()> { + tx.blocking_send(chunk.to_vec()) + .map_err(|_| std::io::Error::other("client disconnected")) + }; + knot_pack::upload_archive_streamed(&repo, &request, &mut sink) + }); + + let mut writer = channel.make_writer(); + let mut forward = Ok(()); + while let Some(chunk) = rx.recv().await { + if writer.write_all(&chunk).await.is_err() { + forward = Err(()); + break; + } + } + drop(rx); + let produced = handle.await; + match &produced { + Ok(Err(error)) => { + tracing::warn!(repo = repo_did.as_str(), %error, "upload-archive failed") + } + Err(join) => { + tracing::error!(repo = repo_did.as_str(), %join, "upload-archive task panicked") + } + Ok(Ok(())) => {} + } + match (forward, produced) { + (Ok(()), Ok(Ok(()))) if writer.flush().await.is_ok() => finish(channel, 0).await, + _ => fail(channel, &state.catalog.ssh.archive_failed.text()).await, + } +} + +async fn read_archive_request(reader: &mut R) -> Result, ()> { + let mut buf = Vec::new(); + loop { + if knot_pack::archive_request_complete(&buf).is_some() { + return Ok(buf); + } + match read_chunk(reader, &mut buf, MAX_UPLOAD_REQUEST).await { + Ok(true) => {} + Ok(false) | Err(_) => return Err(()), + } + } +} + +async fn serve_upload( + state: Arc>, + mut channel: Channel, + repo_did: RepoDid, + protocol_v2: bool, +) { + let advert = { + let layout = state.layout.clone(); + let did = repo_did.clone(); + tokio::task::spawn_blocking(move || -> Result, PackError> { + let repo = layout.open(&did)?; + if protocol_v2 { + knot_pack::advertise_upload_ssh(&repo) + } else { + knot_pack::advertise_upload_v0_ssh(&repo) + } + }) + .await + }; + let advert = match advert { + Ok(Ok(bytes)) => bytes, + _ => return fail(channel, &state.catalog.ssh.advertise_failed.text()).await, + }; + + let mut writer = channel.make_writer(); + if writer.write_all(&advert).await.is_err() || writer.flush().await.is_err() { + return; + } + + let outcome = { + let mut reader = channel.make_reader(); + if protocol_v2 { + upload_loop_v2(&state, &repo_did, &mut reader, &mut writer).await + } else { + upload_loop_v0(&state, &repo_did, &mut reader, &mut writer).await + } + }; + let status = match outcome { + Ok(()) => 0, + Err(()) => 1, + }; + finish(channel, status).await; +} + +async fn upload_loop_v2( + state: &Arc>, + repo_did: &RepoDid, + reader: &mut R, + writer: &mut W, +) -> Result<(), ()> +where + H: HttpTransport, + C: Clock, + R: AsyncRead + Unpin, + W: AsyncWriteExt + Unpin, +{ + let mut buf = Vec::new(); + let mut framer = knot_pack::UploadFramer::new(); + loop { + if let Some(len) = framer.advance(&buf) { + let request: Vec = buf.drain(..len).collect(); + stream_upload(state, repo_did, request, writer).await?; + framer = knot_pack::UploadFramer::new(); + continue; + } + match read_chunk(reader, &mut buf, MAX_UPLOAD_REQUEST).await { + Ok(true) => {} + Ok(false) => return Ok(()), + Err(_) => return Err(()), + } + } +} + +async fn upload_loop_v0( + state: &Arc>, + repo_did: &RepoDid, + reader: &mut R, + writer: &mut W, +) -> Result<(), ()> +where + H: HttpTransport, + C: Clock, + R: AsyncRead + Unpin, + W: AsyncWriteExt + Unpin, +{ + let mut buf = Vec::new(); + let mut framer = knot_pack::UploadFramer::new(); + let mut naks_sent = 0usize; + loop { + if let Some(len) = framer.advance(&buf) { + let request: Vec = buf.drain(..len).collect(); + return stream_upload(state, repo_did, request, writer).await; + } + let needed = framer.unanswered_flushes(); + if naks_sent < needed { + let nak = knot_pack::upload_v0_nak(); + if writer.write_all(&nak).await.is_err() || writer.flush().await.is_err() { + return Err(()); + } + naks_sent += 1; + continue; + } + match read_chunk(reader, &mut buf, MAX_UPLOAD_REQUEST).await { + Ok(true) => {} + Ok(false) => return Ok(()), + Err(_) => return Err(()), + } + } +} + +async fn stream_upload( + state: &Arc>, + repo_did: &RepoDid, + request: Vec, + writer: &mut W, +) -> Result<(), ()> +where + H: HttpTransport, + C: Clock, + W: AsyncWriteExt + Unpin, +{ + let permit = state.slots.pack.acquire().await; + let (tx, mut rx) = mpsc::channel::>(16); + let layout = state.layout.clone(); + let did = repo_did.clone(); + let catalog = Arc::clone(&state.catalog); + let knot = state.hostname.clone(); + let handle = tokio::task::spawn_blocking(move || -> Result<(), PackError> { + let _permit = permit; + let repo = layout.open(&did)?; + let mut sink = |chunk: &[u8]| -> std::io::Result<()> { + tx.blocking_send(chunk.to_vec()) + .map_err(|_| std::io::Error::other("client disconnected")) + }; + knot_pack::upload_pack_streamed(&repo, &request, &catalog.fetch, &knot, &mut sink) + }); + + let mut forward = Ok(()); + while let Some(chunk) = rx.recv().await { + if writer.write_all(&chunk).await.is_err() { + forward = Err(()); + break; + } + } + drop(rx); + match (forward, handle.await) { + (Ok(()), Ok(Ok(()))) => writer.flush().await.map_err(|_| ()), + _ => Err(()), + } +} + +async fn serve_receive( + state: Arc>, + key: Option, + mut channel: Channel, + repo_did: RepoDid, +) { + let advert = { + let layout = state.layout.clone(); + let did = repo_did.clone(); + tokio::task::spawn_blocking(move || -> Result<(Vec, ObjectFormat), PackError> { + let repo = layout.open(&did)?; + let bytes = knot_pack::advertise_receive_ssh(&repo)?; + Ok((bytes, repo.object_format())) + }) + .await + }; + let (advert, object_format) = match advert { + Ok(Ok(pair)) => pair, + _ => return fail(channel, &state.catalog.ssh.advertise_failed.text()).await, + }; + + let mut writer = channel.make_writer(); + if writer.write_all(&advert).await.is_err() || writer.flush().await.is_err() { + return; + } + + let pusher = resolve_pusher(&state, key.as_ref(), &repo_did).await; + let allowed = |did: &AccountDid| { + let acl = KnotAcl::new(&state.admins, state.admission, &state.index); + can_push(&acl, did, &repo_did).is_allowed() + }; + let committer = match pusher { + Some(did) if allowed(&did) => did, + Some(_) => { + tracing::warn!( + repo = repo_did.as_str(), + registered = true, + "ssh push denied" + ); + return fail(channel, &state.catalog.ssh.push_denied.text()).await; + } + None => { + tracing::warn!( + repo = repo_did.as_str(), + registered = false, + "ssh push denied" + ); + return fail(channel, &state.catalog.ssh.key_not_registered.text()).await; + } + }; + + let _receive_permit = state.slots.receive.acquire().await; + + let limits = state.limits; + let body = { + let mut reader = channel.make_reader(); + let dir = state.layout.scratch_dir().to_path_buf(); + match tokio::time::timeout( + RECEIVE_BODY_DEADLINE, + read_receive( + &mut reader, + dir, + state.max_pack_bytes, + limits, + object_format, + ), + ) + .await + { + Ok(result) => result, + Err(_) => Err(ReadError::Deadline), + } + }; + let body = match body { + Ok(body) => body, + Err(ReadError::TooLarge) => { + return fail(channel, &state.catalog.ssh.push_too_large.text()).await; + } + Err(ReadError::Deadline) => { + return fail(channel, &state.catalog.ssh.receive_deadline.text()).await; + } + Err(ReadError::Pack(error)) => { + tracing::warn!(repo = repo_did.as_str(), %error, "receive framing failed"); + return fail(channel, &state.catalog.ssh.malformed_pack.text()).await; + } + Err(ReadError::Io(error)) => { + tracing::warn!(repo = repo_did.as_str(), %error, "receive read error"); + return fail(channel, &state.catalog.ssh.receive_read_error.text()).await; + } + Err(ReadError::Truncated) => { + return fail(channel, &state.catalog.ssh.receive_ended_early.text()).await; + } + }; + if body.is_empty() { + return finish(channel, 0).await; + } + + let _pack_permit = state.slots.pack.acquire().await; + let landed = knot_receive::land(knot_receive::Push { + layout: &state.layout, + repo_did: &repo_did, + received: body, + limits: state.limits, + knot_actor: state.knot_actor.clone(), + committer, + events: Arc::clone(&state.events), + index: &state.index, + atproto: &state.atproto, + resolve_slots: &state.slots.resolve, + appview: &state.appview, + maintenance: &state.maintenance, + hostname: &state.hostname, + languages_push_budget: state.languages_push_budget, + catalog: Arc::clone(&state.catalog), + ci_logs: state.ci_logs.clone(), + }) + .await; + match landed { + Ok(framed) => { + let _ = writer.write_all(&framed).await; + let _ = writer.flush().await; + finish(channel, 0).await; + } + Err(error) => { + tracing::warn!(repo = repo_did.as_str(), %error, "receive-pack failed"); + fail(channel, &state.catalog.ssh.receive_failed.text()).await; + } + } +} + +pub(crate) async fn run_greeting( + state: Arc>, + key: Option, + channel: Channel, +) { + let who = greeting_identity(&state, key.as_ref()).await; + let greeting = state.catalog.ssh.greeting.lines(|key| match key { + knot_messages::GreetingKey::User => who.clone(), + knot_messages::GreetingKey::Knot => state.hostname.as_str().to_string(), + }); + if greeting.is_empty() { + return finish(channel, 0).await; + } + let body = greeting.join("\r\n"); + let _ = channel + .extended_data_bytes(1, format!("{body}\r\n").into_bytes()) + .await; + finish(channel, 0).await; +} + +async fn greeting_identity( + state: &Arc>, + key: Option<&OfferedKey>, +) -> String { + let Some(did) = key.and_then(|key| state.roster.did_for(key)) else { + return "there".to_string(); + }; + match knot_receive::resolve_handle(&state.atproto, &state.slots.resolve, &did).await { + Some(handle) => format!("@{}", handle.as_str()), + None => did.as_str().to_string(), + } +} + +async fn resolve_pusher( + state: &Arc>, + key: Option<&OfferedKey>, + repo: &RepoDid, +) -> Option { + let key = key?; + let owner = match state.index.owner_of(repo) { + Resolved::Ready(Some(owner)) => Some(AccountDid::from(owner)), + _ => None, + }; + { + let index = Arc::clone(&state.index); + let target = repo.clone(); + let _ = tokio::task::spawn_blocking(move || index.ensure_collaborators(&target)).await; + } + let collaborators = match state.index.collaborators_of(repo) { + Resolved::Ready(collaborators) => collaborators, + _ => Vec::new(), + }; + let candidates: Vec = owner.into_iter().chain(collaborators).collect(); + if let Resolved::Ready(Some(cached)) = state.index.owner_of_key(key) + && candidates.contains(&cached) + { + return Some(cached); + } + let _permit = state.slots.resolve.acquire().await; + let matches = futures::stream::iter(candidates).filter_map(|did| async move { + let keys = state.atproto.resolve_pubkeys(&did).await.ok()?; + keys.iter() + .for_each(|resolved| state.index.cache_key(resolved.clone(), &did)); + keys.iter().any(|resolved| resolved == key).then_some(did) + }); + futures::pin_mut!(matches); + matches.next().await +} + +async fn read_chunk( + reader: &mut R, + buf: &mut Vec, + limit: usize, +) -> Result { + let mut chunk = [0u8; READ_CHUNK]; + let read = reader.read(&mut chunk).await.map_err(ReadError::Io)?; + if read == 0 { + return Ok(false); + } + buf.extend_from_slice(&chunk[..read]); + if buf.len() > limit { + return Err(ReadError::TooLarge); + } + Ok(true) +} + +async fn read_receive( + reader: &mut R, + dir: PathBuf, + limit: knot_pack::MaxWireBytes, + limits: PackLimits, + format: ObjectFormat, +) -> Result { + let (tx, rx) = mpsc::channel::>(8); + let mut framer = + tokio::task::spawn_blocking(move || frame_receive(rx, dir, limit, limits, format)); + let mut chunk = [0u8; READ_CHUNK]; + let mut io_error = None; + loop { + tokio::select! { + biased; + framed = &mut framer => return join_framed(framed, io_error), + read = reader.read(&mut chunk) => match read { + Ok(0) => break, + Ok(read) => { + if tx.send(chunk[..read].to_vec()).await.is_err() { + break; + } + } + Err(error) => { + io_error = Some(error); + break; + } + }, + } + } + drop(tx); + join_framed(framer.await, io_error) +} + +fn join_framed( + framed: Result, tokio::task::JoinError>, + io_error: Option, +) -> Result { + match framed { + Ok(Ok(body)) => Ok(body), + Ok(Err(ReadError::Truncated)) => { + Err(io_error.map(ReadError::Io).unwrap_or(ReadError::Truncated)) + } + Ok(Err(other)) => Err(other), + Err(_) => Err(ReadError::Truncated), + } +} + +fn read_error(error: knot_pack::ReceiveReadError) -> ReadError { + match error { + knot_pack::ReceiveReadError::Io(error) => ReadError::Io(error), + knot_pack::ReceiveReadError::Pack(error) => ReadError::Pack(error), + knot_pack::ReceiveReadError::TooLarge => ReadError::TooLarge, + knot_pack::ReceiveReadError::Truncated => ReadError::Truncated, + } +} + +fn frame_receive( + mut rx: mpsc::Receiver>, + dir: PathBuf, + limit: knot_pack::MaxWireBytes, + limits: PackLimits, + format: ObjectFormat, +) -> Result { + let mut receiver = + knot_pack::PackReceiver::new(&dir, limit, limits, format.kind()).map_err(ReadError::Io)?; + loop { + match rx.blocking_recv() { + Some(chunk) => { + if receiver.write(&chunk).map_err(read_error)? { + return receiver.finish().map_err(read_error); + } + } + None => return receiver.finish().map_err(read_error), + } + } +} + +async fn fail(channel: Channel, message: &str) { + let _ = channel + .extended_data_bytes(1, format!("{message}\n").into_bytes()) + .await; + finish(channel, 1).await; +} + +async fn finish(channel: Channel, status: u32) { + let _ = channel.exit_status(status).await; + let _ = channel.eof().await; + let _ = channel.close().await; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_lfs_progress_budget_spares_slow_links_and_cuts_trickles() { + assert!(lfs_within_progress_budget(Duration::from_secs(59), 0)); + assert!(!lfs_within_progress_budget(Duration::from_secs(61), 0)); + assert!(lfs_within_progress_budget( + Duration::from_secs(50_000), + 5 * 1024 * 1024 * 1024 + )); + assert!(!lfs_within_progress_budget(Duration::from_secs(1_000), 10)); + } + + #[test] + fn the_repo_path_parser_separates_dids_from_handles() { + assert!(matches!( + parse_repo_path("did:plc:nel/squid"), + Some(RepoRef::OwnerRkey(..)) + )); + assert!(matches!( + parse_repo_path("nel.pet/squid"), + Some(RepoRef::HandleRkey(..)) + )); + assert!(matches!( + parse_repo_path("did:plc:barnacle"), + Some(RepoRef::Did(_)) + )); + assert!(parse_repo_path("did:nonsense/squid").is_none()); + assert!(parse_repo_path("nel.pet").is_none()); + } +} diff --git a/knot2/crates/knot-ssh/src/lib.rs b/knot2/crates/knot-ssh/src/lib.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-ssh/src/lib.rs @@ -0,0 +1,358 @@ +mod exec; +mod roster; +mod server; + +use std::collections::BTreeSet; +use std::net::SocketAddr; +use std::path::Path; +use std::sync::Arc; +use std::time::Duration; + +use knot_atproto::Atproto; +use knot_events::EventLog; +use knot_git::Layout; +use knot_index::Index; +use knot_maintenance::MaintenanceHandle; +use knot_pack::{MaxWireBytes, PackLimits}; +use knot_postreceive::LanguagesPushBudget; +use knot_runtime::{Clock, Entropy, HttpTransport, OsEntropy}; +use knot_types::{AccountDid, ActorId, AdmissionPolicy, AppviewEndpoint, CiLogsAddr, KnotHostname}; +use russh::keys::ssh_key::rand_core; +use russh::keys::{Algorithm, PrivateKey, ssh_key}; +use russh::server::{Config, Server as _}; +use russh::{MethodKind, MethodSet}; +use tokio::sync::Semaphore; +use tokio_util::sync::CancellationToken; +use tokio_util::task::TaskTracker; + +use knot_resource::{LimitConfig, PerPeerInflight, PreAuthLimiter, Slots}; +use roster::KeyRoster; +use server::KnotSshServer; + +const MAX_INFLIGHT_PER_PEER: usize = 4; +const INACTIVITY_TIMEOUT: Duration = Duration::from_secs(120); +const KEEPALIVE_INTERVAL: Duration = Duration::from_secs(30); +const AUTH_REJECTION_TIME: Duration = Duration::from_millis(250); +const DRAIN_GRACE: Duration = Duration::from_secs(30); +const ACCEPT_BACKOFF: Duration = Duration::from_millis(250); + +#[derive(Debug, thiserror::Error)] +pub enum SshError { + #[error("ssh host key {path}: {message}")] + HostKey { path: String, message: String }, + #[error("ssh server bind or serve: {0}")] + Serve(#[from] std::io::Error), +} + +pub struct SshState { + layout: Layout, + index: Arc, + atproto: Arc>, + knot_actor: ActorId, + events: Arc>, + hostname: KnotHostname, + appview: AppviewEndpoint, + admins: BTreeSet, + admission: AdmissionPolicy, + limits: PackLimits, + max_pack_bytes: MaxWireBytes, + languages_push_budget: LanguagesPushBudget, + ci_logs: Option, + slots: Slots, + peer_slots: Arc, + roster: Arc, + maintenance: MaintenanceHandle, + lfs: Option, + catalog: Arc, +} + +#[derive(Clone)] +pub(crate) struct LfsRuntime { + pub(crate) handle: knot_lfs::LfsHandle, + pub(crate) slots: Arc, + pub(crate) peer_slots: Arc, +} + +impl SshState { + #[allow(clippy::too_many_arguments)] + pub fn new( + layout: Layout, + index: Arc, + atproto: Arc>, + knot_actor: ActorId, + events: Arc>, + hostname: KnotHostname, + appview: AppviewEndpoint, + admins: BTreeSet, + admission: AdmissionPolicy, + max_pack_bytes: MaxWireBytes, + languages_push_budget: LanguagesPushBudget, + ci_logs: Option, + ) -> Self { + Self { + layout, + index, + atproto, + knot_actor, + events, + hostname, + appview, + admins, + admission, + limits: PackLimits::default(), + max_pack_bytes, + languages_push_budget, + ci_logs, + slots: Slots::for_machine(), + peer_slots: Arc::new(PreAuthLimiter::with_config(LimitConfig::per_peer_only( + PerPeerInflight::new(MAX_INFLIGHT_PER_PEER), + ))), + roster: Arc::new(KeyRoster::new()), + maintenance: MaintenanceHandle::disabled(), + lfs: None, + catalog: Arc::new(knot_messages::Catalog::defaults()), + } + } + + pub fn with_catalog(mut self, catalog: Arc) -> Self { + self.catalog = catalog; + self + } + + pub fn with_slots(mut self, slots: Slots) -> Self { + self.slots = slots; + self + } + + pub fn with_maintenance(mut self, maintenance: MaintenanceHandle) -> Self { + self.maintenance = maintenance; + self + } + + pub fn with_lfs(mut self, handle: knot_lfs::LfsHandle, max_transfers: usize) -> Self { + self.lfs = Some(LfsRuntime { + handle, + slots: Arc::new(Semaphore::new(max_transfers)), + peer_slots: Arc::new(PreAuthLimiter::with_config(LimitConfig::per_peer_only( + PerPeerInflight::new(max_transfers), + ))), + }); + self + } + + pub fn with_limits(mut self, limits: PackLimits) -> Self { + self.limits = limits; + self + } +} + +fn server_config(host_key: PrivateKey) -> Arc { + Arc::new(Config { + keys: vec![host_key], + methods: MethodSet::from(&[MethodKind::PublicKey][..]), + inactivity_timeout: Some(INACTIVITY_TIMEOUT), + keepalive_interval: Some(KEEPALIVE_INTERVAL), + auth_rejection_time: AUTH_REJECTION_TIME, + ..Config::default() + }) +} + +pub async fn serve( + addr: SocketAddr, + host_key: PrivateKey, + state: Arc>, + shutdown: CancellationToken, +) -> Result<(), SshError> { + let listener = tokio::net::TcpListener::bind(addr).await?; + serve_drained(listener, host_key, state, shutdown).await +} + +pub async fn serve_on_socket( + listener: tokio::net::TcpListener, + host_key: PrivateKey, + state: Arc>, +) -> Result<(), SshError> { + serve_drained(listener, host_key, state, CancellationToken::new()).await +} + +#[doc(hidden)] +pub async fn serve_drained( + listener: tokio::net::TcpListener, + host_key: PrivateKey, + state: Arc>, + shutdown: CancellationToken, +) -> Result<(), SshError> { + let config = server_config(host_key); + let tracker = TaskTracker::new(); + state.roster.prime(&state.index, &state.atproto); + let mut server = KnotSshServer { + state, + tracker: tracker.clone(), + }; + loop { + let accepted = tokio::select! { + () = shutdown.cancelled() => break, + accepted = listener.accept() => accepted, + }; + let (stream, peer) = match accepted { + Ok(pair) => pair, + Err(error) if is_connection_error(&error) => continue, + Err(error) => { + tracing::warn!("ssh accept failed, backing off: {error}"); + tokio::select! { + () = shutdown.cancelled() => break, + () = tokio::time::sleep(ACCEPT_BACKOFF) => {} + } + continue; + } + }; + let handler = server.new_client(Some(peer)); + let config = Arc::clone(&config); + tracker.spawn(async move { + if let Ok(session) = russh::server::run_stream(config, stream, handler).await { + let _ = session.await; + } + }); + } + tracker.close(); + let _ = tokio::time::timeout(DRAIN_GRACE, tracker.wait()).await; + Ok(()) +} + +fn is_connection_error(error: &std::io::Error) -> bool { + matches!( + error.kind(), + std::io::ErrorKind::ConnectionRefused + | std::io::ErrorKind::ConnectionAborted + | std::io::ErrorKind::ConnectionReset + ) +} + +pub fn load_or_create_host_key(path: &Path) -> Result { + let report = |message: String| SshError::HostKey { + path: path.display().to_string(), + message, + }; + let load = || { + ensure_secure_perms(path).map_err(&report)?; + russh::keys::load_secret_key(path, None).map_err(|error| report(error.to_string())) + }; + if path.exists() { + return load(); + } + let key = PrivateKey::random(&mut EntropyRng, Algorithm::Ed25519) + .map_err(|error| report(error.to_string()))?; + match persist_host_key(path, &key)? { + Claim::Won => Ok(key), + Claim::Lost => load(), + } +} + +enum Claim { + Won, + Lost, +} + +fn persist_host_key(path: &Path, key: &PrivateKey) -> Result { + let report = |message: String| SshError::HostKey { + path: path.display().to_string(), + message, + }; + let pem = key + .to_openssh(ssh_key::LineEnding::LF) + .map_err(|error| report(error.to_string()))?; + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(|error| report(error.to_string()))?; + } + let temp = unique_temp(path); + write_secret(&temp, pem.as_bytes()).map_err(|error| report(error.to_string()))?; + // `hard_link` instead of `rename` so that in case two knots + // are booting at the same time they don't get borked + // if one clobber's the other's key. here the loser reloads + // winner's key. + let claim = match std::fs::hard_link(&temp, path) { + Ok(()) => Claim::Won, + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => Claim::Lost, + Err(error) => { + let _ = std::fs::remove_file(&temp); + return Err(report(error.to_string())); + } + }; + let _ = std::fs::remove_file(&temp); + if let Some(parent) = path.parent() + && let Ok(dir) = std::fs::File::open(parent) + { + let _ = dir.sync_all(); + } + Ok(claim) +} + +#[cfg(unix)] +fn ensure_secure_perms(path: &Path) -> Result<(), String> { + use std::os::unix::fs::MetadataExt; + let mode = std::fs::metadata(path) + .map_err(|error| error.to_string())? + .mode(); + if mode & 0o077 != 0 { + return Err(format!( + "private host key is group or other accessible at mode {:o}, run chmod 600 on it", + mode & 0o777 + )); + } + Ok(()) +} + +#[cfg(not(unix))] +fn ensure_secure_perms(_path: &Path) -> Result<(), String> { + Ok(()) +} + +fn unique_temp(path: &Path) -> std::path::PathBuf { + use std::sync::atomic::{AtomicU64, Ordering}; + static COUNTER: AtomicU64 = AtomicU64::new(0); + let nonce = COUNTER.fetch_add(1, Ordering::Relaxed); + let stem = path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("ssh_host_key"); + path.with_file_name(format!(".{stem}.{}.{nonce}.tmp", std::process::id())) +} + +fn write_secret(path: &Path, bytes: &[u8]) -> std::io::Result<()> { + use std::io::Write; + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options.open(path)?; + file.write_all(bytes)?; + file.sync_all() +} + +struct EntropyRng; + +impl rand_core::TryRng for EntropyRng { + type Error = std::convert::Infallible; + + fn try_next_u32(&mut self) -> Result { + let mut bytes = [0u8; 4]; + OsEntropy.fill(&mut bytes); + Ok(u32::from_le_bytes(bytes)) + } + + fn try_next_u64(&mut self) -> Result { + let mut bytes = [0u8; 8]; + OsEntropy.fill(&mut bytes); + Ok(u64::from_le_bytes(bytes)) + } + + fn try_fill_bytes(&mut self, dst: &mut [u8]) -> Result<(), Self::Error> { + OsEntropy.fill(dst); + Ok(()) + } +} + +impl rand_core::TryCryptoRng for EntropyRng {} diff --git a/knot2/crates/knot-ssh/src/roster.rs b/knot2/crates/knot-ssh/src/roster.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-ssh/src/roster.rs @@ -0,0 +1,522 @@ +use std::collections::{HashMap, HashSet}; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use futures::StreamExt; +use knot_atproto::Atproto; +use knot_index::{Index, IndexGeneration, Resolved}; +use knot_runtime::{Clock, HttpTransport, UnixMicros}; +use knot_types::{AccountDid, OfferedKey}; + +const FRESH_TTL: Duration = Duration::from_secs(60); +const DEGRADED_TTL: Duration = Duration::from_secs(5); +const BACKOFF_SHIFT_LIMIT: u32 = 4; +const MISS_REVALIDATE_BUDGET: Duration = Duration::from_secs(30); +const RESOLVE_FANOUT: usize = 16; + +fn degraded_ttl(consecutive_failures: u32) -> Duration { + let secs = DEGRADED_TTL + .as_secs() + .saturating_mul(1u64 << consecutive_failures.min(BACKOFF_SHIFT_LIMIT)) + .min(FRESH_TTL.as_secs()); + Duration::from_secs(secs) +} + +struct Freshness { + due: UnixMicros, + generation: IndexGeneration, +} + +#[derive(Debug, PartialEq, Eq)] +enum Staleness { + Fresh, + Revalidate, + Cold, +} + +pub(crate) struct KeyRoster { + by_did: Mutex>>, + recognized: Mutex>, + freshness: Mutex>, + failures: AtomicU32, + refresh: tokio::sync::Mutex<()>, + refresh_in_flight: AtomicBool, +} + +impl KeyRoster { + pub(crate) fn new() -> Self { + Self { + by_did: Mutex::new(HashMap::new()), + recognized: Mutex::new(HashSet::new()), + freshness: Mutex::new(None), + failures: AtomicU32::new(0), + refresh: tokio::sync::Mutex::new(()), + refresh_in_flight: AtomicBool::new(false), + } + } + + pub(crate) fn recognizes(&self, key: &OfferedKey) -> bool { + self.recognized + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .contains(key) + } + + pub(crate) fn did_for(&self, key: &OfferedKey) -> Option { + self.by_did + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .iter() + .find(|(_, keys)| keys.contains(key)) + .map(|(did, _)| did.clone()) + } + + fn is_fresh(&self, now: UnixMicros, generation: IndexGeneration) -> bool { + self.freshness + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .as_ref() + .is_some_and(|fresh| now.get() < fresh.due.get() && fresh.generation == generation) + } + + pub(crate) fn prime( + self: &Arc, + index: &Arc, + atproto: &Arc>, + ) { + self.spawn_refresh(index, atproto); + } + + pub(crate) fn ensure_fresh( + self: &Arc, + index: &Arc, + atproto: &Arc>, + ) { + match self.staleness(atproto.now(), index.generation()) { + Staleness::Fresh => {} + Staleness::Revalidate | Staleness::Cold => self.spawn_refresh(index, atproto), + } + } + + pub(crate) async fn recognizes_fresh( + self: &Arc, + key: &OfferedKey, + index: &Arc, + atproto: &Arc>, + ) -> bool { + if self.recognizes(key) { + self.ensure_fresh(index, atproto); + return true; + } + if self.is_fresh(atproto.now(), index.generation()) { + return false; + } + let _ = tokio::time::timeout(MISS_REVALIDATE_BUDGET, self.refresh(index, atproto)).await; + self.recognizes(key) + } + + fn staleness(&self, now: UnixMicros, generation: IndexGeneration) -> Staleness { + match self + .freshness + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .as_ref() + { + None => Staleness::Cold, + Some(fresh) if fresh.generation != generation => Staleness::Revalidate, + Some(fresh) if now.get() < fresh.due.get() => Staleness::Fresh, + Some(_) => Staleness::Revalidate, + } + } + + fn spawn_refresh( + self: &Arc, + index: &Arc, + atproto: &Arc>, + ) { + if self + .refresh_in_flight + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + return; + } + let roster = Arc::clone(self); + let index = Arc::clone(index); + let atproto = Arc::clone(atproto); + tokio::spawn(async move { + let _in_flight = InFlightGuard(&roster.refresh_in_flight); + roster.refresh(&index, &atproto).await; + }); + } + + async fn refresh(&self, index: &Index, atproto: &Atproto) { + let _single_flight = self.refresh.lock().await; + if self.is_fresh(atproto.now(), index.generation()) { + return; + } + let generation = index.generation(); + let (dids, incomplete) = relevant_dids(index); + let resolved: Vec<(AccountDid, Option>)> = futures::stream::iter(dids) + .map(|did| async move { + let keys = atproto.resolve_pubkeys(&did).await.ok(); + (did, keys) + }) + .buffer_unordered(RESOLVE_FANOUT) + .collect() + .await; + let any_failed = resolved.iter().any(|(_, keys)| keys.is_none()); + let relevant: HashSet = resolved.iter().map(|(did, _)| did.clone()).collect(); + { + let mut by_did = self + .by_did + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + by_did.retain(|did, _| relevant.contains(did)); + resolved.into_iter().for_each(|(did, keys)| { + if let Some(keys) = keys { + by_did.insert(did, keys.into_iter().collect()); + } + }); + let union: HashSet = by_did.values().flatten().cloned().collect(); + *self + .recognized + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) = union; + } + let ttl = if any_failed { + degraded_ttl(self.failures.fetch_add(1, Ordering::Relaxed)) + } else { + self.failures.store(0, Ordering::Relaxed); + if incomplete { DEGRADED_TTL } else { FRESH_TTL } + }; + let due = UnixMicros::new(atproto.now().get().saturating_add(ttl.as_micros() as u64)); + *self + .freshness + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(Freshness { due, generation }); + } +} + +struct InFlightGuard<'a>(&'a AtomicBool); + +impl Drop for InFlightGuard<'_> { + fn drop(&mut self) { + self.0.store(false, Ordering::Release); + } +} + +fn relevant_dids(index: &Index) -> (Vec, bool) { + let (mut dids, incomplete): (Vec, bool) = index + .hosted_repos() + .iter() + .map(|repo| { + let (owner, owner_warming) = match index.owner_of(repo) { + Resolved::Ready(Some(owner)) => (Some(AccountDid::from(owner)), false), + Resolved::Ready(None) => (None, false), + Resolved::Warming => (None, true), + }; + let (collaborators, collaborators_warming) = match index.collaborators_of(repo) { + Resolved::Ready(collaborators) => (collaborators, false), + Resolved::Warming => (Vec::new(), true), + }; + ( + owner.into_iter().chain(collaborators).collect::>(), + owner_warming || collaborators_warming, + ) + }) + .fold( + (Vec::new(), false), + |(mut acc, warming), (dids, repo_warming)| { + acc.extend(dids); + (acc, warming || repo_warming) + }, + ); + dids.sort(); + dids.dedup(); + (dids, incomplete) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; + + use knot_atproto::Atproto; + use knot_cob::{CobHome, CobStore}; + use knot_cobs::{Registration, RegistryChange}; + use knot_git::{Layout, Repo}; + use knot_runtime::{ + FakeHttp, HttpRequest, HttpResponse, K256Signer, NetworkError, SeededEntropy, Signer, + }; + use knot_types::{KnotId, OwnerDid, RepoDid, RepoName, RepoRkey, UnixSeconds, crypto}; + use russh::keys::{Algorithm, PrivateKey}; + use url::Url; + + struct SharedClock(Arc); + impl Clock for SharedClock { + fn now_unix_micros(&self) -> UnixMicros { + UnixMicros::new(self.0.load(Ordering::SeqCst)) + } + } + + fn line_and_offered() -> (String, OfferedKey) { + let key = PrivateKey::random(&mut crate::EntropyRng, Algorithm::Ed25519).unwrap(); + let public = key.public_key(); + ( + public.to_openssh().unwrap(), + OfferedKey::from_bytes(public.to_bytes().unwrap()), + ) + } + + type Responder = Box Result + Send + Sync>; + + struct Harness { + index: Arc, + atproto: Arc, SharedClock>>, + published: Arc>>, + list_calls: Arc, + _dir: tempfile::TempDir, + } + + fn harness(initial: Vec) -> Harness { + let dir = tempfile::tempdir().unwrap(); + let meta_path = dir.path().join("meta"); + Repo::create(&meta_path).unwrap(); + let layout = Layout::new(dir.path().join("repos")); + let repo_did = RepoDid::new("did:plc:squid").unwrap(); + layout.create(&repo_did).unwrap(); + let cob_signer = K256Signer::generate(&SeededEntropy::new(2)); + { + let meta = Repo::open(&meta_path).unwrap(); + CobStore::new(&meta) + .create( + &CobHome::from(&KnotId::new("did:web:nel.pet").unwrap()), + &RegistryChange::Register(Registration { + owner: OwnerDid::new("did:plc:nel").unwrap(), + rkey: RepoRkey::new("anemone").unwrap(), + name: RepoName::new("anemone").unwrap(), + repo: repo_did.clone(), + created_at: UnixSeconds::new(1), + }), + &cob_signer, + UnixSeconds::new(1), + ) + .unwrap(); + } + let index = Arc::new(Index::new(meta_path, layout.clone())); + index.rebuild().unwrap(); + + let published = Arc::new(std::sync::Mutex::new(initial)); + let list_calls = Arc::new(AtomicUsize::new(0)); + let multikey = crypto::multikey( + 0xe7, + K256Signer::generate(&SeededEntropy::new(7)) + .public_key() + .as_bytes(), + ); + let clock = Arc::new(AtomicU64::new(1_000_000_000)); + + let responder: Responder = { + let published = Arc::clone(&published); + let list_calls = Arc::clone(&list_calls); + Box::new(move |request: &HttpRequest| { + let host = request.url.host_str().unwrap_or_default().to_string(); + let body = if host == "pds.oyster.cafe" { + list_calls.fetch_add(1, Ordering::SeqCst); + let records: Vec<_> = published + .lock() + .unwrap() + .iter() + .map(|line| { + serde_json::json!({ + "uri": "at://did:plc:nel/sh.tangled.publicKey/1", + "value": { + "$type": "sh.tangled.publicKey", + "key": line, + "name": "laptop", + "createdAt": "2026-06-08T00:00:00Z" + } + }) + }) + .collect(); + serde_json::to_vec(&serde_json::json!({ "records": records })).unwrap() + } else if host == "plc.directory" { + serde_json::to_vec(&serde_json::json!({ + "id": "did:plc:nel", + "alsoKnownAs": ["at://nel.pet"], + "verificationMethod": [{ + "id": "did:plc:nel#atproto", + "type": "Multikey", + "controller": "did:plc:nel", + "publicKeyMultibase": multikey + }], + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": "https://pds.oyster.cafe" + }] + })) + .unwrap() + } else { + return Ok(HttpResponse { + status: http::StatusCode::NOT_FOUND, + headers: http::HeaderMap::new(), + body: bytes::Bytes::new(), + }); + }; + Ok(HttpResponse { + status: http::StatusCode::OK, + headers: http::HeaderMap::new(), + body: bytes::Bytes::from(body), + }) + }) + }; + + let atproto = Arc::new(Atproto::new( + FakeHttp::new(responder), + SharedClock(clock), + KnotId::new("did:web:nel.pet").unwrap(), + knot_atproto::PlcDirectory::new(Url::parse("https://plc.directory/").unwrap()).unwrap(), + )); + + Harness { + index, + atproto, + published, + list_calls, + _dir: dir, + } + } + + async fn wait_recognized(roster: &KeyRoster, key: &OfferedKey) { + for _ in 0..1000 { + if roster.recognizes(key) { + return; + } + tokio::task::yield_now().await; + } + } + + #[tokio::test] + async fn an_acl_write_makes_a_freshly_published_key_recognized_without_waiting_for_the_ttl() { + let (line1, offered1) = line_and_offered(); + let (line2, offered2) = line_and_offered(); + + let Harness { + index, + atproto, + published, + list_calls, + _dir, + } = harness(vec![line1]); + + let roster = Arc::new(KeyRoster::new()); + roster.ensure_fresh(&index, &atproto); + wait_recognized(&roster, &offered1).await; + assert!(roster.recognizes(&offered1)); + assert_eq!(list_calls.load(Ordering::SeqCst), 1); + + published.lock().unwrap().push(line2.clone()); + + roster.ensure_fresh(&index, &atproto); + assert!( + !roster.recognizes(&offered2), + "stable index and unexpired TTL still serves cached roster, no re-resolution" + ); + assert_eq!(list_calls.load(Ordering::SeqCst), 1); + + index.refresh_members().unwrap(); + roster.ensure_fresh(&index, &atproto); + wait_recognized(&roster, &offered2).await; + assert!( + roster.recognizes(&offered2), + "ACL write bumps generation, so roster revalidates off the auth path" + ); + assert_eq!( + list_calls.load(Ordering::SeqCst), + 2, + "exactly one async re-resolution off the auth path" + ); + } + + #[tokio::test] + async fn a_miss_against_a_stale_roster_blocks_bounded_to_revalidate_before_rejecting() { + let (line1, offered1) = line_and_offered(); + let (line2, offered2) = line_and_offered(); + let Harness { + index, + atproto, + published, + list_calls, + _dir, + } = harness(vec![line1]); + let roster = Arc::new(KeyRoster::new()); + + assert!( + roster.recognizes_fresh(&offered1, &index, &atproto).await, + "the first handshake blocks on the primed resolve and recognizes the published key" + ); + assert_eq!(list_calls.load(Ordering::SeqCst), 1); + + published.lock().unwrap().push(line2.clone()); + index.refresh_members().unwrap(); + + assert!( + roster.recognizes_fresh(&offered2, &index, &atproto).await, + "a generation-bumped miss blocks to revalidate and picks up the new key on the first attempt" + ); + assert_eq!( + list_calls.load(Ordering::SeqCst), + 2, + "the miss triggers exactly one bounded re-resolution" + ); + } + + #[test] + fn staleness_classifies_cold_fresh_and_revalidate() { + let roster = KeyRoster::new(); + assert_eq!( + roster.staleness(UnixMicros::new(0), IndexGeneration::new(0)), + Staleness::Cold, + "with no roster yet the first auth is cold and must revalidate before it can answer a miss" + ); + *roster + .freshness + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(Freshness { + due: UnixMicros::new(1_000), + generation: IndexGeneration::new(0), + }); + assert_eq!( + roster.staleness(UnixMicros::new(500), IndexGeneration::new(0)), + Staleness::Fresh + ); + assert_eq!( + roster.staleness(UnixMicros::new(500), IndexGeneration::new(1)), + Staleness::Revalidate, + "an ACL write moves the generation, so the cached roster is stale" + ); + assert_eq!( + roster.staleness(UnixMicros::new(2_000), IndexGeneration::new(0)), + Staleness::Revalidate, + "an expired ttl at the same generation is stale too" + ); + } + + #[test] + fn degraded_ttl_backs_off_from_the_short_retry_to_the_fresh_ceiling() { + assert_eq!(degraded_ttl(0), Duration::from_secs(5)); + assert_eq!(degraded_ttl(1), Duration::from_secs(10)); + assert_eq!(degraded_ttl(2), Duration::from_secs(20)); + assert_eq!(degraded_ttl(3), Duration::from_secs(40)); + assert_eq!(degraded_ttl(4), Duration::from_secs(60)); + assert_eq!( + degraded_ttl(50), + Duration::from_secs(60), + "a persistently unresolvable did clamps the retry to the fresh ttl instead of storming" + ); + } +} diff --git a/knot2/crates/knot-ssh/src/server.rs b/knot2/crates/knot-ssh/src/server.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-ssh/src/server.rs @@ -0,0 +1,183 @@ +use std::collections::{HashMap, HashSet}; +use std::net::{IpAddr, SocketAddr}; +use std::sync::Arc; + +use knot_runtime::{Clock, HttpTransport}; +use knot_types::OfferedKey; +use russh::keys::ssh_key; +use russh::server::{Auth, Handler, Msg, Server, Session}; +use russh::{Channel, ChannelId}; +use tokio_util::task::TaskTracker; + +use crate::SshState; +use crate::exec::run_exec; + +pub(crate) struct KnotSshServer { + pub(crate) state: Arc>, + pub(crate) tracker: TaskTracker, +} + +impl Server for KnotSshServer { + type Handler = KnotSession; + + fn new_client(&mut self, peer: Option) -> Self::Handler { + KnotSession::new( + Arc::clone(&self.state), + self.tracker.clone(), + peer.map(|addr| addr.ip()), + ) + } +} + +pub(crate) struct KnotSession { + state: Arc>, + tracker: TaskTracker, + key: Option, + channels: HashMap>, + protocols: HashSet, + peer: Option, +} + +impl KnotSession { + fn new(state: Arc>, tracker: TaskTracker, peer: Option) -> Self { + Self { + state, + tracker, + key: None, + channels: HashMap::new(), + protocols: HashSet::new(), + peer, + } + } +} + +impl Handler for KnotSession { + type Error = russh::Error; + + async fn auth_publickey( + &mut self, + _user: &str, + public_key: &ssh_key::PublicKey, + ) -> Result { + let reject = Auth::Reject { + proceed_with_methods: None, + partial_success: false, + }; + let Ok(blob) = public_key.to_bytes() else { + return Ok(reject); + }; + let key = OfferedKey::from_bytes(blob); + if self + .state + .roster + .recognizes_fresh(&key, &self.state.index, &self.state.atproto) + .await + { + self.key = Some(key); + Ok(Auth::Accept) + } else { + Ok(reject) + } + } + + async fn channel_open_session( + &mut self, + channel: Channel, + _session: &mut Session, + ) -> Result { + self.channels.insert(channel.id(), channel); + Ok(true) + } + + async fn env_request( + &mut self, + channel: ChannelId, + variable_name: &str, + variable_value: &str, + _session: &mut Session, + ) -> Result<(), Self::Error> { + if variable_name == "GIT_PROTOCOL" + && variable_value + .split(':') + .any(|token| token.trim() == "version=2") + { + self.protocols.insert(channel); + } + Ok(()) + } + + async fn channel_eof( + &mut self, + channel: ChannelId, + _session: &mut Session, + ) -> Result<(), Self::Error> { + self.protocols.remove(&channel); + Ok(()) + } + + async fn channel_close( + &mut self, + channel: ChannelId, + _session: &mut Session, + ) -> Result<(), Self::Error> { + self.channels.remove(&channel); + self.protocols.remove(&channel); + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + async fn pty_request( + &mut self, + channel: ChannelId, + _term: &str, + _col_width: u32, + _row_height: u32, + _pix_width: u32, + _pix_height: u32, + _modes: &[(russh::Pty, u32)], + session: &mut Session, + ) -> Result<(), Self::Error> { + session.channel_success(channel)?; + Ok(()) + } + + async fn shell_request( + &mut self, + channel: ChannelId, + session: &mut Session, + ) -> Result<(), Self::Error> { + let Some(handle) = self.channels.remove(&channel) else { + session.channel_failure(channel)?; + return Ok(()); + }; + session.channel_success(channel)?; + let state = Arc::clone(&self.state); + let key = self.key.clone(); + self.tracker.spawn(async move { + crate::exec::run_greeting(state, key, handle).await; + }); + Ok(()) + } + + async fn exec_request( + &mut self, + channel: ChannelId, + data: &[u8], + session: &mut Session, + ) -> Result<(), Self::Error> { + let Some(handle) = self.channels.remove(&channel) else { + session.channel_failure(channel)?; + return Ok(()); + }; + session.channel_success(channel)?; + let protocol_v2 = self.protocols.remove(&channel); + let state = Arc::clone(&self.state); + let key = self.key.clone(); + let peer = self.peer; + let command = data.to_vec(); + self.tracker.spawn(async move { + run_exec(state, key, handle, &command, protocol_v2, peer).await; + }); + Ok(()) + } +} diff --git a/knot2/crates/knot-ssh/tests/ssh_push.rs b/knot2/crates/knot-ssh/tests/ssh_push.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-ssh/tests/ssh_push.rs @@ -0,0 +1,1442 @@ +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::Arc; + +use futures::stream::StreamExt; +use knot_atproto::Atproto; +use knot_cob::{CobHome, CobStore}; +use knot_cobs::{CollaboratorsChange, Grant, MembersChange, Registration, RegistryChange}; +use knot_git::{Layout, Repo}; +use knot_index::Index; +use knot_pack::MaxWireBytes; +use knot_postreceive::LanguagesPushBudget; +use knot_runtime::{ + FakeDns, FakeHttp, HttpResponse, K256Signer, ManualClock, SeededEntropy, Signer, UnixMicros, +}; +use knot_types::{ + AccountDid, KnotId, Oid, OwnerDid, RefName, RepoDid, RepoName, RepoRkey, UnixSeconds, +}; +use tempfile::TempDir; +use tokio::net::TcpListener; +use url::Url; + +const REPO_DID: &str = "did:plc:squid"; +const REPO_NAME: &str = "anemone"; +const OWNER_DID: &str = "did:plc:nel"; +const PDS_HOST: &str = "pds.oyster.cafe"; + +fn git(cwd: &Path, env: &[(&str, &str)], args: &[&str]) -> (bool, String) { + let mut command = knot_fixtures::command(cwd); + command.args(args); + env.iter().for_each(|(key, value)| { + command.env(key, value); + }); + let out = command.output().expect("git runs"); + let combined = format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + (out.status.success(), combined) +} + +fn keygen(dir: &Path, name: &str) -> (String, String) { + let path = dir.join(name); + let out = Command::new("ssh-keygen") + .args([ + "-t", + "ed25519", + "-N", + "", + "-C", + "nel@oyster.cafe", + "-f", + path.to_str().unwrap(), + ]) + .output() + .expect("ssh-keygen runs"); + assert!( + out.status.success(), + "ssh-keygen failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + let public_line = std::fs::read_to_string(dir.join(format!("{name}.pub"))) + .unwrap() + .trim() + .to_string(); + (path.to_str().unwrap().to_string(), public_line) +} + +fn did_document(signer: &K256Signer, did: &str, pds: &str) -> Vec { + let multikey = knot_types::crypto::multikey(0xe7, signer.public_key().as_bytes()); + serde_json::to_vec(&serde_json::json!({ + "id": did, + "alsoKnownAs": ["at://nel.pet"], + "verificationMethod": [{ + "id": format!("{did}#atproto"), + "type": "Multikey", + "controller": did, + "publicKeyMultibase": multikey + }], + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": pds + }] + })) + .unwrap() +} + +fn list_records_body(lines: &[&str]) -> Vec { + let records: Vec<_> = lines + .iter() + .map(|line| { + serde_json::json!({ + "value": { + "$type": "sh.tangled.publicKey", + "key": line, + "name": "laptop", + "createdAt": "2026-06-08T00:00:00Z" + } + }) + }) + .collect(); + serde_json::to_vec(&serde_json::json!({ "records": records })).unwrap() +} + +fn ok_body(body: Vec) -> HttpResponse { + HttpResponse { + status: http::StatusCode::OK, + headers: http::HeaderMap::new(), + body: bytes::Bytes::from(body), + } +} + +fn fake_dns() -> impl knot_runtime::DnsTxtResolver { + FakeDns::new(|name: &str| { + Ok(match name { + "_atproto.nel.pet" => vec![format!("did={OWNER_DID}")], + _ => Vec::new(), + }) + }) +} + +fn not_found() -> HttpResponse { + HttpResponse { + status: http::StatusCode::NOT_FOUND, + headers: http::HeaderMap::new(), + body: bytes::Bytes::new(), + } +} + +fn fake_http(published_line: String) -> impl knot_runtime::HttpTransport { + let signer = K256Signer::generate(&SeededEntropy::new(1)); + let pds = format!("https://{PDS_HOST}"); + FakeHttp::new(move |request| { + let host = request.url.host_str().unwrap_or_default().to_string(); + let path = request.url.path().to_string(); + let body = if host == PDS_HOST { + list_records_body(&[&published_line]) + } else if path.ends_with(REPO_DID) { + did_document(&signer, REPO_DID, &pds) + } else if path.ends_with(OWNER_DID) { + did_document(&signer, OWNER_DID, &pds) + } else { + return Ok(not_found()); + }; + Ok(ok_body(body)) + }) +} + +fn multi_http(identities: HashMap>) -> impl knot_runtime::HttpTransport { + let signer = K256Signer::generate(&SeededEntropy::new(77)); + FakeHttp::new(move |request| { + let host = request.url.host_str().unwrap_or_default().to_string(); + if host == "plc.directory" { + let did = request.url.path().trim_start_matches('/').to_string(); + return Ok(ok_body(did_document(&signer, &did, "https://pds.test"))); + } + if host == "pds.test" { + let repo = request + .url + .query_pairs() + .find(|(key, _)| key == "repo") + .map(|(_, value)| value.into_owned()) + .unwrap_or_default(); + let lines = identities.get(&repo).cloned().unwrap_or_default(); + let refs: Vec<&str> = lines.iter().map(String::as_str).collect(); + return Ok(ok_body(list_records_body(&refs))); + } + Ok(not_found()) + }) +} + +fn actor_for_seed(seed: u64) -> knot_types::ActorId { + knot_types::ActorId::from_secp256k1( + K256Signer::generate(&SeededEntropy::new(seed)) + .public_key() + .as_bytes(), + ) +} + +struct Server { + _scan: TempDir, + layout: Layout, + repo_did: RepoDid, + port: u16, + events: Arc>, +} + +async fn spawn_server( + published_line: String, + max_pack_bytes: MaxWireBytes, +) -> (Server, Arc) { + spawn_server_with(published_line, max_pack_bytes, true).await +} + +async fn spawn_server_with( + published_line: String, + max_pack_bytes: MaxWireBytes, + warm: bool, +) -> (Server, Arc) { + let (server, index, _, _) = spawn_server_core(published_line, max_pack_bytes, warm, None).await; + (server, index) +} + +async fn spawn_server_core( + published_line: String, + max_pack_bytes: MaxWireBytes, + warm: bool, + lfs: Option, +) -> ( + Server, + Arc, + tokio_util::sync::CancellationToken, + tokio::task::JoinHandle<()>, +) { + let scan = tempfile::tempdir().unwrap(); + let meta_path = scan.path().join("meta"); + Repo::create(&meta_path).unwrap(); + let layout = Layout::new(scan.path().join("repos")); + let repo_did = RepoDid::new(REPO_DID).unwrap(); + layout.create(&repo_did).unwrap(); + + let signer = K256Signer::generate(&SeededEntropy::new(2)); + let meta = Repo::open(&meta_path).unwrap(); + let store = CobStore::new(&meta); + store + .create( + &CobHome::from(&KnotId::new("did:web:nel.pet").unwrap()), + &RegistryChange::Register(Registration { + owner: OwnerDid::new(OWNER_DID).unwrap(), + rkey: RepoRkey::new(REPO_NAME).unwrap(), + name: RepoName::new(REPO_NAME).unwrap(), + repo: repo_did.clone(), + created_at: UnixSeconds::new(1), + }), + &signer, + UnixSeconds::new(1), + ) + .unwrap(); + + let index = Arc::new(Index::new(meta_path, layout.clone())); + if warm { + index.rebuild().unwrap(); + } + + let atproto = Arc::new( + Atproto::new( + fake_http(published_line), + ManualClock::new(UnixMicros::new(1_000_000_000)), + KnotId::new("did:web:nel.pet").unwrap(), + knot_atproto::PlcDirectory::new(Url::parse("https://plc.directory/").unwrap()).unwrap(), + ) + .with_dns(Arc::new(fake_dns())), + ); + + let key_dir = scan.path().join("hostkey"); + std::fs::create_dir_all(&key_dir).unwrap(); + let host_key = knot_ssh::load_or_create_host_key(&key_dir.join("host")).unwrap(); + + let events = Arc::new(knot_events::EventLog::new( + ManualClock::new(UnixMicros::new(1_000_000_000)), + knot_events::ReplayBounds::new( + knot_events::ReplayEvents::new(64).unwrap(), + knot_events::ReplayBytes::new(16 << 20).unwrap(), + ), + )); + let base = knot_ssh::SshState::new( + layout.clone(), + Arc::clone(&index), + atproto, + actor_for_seed(1), + Arc::clone(&events), + knot_types::KnotHostname::new("knot.test").unwrap(), + knot_types::AppviewEndpoint::new("https://tangled.test").unwrap(), + std::collections::BTreeSet::new(), + knot_types::AdmissionPolicy::Closed, + max_pack_bytes, + LanguagesPushBudget::new(std::time::Duration::from_secs(2)), + None, + ); + let state = Arc::new(match lfs { + Some(handle) => base.with_lfs(handle, 2), + None => base, + }); + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let shutdown = tokio_util::sync::CancellationToken::new(); + let serve_task = tokio::spawn({ + let shutdown = shutdown.clone(); + async move { + let _ = knot_ssh::serve_drained(listener, host_key, state, shutdown).await; + } + }); + + ( + Server { + _scan: scan, + layout, + repo_did, + port, + events, + }, + index, + shutdown, + serve_task, + ) +} + +fn ssh_command(key_path: &str) -> String { + format!( + "ssh -i {key_path} -o IdentitiesOnly=yes -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null -o PreferredAuthentications=publickey -o BatchMode=yes" + ) +} + +async fn git_ssh(cwd: &Path, key: &str, args: &[&str]) -> (bool, String) { + let ssh = ssh_command(key); + let cwd = cwd.to_path_buf(); + let owned: Vec = args.iter().map(|arg| arg.to_string()).collect(); + tokio::task::spawn_blocking(move || { + let argv: Vec<&str> = owned.iter().map(String::as_str).collect(); + git(&cwd, &[("GIT_SSH_COMMAND", &ssh)], &argv) + }) + .await + .unwrap() +} + +async fn push(work: &Path, url: &str, key: &str, refspecs: &[&str]) -> (bool, String) { + let args: Vec<&str> = std::iter::once("push") + .chain(std::iter::once(url)) + .chain(refspecs.iter().copied()) + .collect(); + git_ssh(work, key, &args).await +} + +async fn clone(url: &str, key: &str, dest: &Path) -> (bool, String) { + git_ssh( + Path::new("/tmp"), + key, + &["clone", "-q", url, dest.to_str().unwrap()], + ) + .await +} + +fn seed_work(work: &Path) -> String { + std::fs::create_dir_all(work).unwrap(); + git(work, &[], &["init", "-q", "-b", "main"]); + std::fs::write(work.join("README.md"), "hello over ssh\n").unwrap(); + git(work, &[], &["add", "-A"]); + git(work, &[], &["commit", "-q", "-m", "initial"]); + let (ok, head) = git(work, &[], &["rev-parse", "HEAD"]); + assert!(ok); + head.trim().to_string() +} + +fn seed_commits(work: &Path, count: usize) { + std::fs::create_dir_all(work).unwrap(); + git(work, &[], &["init", "-q", "-b", "main"]); + (0..count).for_each(|i| { + std::fs::write(work.join("log.txt"), format!("line {i}\n")).unwrap(); + git(work, &[], &["add", "-A"]); + git(work, &[], &["commit", "-q", "-m", &format!("c{i}")]); + }); +} + +fn seed_cob(work: &Path, signer_seed: u64, subject: &str, home: &CobHome) -> (Oid, String, String) { + let repo = Repo::open(work).unwrap(); + let signer = K256Signer::generate(&SeededEntropy::new(signer_seed)); + let created = CobStore::new(&repo) + .create( + home, + &MembersChange::Add(Grant { + subject: AccountDid::new(subject).unwrap(), + added_by: AccountDid::new(OWNER_DID).unwrap(), + created_at: UnixSeconds::new(1), + }), + &signer, + UnixSeconds::new(1), + ) + .unwrap(); + let cob_ref = format!( + "refs/cobs/sh.tangled.knot.member/{}", + created.object.oid().to_hex() + ); + let spec = format!("{cob_ref}:{cob_ref}"); + (created.tip.oid(), cob_ref, spec) +} + +fn main_tip(layout: &Layout, repo: &RepoDid) -> Option { + layout + .open(repo) + .unwrap() + .find_ref(&RefName::new("refs/heads/main").unwrap()) + .unwrap() +} + +fn ref_names(server: &Server) -> Vec { + server + .layout + .open(&server.repo_did) + .unwrap() + .references() + .unwrap() + .iter() + .map(|record| record.name.as_str().to_string()) + .collect() +} + +fn replay_bounds() -> knot_events::ReplayBounds { + knot_events::ReplayBounds::new( + knot_events::ReplayEvents::new(32).unwrap(), + knot_events::ReplayBytes::new(16 << 20).unwrap(), + ) +} + +async fn poll_for_event( + events: &knot_events::EventLog, + nsid: &str, +) -> serde_json::Value { + for _ in 0..50 { + if let Some(payload) = events + .replay(knot_events::EventCursor::START, replay_bounds()) + .events + .into_iter() + .find(|event| event.nsid == nsid) + .map(|event| serde_json::to_value(&*event).unwrap()["event"].clone()) + { + return payload; + } + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + panic!("no {nsid} event was published within the polling window"); +} + +struct Fixture { + scratch: TempDir, + server: Server, + index: Arc, + key_path: String, + url: String, + work: PathBuf, +} + +async fn fixture() -> Fixture { + let scratch = tempfile::tempdir().unwrap(); + let (key_path, public_line) = keygen(scratch.path(), "client"); + let (server, index) = spawn_server(public_line, MaxWireBytes::new(1 << 30)).await; + let url = format!( + "ssh://git@127.0.0.1:{}/{OWNER_DID}/{REPO_NAME}", + server.port + ); + let work = scratch.path().join("work"); + Fixture { + scratch, + server, + index, + key_path, + url, + work, + } +} + +fn fetch_main_exit(clone_dir: &Path, ssh: &str, extra_git: &[&str]) -> Option { + let mut args = vec!["-k", "3", "20", "git"]; + args.extend_from_slice(extra_git); + args.extend_from_slice(&["fetch", "origin", "main"]); + Command::new("timeout") + .args(&args) + .current_dir(clone_dir) + .env("GIT_SSH_COMMAND", ssh) + .status() + .expect("timeout/git runs") + .code() +} + +async fn incremental_fetch_exit( + seed_count: usize, + extra_git: &'static [&'static str], +) -> Option { + let scratch = tempfile::tempdir().unwrap(); + let (key_path, public_line) = keygen(scratch.path(), "client"); + let (server, _index) = spawn_server(public_line, MaxWireBytes::new(1 << 30)).await; + let url = format!( + "ssh://git@127.0.0.1:{}/{OWNER_DID}/{REPO_NAME}", + server.port + ); + + let work = scratch.path().join("work"); + seed_commits(&work, seed_count); + let (ok, out) = push(&work, &url, &key_path, &["main"]).await; + assert!(ok, "seeding push must land:\n{out}"); + + let clone_dir = scratch.path().join("clone"); + let (ok, out) = clone(&url, &key_path, &clone_dir).await; + assert!(ok, "clone over ssh must succeed:\n{out}"); + + git( + &work, + &[], + &["commit", "-q", "--allow-empty", "-m", "advance"], + ); + let (ok, out) = push(&work, &url, &key_path, &["main"]).await; + assert!(ok, "advancing server tip must succeed:\n{out}"); + + let ssh = ssh_command(&key_path); + let exit = tokio::task::spawn_blocking(move || fetch_main_exit(&clone_dir, &ssh, extra_git)) + .await + .unwrap(); + drop(server); + exit +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn incremental_fetch_over_ssh_completes() { + let cases: [(&'static [&'static str], &str); 2] = [ + ( + &["-c", "protocol.version=0"], + "diverged v0 fetch sends more than 32 haves and blocks on an ACK/NAK. Upload loop \ + answers each have-batch flush with a NAK instead of waiting for done, so it never \ + hangs", + ), + ( + &[], + "git forwards GIT_PROTOCOL over ssh, so default fetch path negotiates with the v2 loop", + ), + ]; + futures::stream::iter(cases) + .for_each(|(extra, rationale)| async move { + assert_eq!( + incremental_fetch_exit(50, extra).await, + Some(0), + "{rationale}" + ); + }) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn addressing_variants_land() { + let fx = fixture().await; + let head = seed_work(&fx.work); + let head_oid = Oid::from_hex(&head).unwrap(); + let port = fx.server.port; + let variants = [ + format!("ssh://git@127.0.0.1:{port}/{OWNER_DID}/{REPO_NAME}"), + format!("ssh://git@127.0.0.1:{port}/{OWNER_DID}/{REPO_NAME}.git"), + format!("ssh://git@127.0.0.1:{port}/nel.pet/{REPO_NAME}"), + format!("ssh://git@127.0.0.1:{port}/{REPO_DID}"), + ]; + let fx = &fx; + futures::stream::iter(variants) + .for_each(|url| async move { + let (ok, out) = push(&fx.work, &url, &fx.key_path, &["main"]).await; + assert!(ok, "addressing {url} must resolve and push:\n{out}"); + assert_eq!( + main_tip(&fx.server.layout, &fx.server.repo_did), + Some(head_oid), + "{url}: pushed commit must be the repository's main tip" + ); + }) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_push_while_the_index_is_warming_is_refused() { + let scratch = tempfile::tempdir().unwrap(); + let (key_path, public_line) = keygen(scratch.path(), "client"); + let (server, _index) = spawn_server_with(public_line, MaxWireBytes::new(1 << 30), false).await; + let url = format!( + "ssh://git@127.0.0.1:{}/{OWNER_DID}/{REPO_NAME}", + server.port + ); + + let work = scratch.path().join("work"); + seed_work(&work); + let (ok, out) = push(&work, &url, &key_path, &["main"]).await; + assert!( + !ok, + "warming index must fail closed at the SSH boundary:\n{out}" + ); + assert_eq!( + main_tip(&server.layout, &server.repo_did), + None, + "no ref lands while index is warming" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn unresolvable_targets_refused() { + let fx = fixture().await; + seed_work(&fx.work); + let port = fx.server.port; + + let bad_name = format!("ssh://git@127.0.0.1:{port}/{OWNER_DID}/conch"); + let (ok, out) = push(&fx.work, &bad_name, &fx.key_path, &["main"]).await; + assert!( + !ok, + "owner/name with no registry entry must be rejected, not silently routed:\n{out}" + ); + + fx.server + .layout + .create(&RepoDid::new("did:plc:clam").unwrap()) + .unwrap(); + let ghost_url = format!("ssh://git@127.0.0.1:{port}/did:plc:clam"); + let dest = fx.scratch.path().join("ghost"); + let (ok, out) = clone(&ghost_url, &fx.key_path, &dest).await; + assert!( + !ok, + "repo present on disk but absent from registry mustn't be served by bare DID:\n{out}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_authorized_push_over_ssh_succeeds_and_a_clone_reads_it_back() { + let fx = fixture().await; + let head = seed_work(&fx.work); + + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &["main"]).await; + assert!(ok, "authorized push over ssh must succeed:\n{out}"); + assert_eq!( + main_tip(&fx.server.layout, &fx.server.repo_did), + Some(Oid::from_hex(&head).unwrap()), + "pushed commit must be the repository's main tip" + ); + + let clone_dir = fx.scratch.path().join("clone"); + let (ok, out) = clone(&fx.url, &fx.key_path, &clone_dir).await; + assert!(ok, "clone over ssh must succeed:\n{out}"); + assert!( + clone_dir.join("README.md").exists(), + "clone must check out the pushed file" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_authorized_push_emits_a_ref_update_event() { + let fx = fixture().await; + let head = seed_work(&fx.work); + + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &["main"]).await; + assert!(ok, "authorized push over ssh must succeed:\n{out}"); + + let event = poll_for_event(&fx.server.events, "sh.tangled.git.refUpdate").await; + assert_eq!(event["ref"], "refs/heads/main"); + assert_eq!(event["newSha"], head); + assert_eq!(event["committerDid"], OWNER_DID); + assert_eq!(event["ownerDid"], OWNER_DID); + assert_eq!(event["meta"]["isDefaultRef"], true); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_oversized_push_is_refused_at_the_ssh_boundary() { + let scratch = tempfile::tempdir().unwrap(); + let (key_path, public_line) = keygen(scratch.path(), "client"); + let (server, _index) = spawn_server(public_line, MaxWireBytes::new(64)).await; + let url = format!( + "ssh://git@127.0.0.1:{}/{OWNER_DID}/{REPO_NAME}", + server.port + ); + + let work = scratch.path().join("work"); + seed_work(&work); + let (ok, out) = push(&work, &url, &key_path, &["main"]).await; + assert!( + !ok, + "push larger than the configured limit must be refused:\n{out}" + ); + assert!( + ref_names(&server).is_empty(), + "oversized push mustn't land any ref" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_up_to_date_push_over_ssh_is_accepted() { + let fx = fixture().await; + seed_work(&fx.work); + + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &["main"]).await; + assert!(ok, "first push must land:\n{out}"); + + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &["main"]).await; + assert!( + ok, + "up-to-date no-op push must succeed instead of failing with a stream error:\n{out}" + ); + assert!( + out.contains("up-to-date") || out.contains("up to date"), + "git must report branch is up to date:\n{out}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_denied_push_over_ssh_leaves_no_objects_in_the_live_odb() { + let scratch = tempfile::tempdir().unwrap(); + let (_registered_path, registered_line) = keygen(scratch.path(), "registered"); + let (attacker_path, _attacker_line) = keygen(scratch.path(), "attacker"); + let (server, _index) = spawn_server(registered_line, MaxWireBytes::new(1 << 30)).await; + let url = format!( + "ssh://git@127.0.0.1:{}/{OWNER_DID}/{REPO_NAME}", + server.port + ); + + let work = scratch.path().join("work"); + let head = seed_work(&work); + let (ok, out) = push(&work, &url, &attacker_path, &["main"]).await; + assert!(!ok, "unauthorized push must be rejected:\n{out}"); + + let repo = server.layout.open(&server.repo_did).unwrap(); + assert!( + repo.references().unwrap().is_empty(), + "denied push must create no ref" + ); + assert!( + !repo.contains(Oid::from_hex(&head).unwrap()), + "denied push must migrate no objects into the live odb" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn ref_namespace_policy() { + let fx = fixture().await; + seed_work(&fx.work); + + let (ok, out) = push( + &fx.work, + &fx.url, + &fx.key_path, + &["main:refs/hidden/feature/main"], + ) + .await; + assert!(!ok, "push to refs/hidden/* must be rejected:\n{out}"); + assert!( + ref_names(&fx.server).is_empty(), + "forbidden-ref push must land nothing" + ); + + let (ok, out) = push( + &fx.work, + &fx.url, + &fx.key_path, + &["main:refs/notes/commits"], + ) + .await; + assert!( + ok, + "push to any non-reserved namespace must be accepted:\n{out}" + ); + assert!( + ref_names(&fx.server) + .iter() + .any(|name| name == "refs/notes/commits"), + "pushed ref must land" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn cob_ref_guard_lifecycle() { + let fx = fixture().await; + seed_work(&fx.work); + let home = CobHome::from(&RepoDid::new(REPO_DID).unwrap()); + let foreign = CobHome::from(&RepoDid::new("did:plc:whelk").unwrap()); + + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &["main"]).await; + assert!(ok, "head must land for the advertisement check:\n{out}"); + + let (owned_tip, owned_ref, owned_spec) = seed_cob(&fx.work, 1, "did:plc:limpet", &home); + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &[owned_spec.as_str()]).await; + assert!( + ok, + "COB ref signed by the repository key must verify and land over ssh:\n{out}" + ); + + let (_forged_tip, forged_ref, forged_spec) = seed_cob(&fx.work, 9, "did:plc:whelk", &home); + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &[forged_spec.as_str()]).await; + assert!( + !ok, + "COB ref signed by a stranger must be refused at the receive boundary:\n{out}" + ); + + let (_transplant_tip, transplant_ref, transplant_spec) = + seed_cob(&fx.work, 1, "did:plc:mussel", &foreign); + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &[transplant_spec.as_str()]).await; + assert!( + !ok, + "same key signing for another repo's home must be refused on transplant:\n{out}" + ); + + let landed = ref_names(&fx.server); + assert!( + landed.contains(&owned_ref), + "owner-signed COB ref must be stored: {landed:?}" + ); + assert!( + !landed.contains(&forged_ref), + "stranger-signed COB ref must be absent: {landed:?}" + ); + assert!( + !landed.contains(&transplant_ref), + "transplanted COB ref must be absent: {landed:?}" + ); + + let cob_name = RefName::new(&owned_ref).unwrap(); + let del = format!(":{owned_ref}"); + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &[del.as_str()]).await; + assert!(!ok, "deleting a COB ref must be refused:\n{out}"); + assert!( + out.contains("append-only"), + "rejection must name the append-only rule:\n{out}" + ); + assert!( + ref_names(&fx.server).contains(&owned_ref), + "COB ref must survive the refused delete" + ); + + let repo = Repo::open(&fx.work).unwrap(); + CobStore::new(&repo) + .update( + &home, + knot_types::CobId::new(owned_tip), + &MembersChange::Add(Grant { + subject: AccountDid::new("did:plc:bailey").unwrap(), + added_by: AccountDid::new(OWNER_DID).unwrap(), + created_at: UnixSeconds::new(2), + }), + &K256Signer::generate(&SeededEntropy::new(1)), + UnixSeconds::new(2), + ) + .unwrap(); + assert_ne!( + repo.find_ref(&cob_name).unwrap(), + Some(owned_tip), + "local COB ref now points at a new, equally valid tip" + ); + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &[owned_spec.as_str()]).await; + assert!( + !ok, + "re-pushing a moved COB ref must be refused instead of silently clobbered:\n{out}" + ); + assert_eq!( + fx.server + .layout + .open(&fx.server.repo_did) + .unwrap() + .find_ref(&cob_name) + .unwrap(), + Some(owned_tip), + "live COB ref must still point at the original tip" + ); + + let (ok, advert) = git_ssh(Path::new("/tmp"), &fx.key_path, &["ls-remote", &fx.url]).await; + assert!(ok, "ls-remote over ssh must succeed:\n{advert}"); + assert!( + advert.contains("refs/heads/main"), + "head must be advertised:\n{advert}" + ); + assert!( + !advert.contains("refs/cobs/"), + "no refs/cobs/* may leak into the ssh advertisement:\n{advert}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn key_recognition_edge_cases() { + let fx = fixture().await; + let head = seed_work(&fx.work); + let head_oid = Oid::from_hex(&head).unwrap(); + + let (unregistered_path, _unregistered_line) = keygen(fx.scratch.path(), "unregistered"); + let two_ids = format!( + "ssh -i {unregistered_path} -i {} -o IdentitiesOnly=yes -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null -o PreferredAuthentications=publickey -o BatchMode=yes", + fx.key_path + ); + let (ok, out) = { + let (work, url) = (fx.work.clone(), fx.url.clone()); + tokio::task::spawn_blocking(move || { + git( + &work, + &[("GIT_SSH_COMMAND", &two_ids)], + &["push", "-q", &url, "main"], + ) + }) + .await + .unwrap() + }; + assert!( + ok, + "rejecting unregistered key must let client cycle to the registered one:\n{out}" + ); + assert_eq!( + main_tip(&fx.server.layout, &fx.server.repo_did), + Some(head_oid) + ); + + let blob = russh::keys::ssh_key::PublicKey::from_openssh( + &std::fs::read_to_string(fx.scratch.path().join("client.pub")).unwrap(), + ) + .unwrap() + .to_bytes() + .unwrap(); + fx.index.cache_key( + knot_types::OfferedKey::from_bytes(blob), + &AccountDid::new("did:plc:whelk").unwrap(), + ); + let (ok, out) = push( + &fx.work, + &fx.url, + &fx.key_path, + &["main:refs/heads/squat-check"], + ) + .await; + assert!( + ok, + "stranger who published the owner's key mustn't deny the owner's push:\n{out}" + ); + assert_eq!( + fx.server + .layout + .open(&fx.server.repo_did) + .unwrap() + .find_ref(&RefName::new("refs/heads/squat-check").unwrap()) + .unwrap(), + Some(head_oid) + ); +} + +#[test] +fn a_group_or_other_readable_host_key_is_refused_on_load() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("host"); + knot_ssh::load_or_create_host_key(&path).unwrap(); + assert_eq!( + std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o600, + "freshly created host key is 0600" + ); + + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap(); + let refused = knot_ssh::load_or_create_host_key(&path); + assert!( + matches!(refused, Err(knot_ssh::SshError::HostKey { .. })), + "world-readable existing host key must be refused on load: {refused:?}" + ); +} + +async fn launch( + host_key_dir: &Path, + layout: Layout, + index: Arc, + identities: HashMap>, +) -> u16 { + let atproto = Arc::new(Atproto::new( + multi_http(identities), + ManualClock::new(UnixMicros::new(1_000_000_000)), + KnotId::new("did:web:nel.pet").unwrap(), + knot_atproto::PlcDirectory::new(Url::parse("https://plc.directory/").unwrap()).unwrap(), + )); + std::fs::create_dir_all(host_key_dir).unwrap(); + let host_key = knot_ssh::load_or_create_host_key(&host_key_dir.join("host")).unwrap(); + let events = Arc::new(knot_events::EventLog::new( + ManualClock::new(UnixMicros::new(1_000_000_000)), + knot_events::ReplayBounds::new( + knot_events::ReplayEvents::new(64).unwrap(), + knot_events::ReplayBytes::new(16 << 20).unwrap(), + ), + )); + let state = Arc::new(knot_ssh::SshState::new( + layout, + index, + atproto, + actor_for_seed(77), + events, + knot_types::KnotHostname::new("knot.test").unwrap(), + knot_types::AppviewEndpoint::new("https://tangled.test").unwrap(), + std::collections::BTreeSet::new(), + knot_types::AdmissionPolicy::Closed, + MaxWireBytes::new(1 << 30), + LanguagesPushBudget::new(std::time::Duration::from_secs(2)), + None, + )); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + let _ = knot_ssh::serve_on_socket(listener, host_key, state).await; + }); + port +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_collaborator_pushes_its_repo_but_a_recognized_key_is_denied_on_a_repo_it_has_no_grant_on() + { + const REPO_A: &str = "did:plc:squid"; + const REPO_B: &str = "did:plc:clam"; + const OWNER: &str = "did:plc:nel"; + const COLLAB: &str = "did:plc:olaren"; + + let scratch = tempfile::tempdir().unwrap(); + let (owner_key, owner_line) = keygen(scratch.path(), "owner"); + let (collab_key, collab_line) = keygen(scratch.path(), "collab"); + + let meta_path = scratch.path().join("meta"); + Repo::create(&meta_path).unwrap(); + let layout = Layout::new(scratch.path().join("repos")); + let repo_a = RepoDid::new(REPO_A).unwrap(); + let repo_b = RepoDid::new(REPO_B).unwrap(); + let git_a = layout.create(&repo_a).unwrap(); + layout.create(&repo_b).unwrap(); + + let signer = K256Signer::generate(&SeededEntropy::new(2)); + let meta = Repo::open(&meta_path).unwrap(); + let store = CobStore::new(&meta); + let knot_home = CobHome::from(&KnotId::new("did:web:nel.pet").unwrap()); + let reg = store + .create( + &knot_home, + &RegistryChange::Register(Registration { + owner: OwnerDid::new(OWNER).unwrap(), + rkey: RepoRkey::new("anemone").unwrap(), + name: RepoName::new("anemone").unwrap(), + repo: repo_a.clone(), + created_at: UnixSeconds::new(1), + }), + &signer, + UnixSeconds::new(1), + ) + .unwrap(); + store + .update( + &knot_home, + reg.object, + &RegistryChange::Register(Registration { + owner: OwnerDid::new(OWNER).unwrap(), + rkey: RepoRkey::new("barnacle").unwrap(), + name: RepoName::new("barnacle").unwrap(), + repo: repo_b.clone(), + created_at: UnixSeconds::new(2), + }), + &signer, + UnixSeconds::new(2), + ) + .unwrap(); + store + .create( + &knot_home, + &MembersChange::Add(Grant { + subject: AccountDid::new(COLLAB).unwrap(), + added_by: AccountDid::new(OWNER).unwrap(), + created_at: UnixSeconds::new(1), + }), + &signer, + UnixSeconds::new(1), + ) + .unwrap(); + CobStore::new(&git_a) + .create( + &CobHome::from(&repo_a), + &CollaboratorsChange::Add(Grant { + subject: AccountDid::new(COLLAB).unwrap(), + added_by: AccountDid::new(OWNER).unwrap(), + created_at: UnixSeconds::new(1), + }), + &signer, + UnixSeconds::new(1), + ) + .unwrap(); + + let index = Arc::new(Index::new(meta_path, layout.clone())); + index.rebuild().unwrap(); + index.warm_collaborators(); + + let identities = HashMap::from([ + (OWNER.to_string(), vec![owner_line]), + (COLLAB.to_string(), vec![collab_line]), + ]); + let port = launch( + &scratch.path().join("hostkey"), + layout.clone(), + Arc::clone(&index), + identities, + ) + .await; + + let work_a = scratch.path().join("work_a"); + let head_a = seed_work(&work_a); + let url_a = format!("ssh://git@127.0.0.1:{port}/{REPO_A}"); + let (ok, out) = push(&work_a, &url_a, &collab_key, &["main"]).await; + assert!( + ok, + "collaborator must push the repo it collaborates on:\n{out}" + ); + assert_eq!( + main_tip(&layout, &repo_a), + Some(Oid::from_hex(&head_a).unwrap()), + "collaborator's commit must be repo A's main tip" + ); + + let work_b = scratch.path().join("work_b"); + seed_work(&work_b); + let url_b = format!("ssh://git@127.0.0.1:{port}/{REPO_B}"); + let (denied, out) = push(&work_b, &url_b, &collab_key, &["main"]).await; + assert!( + !denied, + "key recognized via repo A but with no grant on repo B must be denied, recognition is \ + not authorization:\n{out}" + ); + assert!( + main_tip(&layout, &repo_b).is_none(), + "denied cross-repo push must land nothing on repo B" + ); + + let work_owner = scratch.path().join("work_owner_b"); + let head_owner = seed_work(&work_owner); + let (ok, out) = push(&work_owner, &url_b, &owner_key, &["main"]).await; + assert!(ok, "owner must push to repo B:\n{out}"); + assert_eq!( + main_tip(&layout, &repo_b), + Some(Oid::from_hex(&head_owner).unwrap()), + "owner's push to repo B must land, isolating the collaborator's denial as authorization" + ); +} + +fn ssh_bare(key_path: &str, port: u16) -> (bool, String) { + let out = Command::new("ssh") + .args([ + "-i", + key_path, + "-o", + "IdentitiesOnly=yes", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "-o", + "PreferredAuthentications=publickey", + "-o", + "BatchMode=yes", + "-p", + &port.to_string(), + "git@127.0.0.1", + ]) + .output() + .expect("ssh runs"); + ( + out.status.success(), + format!( + "{}{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ), + ) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_bare_ssh_session_greets_the_recognized_user() { + let fx = fixture().await; + let port = fx.server.port; + let key_path = fx.key_path.clone(); + let (_ok, out) = tokio::task::spawn_blocking(move || ssh_bare(&key_path, port)) + .await + .unwrap(); + assert!( + out.contains("@nel.pet"), + "greeting resolves and addresses the user by handle:\n{out}" + ); + assert!(out.contains("knot.test"), "greeting names the knot:\n{out}"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_push_to_a_new_branch_offers_a_pull_request_link() { + let fx = fixture().await; + seed_work(&fx.work); + + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &["main"]).await; + assert!(ok, "seeding main must land:\n{out}"); + + git(&fx.work, &[], &["checkout", "-q", "-b", "feature"]); + std::fs::write(fx.work.join("feature.txt"), "work\n").unwrap(); + git(&fx.work, &[], &["add", "-A"]); + git(&fx.work, &[], &["commit", "-q", "-m", "feature work"]); + + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &["feature"]).await; + assert!(ok, "feature-branch push must land:\n{out}"); + assert!( + out.contains("https://tangled.test/nel.pet/anemone/pulls/new"), + "new non-default branch is answered with a pull-request link:\n{out}" + ); + assert!( + out.contains("sourceBranch=feature") && out.contains("targetBranch=main"), + "link points the new branch at the default:\n{out}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_verbose_ci_push_option_reports_a_clean_pipeline() { + let fx = fixture().await; + std::fs::create_dir_all(fx.work.join(".tangled/workflows")).unwrap(); + git(&fx.work, &[], &["init", "-q", "-b", "main"]); + std::fs::write( + fx.work.join(".tangled/workflows/ci.yml"), + "engine: nixery.dev/x\nwhen:\n - event: push\n branch: ['**']\n", + ) + .unwrap(); + git(&fx.work, &[], &["add", "-A"]); + git(&fx.work, &[], &["commit", "-q", "-m", "add ci"]); + + let (ok, out) = push( + &fx.work, + &fx.url, + &fx.key_path, + &["--push-option=verbose-ci", "main"], + ) + .await; + assert!(ok, "push with a push option must land:\n{out}"); + assert!( + out.contains("no diagnostics"), + "verbose-ci reports clean compile over the sideband:\n{out}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn git_archive_remote_over_ssh_streams_a_tar_of_the_tree() { + let fx = fixture().await; + seed_work(&fx.work); + let (ok, out) = push(&fx.work, &fx.url, &fx.key_path, &["main"]).await; + assert!(ok, "seeding push must land before archiving:\n{out}"); + + let out_tar = fx.scratch.path().join("archive.tar"); + let (ok, out) = git_ssh( + &fx.work, + &fx.key_path, + &[ + "archive", + "--format=tar", + "--remote", + &fx.url, + "-o", + out_tar.to_str().unwrap(), + "HEAD", + ], + ) + .await; + assert!(ok, "git archive --remote over ssh must succeed:\n{out}"); + + let tar = std::fs::read(&out_tar).unwrap(); + assert!( + tar.windows(b"README.md".len()).any(|w| w == b"README.md"), + "archived tar must contain the README.md entry" + ); +} + +fn pkt(payload: &[u8]) -> Vec { + let mut framed = format!("{:04x}", payload.len() + 4).into_bytes(); + framed.extend_from_slice(payload); + framed +} + +fn pkt_text(line: &str) -> Vec { + pkt(format!("{line}\n").as_bytes()) +} + +fn read_until(reader: &mut impl std::io::Read, needle: &[u8], buffer: &mut Vec) { + std::iter::from_fn(|| { + let mut byte = [0u8; 1]; + match reader.read(&mut byte) { + Ok(0) | Err(_) => None, + Ok(_) => { + buffer.push(byte[0]); + Some(buffer.ends_with(needle)) + } + } + }) + .find(|done| *done) + .expect("the session must answer before closing the stream"); +} + +fn trickled_lfs_upload( + key_path: &str, + port: u16, + body: &[u8], + oid: &str, + midway: std::sync::mpsc::Sender<()>, +) -> (bool, String) { + use std::io::Write; + let mut child = Command::new("ssh") + .args([ + "-i", + key_path, + "-o", + "IdentitiesOnly=yes", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "-o", + "PreferredAuthentications=publickey", + "-o", + "BatchMode=yes", + "-p", + &port.to_string(), + "git@127.0.0.1", + &format!("git-lfs-transfer '{OWNER_DID}/{REPO_NAME}' upload"), + ]) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::null()) + .spawn() + .expect("ssh runs"); + let mut stdin = child.stdin.take().unwrap(); + let mut stdout = child.stdout.take().unwrap(); + let mut transcript = Vec::new(); + + read_until(&mut stdout, b"version=1\n0000", &mut transcript); + + let (first, second) = body.split_at(body.len() / 2); + stdin + .write_all(&pkt_text(&format!("put-object {oid}"))) + .unwrap(); + stdin + .write_all(&pkt_text(&format!("size={}", body.len()))) + .unwrap(); + stdin.write_all(b"0001").unwrap(); + first.chunks(32 * 1024).for_each(|chunk| { + stdin.write_all(&pkt(chunk)).unwrap(); + }); + stdin.flush().unwrap(); + midway.send(()).unwrap(); + std::thread::sleep(std::time::Duration::from_millis(900)); + + second.chunks(32 * 1024).for_each(|chunk| { + stdin.write_all(&pkt(chunk)).unwrap(); + }); + stdin.write_all(b"0000").unwrap(); + stdin.flush().unwrap(); + read_until(&mut stdout, b"status 200\n0000", &mut transcript); + + stdin.write_all(&pkt_text("quit")).unwrap(); + stdin.write_all(b"0000").unwrap(); + stdin.flush().unwrap(); + drop(stdin); + use std::io::Read; + let _ = stdout.read_to_end(&mut transcript); + let status = child.wait().expect("ssh exits"); + ( + status.success(), + String::from_utf8_lossy(&transcript).into_owned(), + ) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn shutdown_drains_an_in_flight_lfs_transfer_before_exit() { + use knot_lfs::LfsStore; + use sha2::Digest; + let scratch = tempfile::tempdir().unwrap(); + let (key_path, public_line) = keygen(scratch.path(), "drain"); + let lfs_dir = scratch.path().join("lfs"); + std::fs::create_dir_all(&lfs_dir).unwrap(); + let handle = knot_lfs::LfsHandle::open( + knot_lfs::LfsStorePath::new(&lfs_dir), + knot_lfs::LfsSize::new(1 << 30), + knot_lfs::FreeSpaceFloor::new(0), + ) + .unwrap(); + let (server, _index, shutdown, serve_task) = spawn_server_core( + public_line, + MaxWireBytes::new(1 << 20), + true, + Some(handle.clone()), + ) + .await; + + let body: Vec = (0..1_048_576u32).map(|n| (n % 251) as u8).collect(); + let oid = knot_lfs::LfsOid::from_digest(sha2::Sha256::digest(&body).into()); + let (midway_tx, midway_rx) = std::sync::mpsc::channel(); + + let client = { + let key_path = key_path.clone(); + let oid = oid.clone(); + let port = server.port; + tokio::task::spawn_blocking(move || { + trickled_lfs_upload(&key_path, port, &body, oid.as_str(), midway_tx) + }) + }; + + tokio::task::spawn_blocking(move || { + midway_rx + .recv_timeout(std::time::Duration::from_secs(20)) + .expect("the upload must reach its midway point") + }) + .await + .unwrap(); + + shutdown.cancel(); + tokio::time::sleep(std::time::Duration::from_millis(150)).await; + assert!( + !serve_task.is_finished(), + "the listener must keep draining while a transfer is in flight" + ); + + let (ok, transcript) = client.await.unwrap(); + assert!( + ok, + "the in-flight upload must finish cleanly across the shutdown:\n{transcript}" + ); + assert!( + transcript.contains("status 200"), + "the server must acknowledge the drained upload:\n{transcript}" + ); + + tokio::time::timeout(std::time::Duration::from_secs(10), serve_task) + .await + .expect("the drained listener must exit promptly once transfers finish") + .unwrap(); + + let repo_did = RepoDid::new(REPO_DID).unwrap(); + assert_eq!( + handle + .store + .probe(&repo_did, &oid) + .unwrap() + .map(|size| size.get()), + Some(1_048_576), + "the drained upload must be durable" + ); + + let (connected, _) = { + let key_path = key_path.clone(); + let port = server.port; + tokio::task::spawn_blocking(move || ssh_bare(&key_path, port)) + .await + .unwrap() + }; + assert!( + !connected, + "a connection after shutdown must be refused, the drain only covers in-flight work" + ); +} diff --git a/knot2/crates/knot-types/src/changes.rs b/knot2/crates/knot-types/src/changes.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-types/src/changes.rs @@ -0,0 +1,156 @@ +use std::ops::ControlFlow; + +use crate::ids::RepoPath; + +const MAX_BYTES: usize = 524_288; +const ENTRY_OVERHEAD_BYTES: usize = 48; +const MAX_ENTRIES: usize = 8_192; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum Listing { + #[default] + Complete, + Truncated, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ChangedFiles { + paths: Vec, + listing: Listing, +} + +impl ChangedFiles { + pub fn none() -> Self { + Self { + paths: Vec::new(), + listing: Listing::Complete, + } + } + + pub fn unknown() -> Self { + Self { + paths: Vec::new(), + listing: Listing::Truncated, + } + } + + pub fn paths(&self) -> &[RepoPath] { + &self.paths + } + + pub fn listing(&self) -> Listing { + self.listing + } + + pub fn into_paths(self) -> Vec { + self.paths + } +} + +#[derive(Debug, Default)] +pub struct ChangedFilesBudget { + paths: Vec, + spent: usize, + listing: Listing, +} + +impl ChangedFilesBudget { + pub fn new() -> Self { + Self::default() + } + + pub fn admit(&mut self, path: RepoPath) -> ControlFlow<()> { + let spent = self.spent + ENTRY_OVERHEAD_BYTES + path.as_str().len(); + match self.listing == Listing::Truncated + || spent > MAX_BYTES + || self.paths.len() == MAX_ENTRIES + { + true => self.truncate(), + false => { + self.spent = spent; + self.paths.push(path); + ControlFlow::Continue(()) + } + } + } + + pub fn truncate(&mut self) -> ControlFlow<()> { + self.listing = Listing::Truncated; + ControlFlow::Break(()) + } + + pub fn finish(self) -> ChangedFiles { + ChangedFiles { + listing: self.listing, + paths: self.paths, + } + } +} + +#[cfg(test)] +mod tests { + use super::{ChangedFiles, ChangedFilesBudget, Listing, MAX_BYTES, MAX_ENTRIES}; + use crate::ids::RepoPath; + use std::ops::ControlFlow; + + fn fill(mut paths: impl Iterator) -> ChangedFiles { + let mut budget = ChangedFilesBudget::new(); + let _ = paths.try_for_each(|path| budget.admit(RepoPath::new(path).unwrap())); + budget.finish() + } + + #[test] + fn a_listing_within_the_budget_stays_complete() { + let changed = fill(["a.txt", "src/deep/main.rs"].into_iter().map(String::from)); + assert_eq!(changed.listing(), Listing::Complete); + assert_eq!(changed.paths().len(), 2); + assert_eq!(ChangedFiles::none().listing(), Listing::Complete); + assert_eq!( + ChangedFiles::unknown().listing(), + Listing::Truncated, + "a listing that couldn't be computed rules no path constraint out" + ); + } + + #[test] + fn the_listing_stops_at_whichever_bound_it_hits_first() { + let wide = fill((0..MAX_ENTRIES * 2).map(|index| format!("f{index}.txt"))); + assert_eq!(wide.listing(), Listing::Truncated); + assert_eq!( + wide.paths().len(), + MAX_ENTRIES, + "the record codec refuses a longer array" + ); + + let deep = fill((0..MAX_ENTRIES).map(|index| format!("{}/f{index}.txt", "d".repeat(256)))); + assert_eq!(deep.listing(), Listing::Truncated); + assert!( + deep.paths().len() < MAX_ENTRIES, + "long paths run out the byte budget before the entry bound: {}", + deep.paths().len() + ); + } + + #[test] + fn truncation_is_sticky_and_keeps_only_the_paths_already_admitted() { + let mut budget = ChangedFilesBudget::new(); + assert_eq!( + budget.admit(RepoPath::new("a.txt").unwrap()), + ControlFlow::Continue(()) + ); + assert_eq!( + budget.admit(RepoPath::new("x".repeat(MAX_BYTES)).unwrap()), + ControlFlow::Break(()), + "a path larger than the whole remaining budget is refused" + ); + assert_eq!(budget.truncate(), ControlFlow::Break(())); + assert_eq!( + budget.admit(RepoPath::new("b.txt").unwrap()), + ControlFlow::Break(()), + "a short path after the break can't reopen the listing" + ); + let changed = budget.finish(); + assert_eq!(changed.paths(), [RepoPath::new("a.txt").unwrap()]); + assert_eq!(changed.listing(), Listing::Truncated); + } +} diff --git a/knot2/crates/knot-types/src/hex.rs b/knot2/crates/knot-types/src/hex.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-types/src/hex.rs @@ -0,0 +1,63 @@ +const HEX_DIGITS: &[u8; 16] = b"0123456789abcdef"; + +pub fn lowercase_hex(bytes: &[u8]) -> String { + bytes + .iter() + .flat_map(|byte| { + [ + HEX_DIGITS[usize::from(byte >> 4)], + HEX_DIGITS[usize::from(byte & 0x0f)], + ] + }) + .map(char::from) + .collect() +} + +pub fn decode_hex(text: impl AsRef<[u8]>) -> Option> { + let text = text.as_ref(); + match text.len() % 2 { + 0 => text + .chunks_exact(2) + .map(|pair| { + let hi = (pair[0] as char).to_digit(16)?; + let lo = (pair[1] as char).to_digit(16)?; + Some((hi * 16 + lo) as u8) + }) + .collect(), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn decode_hex_inverts_an_encoder_that_covers_every_nibble() { + [ + (&[][..], ""), + (&[0x00][..], "00"), + (&[0xff][..], "ff"), + ( + &[0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef][..], + "0123456789abcdef", + ), + ] + .iter() + .for_each(|&(bytes, expected)| { + assert_eq!(lowercase_hex(bytes), expected); + assert_eq!(decode_hex(expected).unwrap(), bytes); + }); + assert_eq!( + decode_hex("abc"), + None, + "an odd digit count can't form whole bytes" + ); + assert_eq!(decode_hex("zz"), None, "a non-hex digit decodes to nothing"); + assert_eq!( + decode_hex("AB").unwrap(), + [0xab], + "uppercase input still decodes even though the encoder emits lowercase" + ); + } +} diff --git a/knot2/crates/knot-types/src/ids.rs b/knot2/crates/knot-types/src/ids.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-types/src/ids.rs @@ -0,0 +1,1395 @@ +use std::fmt; +use std::str::FromStr; + +use jacquard_common::types::crypto::{PublicKey, multikey}; +use jacquard_common::types::string::{Did as SpecDid, Handle, Nsid, Rkey}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum ParseError { + #[error("invalid {kind}: {value:?}")] + Invalid { kind: &'static str, value: String }, +} + +impl ParseError { + fn invalid(kind: &'static str, value: impl Into) -> Self { + Self::Invalid { + kind, + value: value.into(), + } + } +} + +#[derive(Clone)] +struct Did(String); + +impl Did { + fn as_str(&self) -> &str { + &self.0 + } +} + +macro_rules! string_id { + (@traits $name:ident) => { + impl PartialEq for $name { + fn eq(&self, other: &Self) -> bool { + self.as_str() == other.as_str() + } + } + + impl Eq for $name {} + + impl PartialOrd for $name { + fn partial_cmp(&self, other: &Self) -> Option<::std::cmp::Ordering> { + Some(self.cmp(other)) + } + } + + impl Ord for $name { + fn cmp(&self, other: &Self) -> ::std::cmp::Ordering { + self.as_str().cmp(other.as_str()) + } + } + + impl ::std::hash::Hash for $name { + fn hash(&self, state: &mut H) { + self.as_str().hash(state); + } + } + + impl fmt::Debug for $name { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple(stringify!($name)).field(&self.as_str()).finish() + } + } + + impl fmt::Display for $name { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // Padding, not `write_str`'ing, + // or else every {:>20} in a log line does nothing + f.pad(self.as_str()) + } + } + + impl AsRef for $name { + fn as_ref(&self) -> &str { + self.as_str() + } + } + + impl FromStr for $name { + type Err = ParseError; + + fn from_str(s: &str) -> Result { + Self::new(s) + } + } + + impl Serialize for $name { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.as_str()) + } + } + + impl<'de> Deserialize<'de> for $name { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let raw = String::deserialize(deserializer)?; + Self::new(raw).map_err(serde::de::Error::custom) + } + } + }; + ($name:ident, $label:literal, via $parse:path => $inner:ty) => { + #[derive(Clone)] + pub struct $name($inner); + + impl $name { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + match $parse(&value) { + Some(inner) => Ok(Self(inner)), + None => Err(ParseError::invalid($label, value)), + } + } + + pub fn as_str(&self) -> &str { + self.0.as_str() + } + } + + string_id!(@traits $name); + }; + ($name:ident, $label:literal, $parse:path) => { + #[derive(Clone)] + pub struct $name(String); + + impl $name { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + $parse(&value).ok_or_else(|| ParseError::invalid($label, value)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + } + + string_id!(@traits $name); + }; +} + +// Names + emails come from commit objects that someone lovingly wrote, +// so refusing them probably isn't very good; +// they go back out into the author-line of a commit we write, +// in which a newline would create the rest of the header. +crate::text_newtype! { + pub struct AuthorName(String) => strip_control; + pub struct Email(String) => strip_control; +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct LanguageName(&'static str); + +impl LanguageName { + pub const fn new(name: &'static str) -> Self { + LanguageName(name) + } + + pub fn as_str(&self) -> &'static str { + self.0 + } +} + +impl fmt::Display for LanguageName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.pad(self.0) + } +} + +impl Serialize for LanguageName { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(self.0) + } +} + +fn parse_did(value: &str) -> Option { + SpecDid::::new_owned(value) + .ok() + .map(|did| canonical_did(did.as_str())) +} + +fn canonical_did(did: &str) -> Did { + match did.strip_prefix("did:web:") { + Some(msid) => { + let (authority, path) = match msid.split_once(':') { + Some((authority, path)) => (authority, Some(path)), + None => (msid, None), + }; + let authority = lowercase_preserving_percent(authority); + Did(match path { + Some(path) => format!("did:web:{authority}:{path}"), + None => format!("did:web:{authority}"), + }) + } + None => Did(did.to_string()), + } +} + +fn lowercase_preserving_percent(authority: &str) -> String { + authority + .chars() + .scan(0u8, |pending, ch| { + let emitted: String = if *pending > 0 { + *pending -= 1; + ch.to_string() + } else if ch == '%' { + *pending = 2; + ch.to_string() + } else { + ch.to_lowercase().collect() + }; + Some(emitted) + }) + .collect() +} + +fn parse_nsid(value: &str) -> Option> { + Nsid::new_owned(value).ok() +} + +fn parse_multikey(value: &str) -> Option { + PublicKey::decode(value) + .is_ok() + .then(|| ActorId(value.to_string())) +} + +fn parse_repo_name(value: &str) -> Option { + is_repo_name(value).then(|| RepoName(value.to_string())) +} + +fn parse_rkey(value: &str) -> Option> { + Rkey::new_owned(value).ok() +} + +fn parse_ref_name(value: &str) -> Option { + is_ref_name(value).then(|| RefName(value.to_string())) +} + +fn is_bare_host(value: &str) -> bool { + !value.contains([':', '/']) && KnotId::new(format!("did:web:{value}")).is_ok() +} + +fn parse_knot_hostname(value: &str) -> Option { + is_bare_host(value).then(|| KnotHostname(value.to_string())) +} + +fn parse_logs_host(value: &str) -> Option { + is_bare_host(value).then(|| LogsHost(value.to_string())) +} + +fn parse_ci_logs_addr(value: &str) -> Option { + let url = url::Url::parse(&format!("ssh://{value}")).ok()?; + (url.path().is_empty() + && url.username().is_empty() + && url.password().is_none() + && url.query().is_none() + && url.fragment().is_none()) + .then_some(())?; + Some(CiLogsAddr { + host: LogsHost::new(url.host_str()?).ok()?, + port: LogsPort::new(url.port()?)?, + }) +} + +fn parse_branch_name(value: &str) -> Option { + RefName::new(format!("refs/heads/{value}")) + .is_ok() + .then(|| BranchName(value.to_string())) +} + +fn parse_tag_name(value: &str) -> Option { + RefName::new(format!("refs/tags/{value}")) + .is_ok() + .then(|| TagName(value.to_string())) +} + +fn parse_repo_path(value: &str) -> Option { + (!value.is_empty() + && value + .split('/') + .all(|part| !part.is_empty() && part != "." && part != ".." && !part.contains('\0'))) + .then(|| RepoPath(value.to_string())) +} + +fn parse_http_base(value: &str) -> Option { + let trimmed = value.trim_end_matches('/'); + let rest = trimmed + .strip_prefix("https://") + .or_else(|| trimmed.strip_prefix("http://"))?; + let url = url::Url::parse(trimmed).ok()?; + (!rest.starts_with('/') + && url.has_host() + && url.username().is_empty() + && url.password().is_none() + && url.query().is_none() + && url.fragment().is_none() + && !trimmed.chars().any(|c| c.is_whitespace() || c.is_control())) + .then_some(url) +} + +fn http_base_string(url: url::Url) -> String { + url.as_str().trim_end_matches('/').to_string() +} + +fn parse_service_url(value: &str) -> Option { + parse_http_base(value) + .filter(|url| matches!(url.path(), "" | "/")) + .map(|url| KnotServiceUrl(http_base_string(url))) +} + +fn parse_appview_endpoint(value: &str) -> Option { + parse_http_base(value).map(|url| AppviewEndpoint(http_base_string(url))) +} + +fn parse_push_option(value: &str) -> Option { + (!value.is_empty() && value.len() <= 1024 && !value.contains(['\0', '\n'])) + .then(|| PushOption(value.to_string())) +} + +fn is_repo_name(value: &str) -> bool { + !value.is_empty() + && value.len() <= 100 + && value != "." + && value != ".." + && !value.contains('/') + && !value.contains('\\') + && !value.contains("..") + && value.chars().all(|c| !c.is_control() && !c.is_whitespace()) +} + +fn is_ref_name(value: &str) -> bool { + value.starts_with("refs/") + && !value.ends_with('.') + && !value.contains("..") + && !value.contains("@{") + && value.split('/').all(|component| { + !component.is_empty() && !component.starts_with('.') && !component.ends_with(".lock") + }) + && value.chars().all(|c| { + !c.is_control() + && !matches!(c, ' ' | '~' | '^' | ':' | '?' | '*' | '[' | '\\' | '\u{7f}') + }) +} + +string_id!(RepoDid, "repo DID", via parse_did => Did); +string_id!(OwnerDid, "owner DID", via parse_did => Did); +string_id!(KnotId, "knot DID", via parse_did => Did); +string_id!(AccountDid, "account DID", via parse_did => Did); +string_id!(ServiceDid, "service DID", via parse_did => Did); +string_id!(RepoName, "repo name", parse_repo_name); +string_id!(RepoRkey, "repo record key", via parse_rkey => Rkey); +string_id!(RefName, "ref name", parse_ref_name); +string_id!(TypeName, "COB type name", via parse_nsid => Nsid); +string_id!(ActorId, "actor public key", parse_multikey); +string_id!(KnotHostname, "knot hostname", parse_knot_hostname); +string_id!(BranchName, "branch name", parse_branch_name); +string_id!(TagName, "tag name", parse_tag_name); +string_id!(RepoPath, "repository path", parse_repo_path); +string_id!(AppviewEndpoint, "appview endpoint", parse_appview_endpoint); +string_id!(KnotServiceUrl, "knot service URL", parse_service_url); +string_id!(LogsHost, "ci logs host", parse_logs_host); +string_id!(PushOption, "push option", parse_push_option); + +#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)] +#[serde(transparent)] +pub struct PushOptions(Vec); + +impl PushOptions { + pub const MAX: usize = 50; + + pub fn new(options: impl IntoIterator) -> Self { + Self(options.into_iter().take(Self::MAX).collect()) + } + + pub fn as_slice(&self) -> &[PushOption] { + &self.0 + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct LogsPort(std::num::NonZeroU16); + +impl LogsPort { + pub fn new(value: u16) -> Option { + std::num::NonZeroU16::new(value).map(Self) + } + + pub fn get(self) -> u16 { + self.0.get() + } +} + +impl fmt::Display for LogsPort { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CiLogsAddr { + host: LogsHost, + port: LogsPort, +} + +impl CiLogsAddr { + pub fn new(value: &str) -> Result { + parse_ci_logs_addr(value).ok_or_else(|| ParseError::invalid("ci logs address", value)) + } + + pub fn host(&self) -> &LogsHost { + &self.host + } + + pub fn port(&self) -> LogsPort { + self.port + } +} + +impl fmt::Display for CiLogsAddr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}:{}", self.host, self.port) + } +} + +#[derive(Clone)] +pub enum OwnerRef { + Did(OwnerDid), + Handle(Handle), +} + +impl OwnerRef { + pub fn parse(segment: &str) -> Option { + match OwnerDid::new(segment) { + Ok(did) => Some(Self::Did(did)), + // Trying a handle only when the segment isn't `did:`-prefixed! + // Not perfect by any means but avoids a network call if DID + // is malformed. + Err(_) if !segment.starts_with("did:") => { + Handle::new_owned(segment).ok().map(Self::Handle) + } + Err(_) => None, + } + } +} + +impl KnotServiceUrl { + pub fn authority(&self) -> &str { + self.0 + .split_once("://") + .map(|(_, rest)| rest) + .unwrap_or(&self.0) + } +} + +impl KnotHostname { + pub fn knot_did(&self) -> KnotId { + KnotId::new(format!("did:web:{}", self.0)).expect("validated knot hostname forms did:web") + } +} + +impl BranchName { + pub fn head_ref(&self) -> RefName { + RefName::new(format!("refs/heads/{}", self.0)) + .expect("validated branch name forms refs/heads ref") + } +} + +impl TagName { + pub fn tag_ref(&self) -> RefName { + RefName::new(format!("refs/tags/{}", self.0)) + .expect("validated tag name forms refs/tags ref") + } +} + +impl RefName { + pub fn branch_name(&self) -> Option { + self.0 + .strip_prefix("refs/heads/") + .map(|name| BranchName(name.to_string())) + } + + pub fn tag_name(&self) -> Option { + self.0 + .strip_prefix("refs/tags/") + .map(|name| TagName(name.to_string())) + } +} + +impl RepoPath { + pub fn components(&self) -> impl Iterator { + self.0.split('/') + } + + pub fn names_dot_git(&self) -> bool { + self.components() + .any(|part| part.eq_ignore_ascii_case(".git")) + } + + pub fn parent(&self) -> Option { + self.0 + .rsplit_once('/') + .map(|(dir, _)| RepoPath(dir.to_string())) + } + + pub fn file_name(&self) -> &str { + self.0 + .rsplit_once('/') + .map(|(_, name)| name) + .unwrap_or(&self.0) + } +} + +impl OwnerDid { + pub fn is(&self, account: &AccountDid) -> bool { + self.as_str() == account.as_str() + } +} + +impl From for AccountDid { + fn from(owner: OwnerDid) -> Self { + AccountDid(owner.0) + } +} + +impl From for OwnerDid { + fn from(account: AccountDid) -> Self { + OwnerDid(account.0) + } +} + +impl From for AccountDid { + fn from(repo: RepoDid) -> Self { + AccountDid(repo.0) + } +} + +impl RepoRkey { + pub fn clone_path_candidates(raw: &str) -> impl Iterator + '_ { + std::iter::once(raw) + .chain(raw.strip_suffix(".git")) + .filter_map(|candidate| Self::new(candidate).ok()) + } +} + +const SECP256K1_MULTICODEC: u64 = 0xe7; + +impl ActorId { + pub fn from_secp256k1(sec1_bytes: &[u8]) -> Self { + Self(multikey(SECP256K1_MULTICODEC, sec1_bytes)) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct OfferedKey(Vec); + +impl OfferedKey { + pub fn from_bytes(bytes: impl Into>) -> Self { + Self(bytes.into()) + } + + pub fn as_bytes(&self) -> &[u8] { + &self.0 + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct Oid(gix_hash::ObjectId); + +impl Oid { + pub const fn null() -> Self { + Self(gix_hash::ObjectId::null(gix_hash::Kind::Sha1)) + } + + pub fn is_null(self) -> bool { + self.0.is_null() + } + + pub fn from_hex(hex: &str) -> Result { + gix_hash::ObjectId::from_hex(hex.as_bytes()) + .map(Self) + .map_err(|_| ParseError::invalid("oid", hex)) + } + + pub fn object_id(self) -> gix_hash::ObjectId { + self.0 + } + + pub fn to_hex(self) -> String { + self.0.to_hex().to_string() + } +} + +impl From for Oid { + fn from(value: gix_hash::ObjectId) -> Self { + Self(value) + } +} + +impl From for gix_hash::ObjectId { + fn from(value: Oid) -> Self { + value.0 + } +} + +impl FromStr for Oid { + type Err = ParseError; + + fn from_str(s: &str) -> Result { + Self::from_hex(s) + } +} + +impl fmt::Display for Oid { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0.to_hex(), f) + } +} + +impl Serialize for Oid { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.to_hex()) + } +} + +impl<'de> Deserialize<'de> for Oid { + fn deserialize>(deserializer: D) -> Result { + let hex = String::deserialize(deserializer)?; + Self::from_hex(&hex).map_err(serde::de::Error::custom) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum RefTransition { + Create { new: Oid }, + Advance { old: Oid, new: Oid }, + Delete { old: Oid }, +} + +impl RefTransition { + pub fn old_oid(self) -> Option { + match self { + Self::Create { .. } => None, + Self::Advance { old, .. } | Self::Delete { old } => Some(old), + } + } + + pub fn new_oid(self) -> Option { + match self { + Self::Create { new } | Self::Advance { new, .. } => Some(new), + Self::Delete { .. } => None, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct ObjectFormat(gix_hash::Kind); + +impl ObjectFormat { + pub const SHA1: Self = Self(gix_hash::Kind::Sha1); + pub const SHA256: Self = Self(gix_hash::Kind::Sha256); + + pub fn from_kind(kind: gix_hash::Kind) -> Self { + Self(kind) + } + + pub fn kind(self) -> gix_hash::Kind { + self.0 + } + + pub fn from_capability(token: &str) -> Option { + match token { + "sha1" => Some(Self::SHA1), + "sha256" => Some(Self::SHA256), + _ => None, + } + } + + pub fn capability(self) -> &'static str { + match self.0 { + gix_hash::Kind::Sha256 => "sha256", + _ => "sha1", + } + } + + pub fn null_oid(self) -> Oid { + Oid(self.0.null()) + } +} + +impl Default for ObjectFormat { + fn default() -> Self { + Self::SHA1 + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(transparent)] +pub struct UnixSeconds(i64); + +impl UnixSeconds { + pub const fn new(seconds: i64) -> Self { + Self(seconds) + } + + pub const fn get(self) -> i64 { + self.0 + } + + pub const fn saturating_add_secs(self, secs: i64) -> Self { + Self(self.0.saturating_add(secs)) + } + + pub const fn saturating_sub_secs(self, secs: i64) -> Self { + Self(self.0.saturating_sub(secs)) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct UnixMicros(u64); + +impl UnixMicros { + pub const fn new(micros: u64) -> Self { + Self(micros) + } + + pub const fn get(self) -> u64 { + self.0 + } + + pub const fn next(self) -> Self { + Self(self.0.saturating_add(1)) + } +} + +impl fmt::Display for UnixSeconds { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct ObjectCount(usize); + +impl ObjectCount { + pub const fn new(count: usize) -> Self { + Self(count) + } + + pub const fn get(self) -> usize { + self.0 + } + + pub const fn succ(self) -> Self { + Self(self.0.saturating_add(1)) + } +} + +impl From for ObjectCount { + fn from(count: u32) -> Self { + Self(count as usize) + } +} + +impl fmt::Display for ObjectCount { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } +} + +#[derive( + Debug, Default, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize, +)] +#[serde(transparent)] +pub struct LanguageBytes(u64); + +impl LanguageBytes { + pub const fn new(bytes: u64) -> Self { + Self(bytes) + } + + pub const fn get(self) -> u64 { + self.0 + } + + pub const fn saturating_add_bytes(self, bytes: u64) -> Self { + Self(self.0.saturating_add(bytes)) + } +} + +impl fmt::Display for LanguageBytes { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(transparent)] +pub struct HttpStatus(u16); + +impl HttpStatus { + pub const fn new(code: u16) -> Self { + Self(code) + } + + pub const fn get(self) -> u16 { + self.0 + } + + pub const fn is_success(self) -> bool { + self.0 >= 200 && self.0 < 300 + } + + pub const fn is_server_error(self) -> bool { + self.0 >= 500 && self.0 < 600 + } + + pub const fn is_transient(self) -> bool { + self.0 == 429 || self.is_server_error() + } +} + +impl From for HttpStatus { + fn from(status: http::StatusCode) -> Self { + Self(status.as_u16()) + } +} + +impl fmt::Display for HttpStatus { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct CobId(Oid); + +impl CobId { + pub fn new(oid: Oid) -> Self { + Self(oid) + } + + pub fn oid(self) -> Oid { + self.0 + } +} + +impl fmt::Display for CobId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } +} + +impl Serialize for CobId { + fn serialize(&self, serializer: S) -> Result { + self.0.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for CobId { + fn deserialize>(deserializer: D) -> Result { + Oid::deserialize(deserializer).map(Self) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct ChangeId(Oid); + +impl ChangeId { + pub fn new(oid: Oid) -> Self { + Self(oid) + } + + pub fn oid(self) -> Oid { + self.0 + } +} + +impl fmt::Display for ChangeId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } +} + +impl Serialize for ChangeId { + fn serialize(&self, serializer: S) -> Result { + self.0.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for ChangeId { + fn deserialize>(deserializer: D) -> Result { + Oid::deserialize(deserializer).map(Self) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn dids_validate_and_canonicalize() { + [ + ("did:plc:nel", Some("did:plc:nel")), + ( + "did:plc:7iza6de2dwap2sbkpav7c6c6", + Some("did:plc:7iza6de2dwap2sbkpav7c6c6"), + ), + ("did:web:oyster.cafe", Some("did:web:oyster.cafe")), + ("did:web:OYSTER.cafe", Some("did:web:oyster.cafe")), + ("did:web:Oyster.Cafe", Some("did:web:oyster.cafe")), + ("at://did:plc:nel", Some("did:plc:nel")), + ("not-a-did", None), + ("", None), + ] + .iter() + .for_each(|&(input, expected)| { + assert_eq!( + RepoDid::new(input).ok().as_ref().map(|did| did.as_str()), + expected, + "{input:?}" + ); + }); + } + + #[test] + fn owner_is_account_compares_by_canonical_did() { + [ + ("did:web:OYSTER.cafe", "did:web:oyster.cafe", true), + ("did:plc:nel", "did:plc:nel", true), + ("did:plc:ABC", "did:plc:abc", false), + ] + .iter() + .for_each(|&(owner, account, matches)| { + assert_eq!( + OwnerDid::new(owner) + .unwrap() + .is(&AccountDid::new(account).unwrap()), + matches, + "{owner:?} vs {account:?}" + ); + }); + } + + #[test] + fn repo_names_reject_traversal() { + assert!(RepoName::new("anemone").is_ok()); + assert!(RepoName::new("with-dash_and.dot").is_ok()); + assert!(RepoName::new("../escape").is_err()); + assert!(RepoName::new("a/b").is_err()); + assert!(RepoName::new("..").is_err()); + assert!(RepoName::new("").is_err()); + } + + #[test] + fn repo_rkeys_follow_record_key_rules() { + assert!(RepoRkey::new("anemone").is_ok()); + assert!(RepoRkey::new("3lkz2mvgyx22a").is_ok()); + assert!(RepoRkey::new("with-dash_and.dot~tilde:colon").is_ok()); + assert!(RepoRkey::new(".").is_err()); + assert!(RepoRkey::new("..").is_err()); + assert!(RepoRkey::new("a/b").is_err()); + assert!(RepoRkey::new("space bar").is_err()); + assert!(RepoRkey::new("").is_err()); + assert!(RepoRkey::new("x".repeat(513)).is_err()); + } + + #[test] + fn appview_endpoints_require_an_http_base_url() { + assert_eq!( + AppviewEndpoint::new("https://tangled.test/") + .unwrap() + .as_str(), + "https://tangled.test" + ); + assert_eq!( + AppviewEndpoint::new("https://tangled.test//") + .unwrap() + .as_str(), + "https://tangled.test" + ); + assert!(AppviewEndpoint::new("http://appview.oyster.cafe/base").is_ok()); + assert_eq!( + AppviewEndpoint::new("https://TANGLED.test") + .unwrap() + .as_str(), + "https://tangled.test" + ); + assert_eq!( + AppviewEndpoint::new("https://tangled.test:443") + .unwrap() + .as_str(), + "https://tangled.test" + ); + assert_eq!( + AppviewEndpoint::new("https://tangled.test:8443/base") + .unwrap() + .as_str(), + "https://tangled.test:8443/base" + ); + assert!(AppviewEndpoint::new("tangled.test").is_err()); + assert!(AppviewEndpoint::new("https://tangled.test:+8443").is_err()); + assert!(AppviewEndpoint::new("ftp://tangled.test").is_err()); + assert!(AppviewEndpoint::new("https://").is_err()); + assert!(AppviewEndpoint::new("https:///pulls").is_err()); + assert!(AppviewEndpoint::new("https://tangled.test/a b").is_err()); + assert!(AppviewEndpoint::new("https://nel@tangled.test").is_err()); + assert!(AppviewEndpoint::new("https://nel:hunter2@tangled.test").is_err()); + assert!(AppviewEndpoint::new("https://tangled.test?utm=knot").is_err()); + assert!(AppviewEndpoint::new("https://tangled.test#pulls").is_err()); + assert!(AppviewEndpoint::new("https://tangled.test#").is_err()); + } + + #[test] + fn clone_path_candidates_try_the_exact_rkey_before_the_stripped_one() { + let suffixed: Vec = RepoRkey::clone_path_candidates("anemone.git").collect(); + assert_eq!( + suffixed, + vec![ + RepoRkey::new("anemone.git").unwrap(), + RepoRkey::new("anemone").unwrap() + ], + "literal .git rkey wins over conventional suffix interpretation" + ); + + let plain: Vec = RepoRkey::clone_path_candidates("anemone").collect(); + assert_eq!(plain, vec![RepoRkey::new("anemone").unwrap()]); + + let bare: Vec = RepoRkey::clone_path_candidates(".git").collect(); + assert_eq!( + bare, + vec![RepoRkey::new(".git").unwrap()], + "stripping .git from bare suffix leaves nothing valid to try" + ); + + assert_eq!(RepoRkey::clone_path_candidates("a/b.git").count(), 0); + } + + #[test] + fn repo_paths_reject_traversal_and_expose_structure() { + assert!(RepoPath::new("src/lib.rs").is_ok()); + assert!(RepoPath::new("a b/c.txt").is_ok()); + assert!(RepoPath::new("back\\slash.txt").is_ok()); + assert!(RepoPath::new(".git/config").is_ok()); + assert!(RepoPath::new("../escape").is_err()); + assert!(RepoPath::new("nested/../escape").is_err()); + assert!(RepoPath::new("nested/./here").is_err()); + assert!(RepoPath::new("/absolute").is_err()); + assert!(RepoPath::new("trailing/").is_err()); + assert!(RepoPath::new("double//slash").is_err()); + assert!(RepoPath::new("").is_err()); + assert!(RepoPath::new("nul\0byte").is_err()); + + let path = RepoPath::new("src/deep/lib.rs").unwrap(); + assert_eq!(path.parent().unwrap().as_str(), "src/deep"); + assert_eq!(path.file_name(), "lib.rs"); + assert!(path.parent().unwrap().parent().unwrap().parent().is_none()); + assert_eq!( + path.components().collect::>(), + vec!["src", "deep", "lib.rs"] + ); + assert!(RepoPath::new(".git/hooks").unwrap().names_dot_git()); + assert!(RepoPath::new("dir/.GIT/x").unwrap().names_dot_git()); + assert!(!RepoPath::new("gitless").unwrap().names_dot_git()); + } + + #[test] + fn ref_transitions_expose_their_edge_oids() { + let before = Oid::from_hex(&"1".repeat(40)).unwrap(); + let after = Oid::from_hex(&"2".repeat(40)).unwrap(); + let create = RefTransition::Create { new: after }; + let advance = RefTransition::Advance { + old: before, + new: after, + }; + let delete = RefTransition::Delete { old: before }; + assert_eq!(create.old_oid(), None); + assert_eq!(create.new_oid(), Some(after)); + assert_eq!(advance.old_oid(), Some(before)); + assert_eq!(advance.new_oid(), Some(after)); + assert_eq!(delete.old_oid(), Some(before)); + assert_eq!(delete.new_oid(), None); + } + + #[test] + fn ref_names_follow_git_rules() { + assert!(RefName::new("refs/heads/main").is_ok()); + assert!(RefName::new("refs/heads/feature/x").is_ok()); + assert!(RefName::new("refs/cobs/sh.tangled.repo.collaborator/limpet").is_ok()); + assert!(RefName::new("refs/heads/bad..name").is_err()); + assert!(RefName::new("refs/heads/space bar").is_err()); + assert!(RefName::new("trailing.lock").is_err()); + assert!(RefName::new("/leading").is_err()); + assert!(RefName::new("refs/heads/.hidden").is_err()); + assert!(RefName::new("refs/x.lock/y").is_err()); + assert!(RefName::new("refs/heads/ends.").is_err()); + assert!(RefName::new("refs/heads//double").is_err()); + assert!(RefName::new("refs/heads/trailing/").is_err()); + assert!(RefName::new("HEAD").is_err()); + assert!(RefName::new("CONFIG").is_err()); + assert!(RefName::new("config").is_err()); + assert!(RefName::new("FETCH_HEAD").is_err()); + assert!(RefName::new("main").is_err()); + assert!(RefName::new("refs").is_err()); + assert!(RefName::new("").is_err()); + } + + #[test] + fn oid_roundtrips_through_hex() { + let hex = "0123456789abcdef0123456789abcdef01234567"; + let oid = Oid::from_hex(hex).expect("valid sha1 hex"); + assert_eq!(oid.to_hex(), hex); + assert_eq!(oid, Oid::from(oid.object_id())); + assert!(Oid::from_hex("zz").is_err()); + assert!(Oid::null().is_null()); + assert!(!oid.is_null()); + } + + #[test] + fn object_format_round_trips_its_capability_token() { + assert_eq!(ObjectFormat::SHA1.capability(), "sha1"); + assert_eq!(ObjectFormat::SHA256.capability(), "sha256"); + assert_eq!( + ObjectFormat::from_capability("sha1"), + Some(ObjectFormat::SHA1) + ); + assert_eq!( + ObjectFormat::from_capability("sha256"), + Some(ObjectFormat::SHA256) + ); + assert_eq!(ObjectFormat::from_capability("md5"), None); + assert_eq!(ObjectFormat::default(), ObjectFormat::SHA1); + } + + #[test] + fn object_format_null_oid_matches_the_hash_width() { + let sha1 = ObjectFormat::SHA1.null_oid(); + let sha256 = ObjectFormat::SHA256.null_oid(); + assert_eq!(sha1.to_hex().len(), 40); + assert_eq!(sha256.to_hex().len(), 64); + assert!(sha1.is_null()); + assert!(sha256.is_null()); + assert_eq!(sha1.object_id().kind(), gix_hash::Kind::Sha1); + assert_eq!(sha256.object_id().kind(), gix_hash::Kind::Sha256); + } + + #[test] + fn type_name_is_an_nsid() { + assert!(TypeName::new("sh.tangled.repo.collaborator").is_ok()); + assert!(TypeName::new("not an nsid").is_err()); + } + + #[test] + fn knot_service_url_validates_and_exposes_its_authority() { + let url = KnotServiceUrl::new("https://knot.nel.pet/").unwrap(); + assert_eq!(url.as_str(), "https://knot.nel.pet"); + assert_eq!(url.authority(), "knot.nel.pet"); + assert_eq!( + KnotServiceUrl::new("https://knot.nel.pet:8443") + .unwrap() + .authority(), + "knot.nel.pet:8443" + ); + assert_eq!( + KnotServiceUrl::new("https://knot.nel.pet:443") + .unwrap() + .authority(), + "knot.nel.pet" + ); + assert!(KnotServiceUrl::new("knot.nel.pet").is_err()); + assert!(KnotServiceUrl::new("ftp://knot.nel.pet").is_err()); + assert!(KnotServiceUrl::new("https://knot.nel.pet/base").is_err()); + assert!(KnotServiceUrl::new("https://nel@knot.nel.pet").is_err()); + assert!(KnotServiceUrl::new("https://nel:hunter2@knot.nel.pet").is_err()); + assert!(KnotServiceUrl::new("https://knot.nel.pet?utm=knot").is_err()); + assert!(KnotServiceUrl::new("https://knot.nel.pet#pulls").is_err()); + assert!(KnotServiceUrl::new("https://").is_err()); + assert!(KnotServiceUrl::new("").is_err()); + } + + #[test] + fn knot_hostname_validates_and_forms_did_web() { + let host = KnotHostname::new("oyster.cafe").unwrap(); + assert_eq!(host.knot_did(), KnotId::new("did:web:oyster.cafe").unwrap()); + assert!(KnotHostname::new("").is_err()); + assert!(KnotHostname::new("not a host").is_err()); + assert!(KnotHostname::new("knot.nel.pet:8443").is_err()); + assert!(KnotHostname::new("knot.nel.pet/path").is_err()); + } + + #[test] + fn a_ci_logs_address_splits_into_a_bare_host_and_a_nonzero_port() { + let addr = CiLogsAddr::new("logs.oyster.cafe:3333").unwrap(); + assert_eq!(addr.host().as_str(), "logs.oyster.cafe"); + assert_eq!(addr.port().get(), 3333); + assert_eq!(addr.to_string(), "logs.oyster.cafe:3333"); + + [ + "logs.oyster.cafe", + "logs.oyster.cafe:0", + ":3333", + "::1", + "logs.oyster.cafe:+3333", + "logs.oyster.cafe:65536", + "logs.oyster.cafe: 3333", + "logs.oyster.cafe:33:33", + "logs.oyster.cafe:3333/logs", + "nel@logs.oyster.cafe:3333", + "logs.oyster.cafe:3333?tail=1", + "logs.oyster.cafe:3333#tail", + ] + .into_iter() + .for_each(|value| { + assert!(CiLogsAddr::new(value).is_err(), "{value}"); + }); + } + + #[test] + fn push_options_enforce_the_lexicon_bounds_at_construction() { + assert_eq!( + PushOption::new("verbose-ci").unwrap().as_str(), + "verbose-ci" + ); + assert!(PushOption::new("x".repeat(1024)).is_ok()); + assert!(PushOption::new("x".repeat(1025)).is_err()); + assert!(PushOption::new("").is_err()); + assert!(PushOption::new("has\0nul").is_err()); + + let options = PushOptions::new( + (0..PushOptions::MAX + 10) + .map(|index| PushOption::new(format!("option-{index}")).unwrap()), + ); + assert_eq!(options.as_slice().len(), PushOptions::MAX); + assert_eq!(options.as_slice()[0].as_str(), "option-0"); + assert!(PushOptions::default().is_empty()); + } + + #[test] + fn branch_name_validates_and_forms_head_ref() { + let branch = BranchName::new("main").unwrap(); + assert_eq!(branch.head_ref(), RefName::new("refs/heads/main").unwrap()); + assert!(BranchName::new("feature/x").is_ok()); + assert!(BranchName::new("bad..name").is_err()); + assert!(BranchName::new("").is_err()); + } + + #[test] + fn author_text_strips_nul_and_newline_at_construction() { + assert_eq!(AuthorName::new("nel\nbailey").as_str(), "nelbailey"); + assert_eq!(Email::new("nel@oyster.cafe\0").as_str(), "nel@oyster.cafe"); + assert_eq!(AuthorName::new("teq").as_str(), "teq"); + } + + #[test] + fn author_text_strips_nul_and_newline_on_deserialize() { + let author: AuthorName = serde_json::from_str("\"nel\\nbailey\"").unwrap(); + assert_eq!(author.as_str(), "nelbailey"); + let email: Email = serde_json::from_str("\"nel@oyster.cafe\\u0000\"").unwrap(); + assert_eq!(email.as_str(), "nel@oyster.cafe"); + } + + #[test] + fn tag_name_validates_and_forms_tag_ref() { + let tag = TagName::new("v1.2.3").unwrap(); + assert_eq!(tag.tag_ref(), RefName::new("refs/tags/v1.2.3").unwrap()); + assert!(TagName::new("release/v1").is_ok()); + assert!(TagName::new("bad..name").is_err()); + assert!(TagName::new("v1.lock").is_err()); + assert!(TagName::new("").is_err()); + } + + #[test] + fn unix_seconds_arithmetic_saturates_at_the_bounds() { + let base = UnixSeconds::new(1_000); + assert_eq!(base.saturating_add_secs(60), UnixSeconds::new(1_060)); + assert_eq!(base.saturating_sub_secs(60), UnixSeconds::new(940)); + assert_eq!( + UnixSeconds::new(i64::MAX).saturating_add_secs(1), + UnixSeconds::new(i64::MAX) + ); + assert_eq!( + UnixSeconds::new(i64::MIN).saturating_sub_secs(1), + UnixSeconds::new(i64::MIN) + ); + assert_eq!(base.get(), 1_000); + assert_eq!(base.to_string(), "1000"); + } + + #[test] + fn http_status_classifies_transient_codes() { + assert!(HttpStatus::new(200).is_success()); + assert!(!HttpStatus::new(200).is_transient()); + assert!(HttpStatus::new(429).is_transient()); + assert!(HttpStatus::new(503).is_transient()); + assert!(HttpStatus::new(503).is_server_error()); + assert!(!HttpStatus::new(404).is_transient()); + assert_eq!(HttpStatus::new(404).get(), 404); + } + + #[test] + fn actor_id_wraps_a_secp256k1_multikey() { + let mut compressed = [0u8; 33]; + compressed[0] = 0x02; + let actor = ActorId::from_secp256k1(&compressed); + let decoded = + PublicKey::decode(actor.as_str()).expect("from_secp256k1 yields decodable multikey"); + assert_eq!( + decoded.codec, + jacquard_common::types::crypto::KeyCodec::Secp256k1 + ); + assert_eq!(decoded.bytes.as_ref(), &compressed); + assert!(ActorId::new("not-a-multikey").is_err()); + } +} + +#[cfg(test)] +mod prop_tests { + use super::*; + use proptest::prelude::*; + + const DID: &str = "did:(plc:[a-z2-7]{24}|web:[a-z][a-z0-9-]{0,20}\\.(cafe|pet|dev))"; + + macro_rules! parse_display_identity { + ($name:ident, $ty:ty, $strategy:expr) => { + proptest! { + #[test] + fn $name(raw in $strategy) { + if let Ok(value) = <$ty>::new(raw.clone()) { + prop_assert_eq!(value.as_str(), raw.as_str()); + let reparsed = <$ty>::new(value.to_string()) + .expect("display output reparses"); + prop_assert_eq!(reparsed, value); + } + } + } + }; + } + + parse_display_identity!(repo_did_identity, RepoDid, DID); + parse_display_identity!(owner_did_identity, OwnerDid, DID); + parse_display_identity!(knot_id_identity, KnotId, DID); + parse_display_identity!(account_did_identity, AccountDid, DID); + parse_display_identity!( + repo_name_identity, + RepoName, + "[A-Za-z0-9][A-Za-z0-9._-]{0,40}" + ); + parse_display_identity!( + repo_rkey_identity, + RepoRkey, + "[A-Za-z0-9][A-Za-z0-9._:~-]{0,40}" + ); + parse_display_identity!( + ref_name_identity, + RefName, + "refs/(heads|tags|cobs)/[a-z][a-z0-9]{0,7}(/[a-z][a-z0-9]{0,7}){0,3}" + ); + parse_display_identity!( + repo_path_identity, + RepoPath, + "[a-z][a-z0-9 ._-]{0,7}(/[a-z][a-z0-9 ._-]{0,7}){0,3}" + ); + parse_display_identity!( + type_name_identity, + TypeName, + "[a-z][a-z0-9]{0,7}(\\.[a-z][a-z0-9]{0,7}){2,4}" + ); + + proptest! { + #[test] + fn oid_hex_identity(hex in "[0-9a-f]{40}") { + let oid = Oid::from_hex(&hex).expect("forty lowercase hex chars are valid sha1"); + prop_assert_eq!(oid.to_hex(), hex.clone()); + prop_assert_eq!(Oid::from_hex(&oid.to_hex()).expect("reparse"), oid); + } + + #[test] + fn actor_id_identity(tag in 2u8..=3u8, body in prop::collection::vec(any::(), 32..=32)) { + let sec1: Vec = std::iter::once(tag).chain(body).collect(); + let actor = ActorId::from_secp256k1(&sec1); + let encoded = actor.as_str().to_string(); + let reparsed = ActorId::new(encoded.clone()).expect("multikey output reparses"); + prop_assert_eq!(reparsed.as_str(), encoded.as_str()); + prop_assert_eq!(reparsed, actor); + } + } +} diff --git a/knot2/crates/knot-types/src/lib.rs b/knot2/crates/knot-types/src/lib.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-types/src/lib.rs @@ -0,0 +1,32 @@ +#[macro_use] +mod newtype; + +mod changes; +pub use changes::{ChangedFiles, ChangedFilesBudget, Listing}; + +mod ids; +pub use ids::{ + AccountDid, ActorId, AppviewEndpoint, AuthorName, BranchName, ChangeId, CiLogsAddr, CobId, + Email, HttpStatus, KnotHostname, KnotId, KnotServiceUrl, LanguageBytes, LanguageName, LogsHost, + LogsPort, ObjectCount, ObjectFormat, OfferedKey, Oid, OwnerDid, OwnerRef, ParseError, + PushOption, PushOptions, RefName, RefTransition, RepoDid, RepoName, RepoPath, RepoRkey, + ServiceDid, TagName, TypeName, UnixMicros, UnixSeconds, +}; + +mod policy; +pub use policy::AdmissionPolicy; + +mod hex; +pub use hex::{decode_hex, lowercase_hex}; + +mod net; +pub use net::forwarded_peer; + +pub use jacquard_common::CowStr; +pub use jacquard_common::DefaultStr; +pub use jacquard_common::service_auth; +pub use jacquard_common::types::collection::Collection; +pub use jacquard_common::types::string::{ + AtUri, Cid, Datetime, Did, DidService, Handle, Nsid, Rkey, Tid, +}; +pub use jacquard_common::types::{crypto, did_doc}; diff --git a/knot2/crates/knot-types/src/net.rs b/knot2/crates/knot-types/src/net.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-types/src/net.rs @@ -0,0 +1,51 @@ +use std::net::IpAddr; + +use http::HeaderMap; +use http::header::AsHeaderName; + +pub fn forwarded_peer(headers: &HeaderMap, header: K) -> Option { + headers + .get(header) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.rsplit(',').next()) + .map(str::trim) + .and_then(|candidate| candidate.parse::().ok()) +} + +#[cfg(test)] +mod tests { + use super::*; + + use http::HeaderName; + + fn headers(value: Option<&str>) -> HeaderMap { + value + .map(|value| { + let mut map = HeaderMap::new(); + map.insert( + HeaderName::from_bytes(b"x-forwarded-for").unwrap(), + value.parse().unwrap(), + ); + map + }) + .unwrap_or_default() + } + + #[test] + fn forwarded_peer_takes_the_rightmost_parseable_entry() { + [ + (Some("203.0.113.7, 198.51.100.4"), Some("198.51.100.4")), + (Some(" 192.0.2.1 "), Some("192.0.2.1")), + (Some("not-an-ip"), None), + (None, None), + ] + .iter() + .for_each(|&(header, expected)| { + assert_eq!( + forwarded_peer(&headers(header), "x-forwarded-for"), + expected.map(|ip| ip.parse::().unwrap()), + "{header:?}" + ); + }); + } +} diff --git a/knot2/crates/knot-types/src/newtype.rs b/knot2/crates/knot-types/src/newtype.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-types/src/newtype.rs @@ -0,0 +1,238 @@ +#[macro_export] +macro_rules! scalar_newtype { + ($( + $(#[$meta:meta])* + $vis:vis struct $name:ident($prim:ty) $(=> $mode:ident)?; + )+) => {$( + $(#[$meta])* + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] + $vis struct $name($prim); + + impl $name { + $crate::scalar_ctor!($($mode)? , $prim); + + pub const fn get(self) -> $prim { + self.0 + } + } + + $crate::scalar_order!($($mode)? , $name); + )+}; +} + +#[macro_export] +macro_rules! scalar_ctor { + (, $prim:ty) => { + pub const fn new(value: $prim) -> Self { + Self(value) + } + }; + (ordered, $prim:ty) => { + $crate::scalar_ctor!(, $prim); + }; + (sealed, $prim:ty) => { + pub(crate) const fn new(value: $prim) -> Self { + Self(value) + } + }; +} + +#[macro_export] +macro_rules! scalar_order { + (, $name:ident) => {}; + (sealed, $name:ident) => {}; + (ordered, $name:ident) => { + impl ::core::cmp::Ord for $name { + fn cmp(&self, other: &Self) -> ::core::cmp::Ordering { + self.0.cmp(&other.0) + } + } + + impl ::core::cmp::PartialOrd for $name { + fn partial_cmp(&self, other: &Self) -> Option<::core::cmp::Ordering> { + Some(::core::cmp::Ord::cmp(self, other)) + } + } + }; +} + +#[macro_export] +macro_rules! text_mode { + (verbatim, $value:expr) => { + $value + }; + (strip_control, $value:expr) => { + match $value { + value if value.contains(['\0', '\n']) => value.replace(['\0', '\n'], ""), + value => value, + } + }; +} + +#[macro_export] +macro_rules! text_newtype { + ($( + $(#[$meta:meta])* + $vis:vis struct $name:ident(String) => $mode:ident; + )+) => { + $crate::text_newtype! {$( + $(#[$meta])* + $vis struct $name(String) => $mode as new; + )+} + }; + ($( + $(#[$meta:meta])* + $vis:vis struct $name:ident(String) => $mode:ident as $ctor:ident; + )+) => {$( + $(#[$meta])* + #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] + $vis struct $name(String); + + impl $name { + pub fn $ctor(value: impl Into) -> Self { + Self($crate::text_mode!($mode, value.into())) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + } + + impl ::std::fmt::Debug for $name { + fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { + f.debug_tuple(stringify!($name)).field(&self.0).finish() + } + } + + impl ::std::fmt::Display for $name { + fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { + f.pad(&self.0) + } + } + + impl ::std::convert::AsRef for $name { + fn as_ref(&self) -> &str { + &self.0 + } + } + + impl ::std::borrow::Borrow for $name { + fn borrow(&self) -> &str { + &self.0 + } + } + + impl ::serde::Serialize for $name { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.0) + } + } + + impl<'de> ::serde::Deserialize<'de> for $name { + fn deserialize>( + deserializer: D, + ) -> Result { + ::deserialize(deserializer).map(Self::$ctor) + } + } + )+}; +} + +#[macro_export] +macro_rules! read_counter { + ($name:ident, $record:ident, $read:ident, $reset:ident, $measure:ident) => { + $crate::scalar_newtype! { + pub struct $name(u64) => sealed; + } + + fn count() -> &'static ::std::thread::LocalKey<::std::cell::Cell> { + ::std::thread_local! { + static COUNT: ::std::cell::Cell = const { ::std::cell::Cell::new(0) }; + } + &COUNT + } + + pub(crate) fn $record() { + count().with(|cell| cell.set(cell.get().wrapping_add(1))); + } + + pub fn $read() -> $name { + $name::new(count().with(::std::cell::Cell::get)) + } + + pub fn $reset() { + count().with(|cell| cell.set(0)); + } + + pub fn $measure(work: impl FnOnce() -> T) -> (T, $name) { + $reset(); + let value = work(); + (value, $read()) + } + }; +} + +#[cfg(test)] +mod tests { + crate::scalar_newtype! { + pub struct Bytes(u64) => ordered; + pub struct Tag(u64); + } + + crate::text_newtype! { + pub struct Header(String) => strip_control; + pub struct Body(String) => verbatim; + } + + crate::text_newtype! { + pub struct Column(String) => strip_control as from_column; + } + + fn sorted(mut values: Vec) -> Vec { + values.sort(); + values + } + + #[test] + fn only_a_newtype_that_declares_an_order_gets_one() { + assert_eq!(Bytes::new(7).get(), 7); + assert!(Bytes::new(7) < Bytes::new(8)); + assert_eq!( + sorted(vec![Bytes::new(8), Bytes::new(7)]), + vec![Bytes::new(7), Bytes::new(8)] + ); + assert_ne!(Tag::new(7), Tag::new(8)); + assert_eq!( + Tag::new(7).get().cmp(&Tag::new(8).get()), + std::cmp::Ordering::Less, + "a plain newtype still compares through the value it wraps, \ + so a caller that wants an ordering declares one instead of inheriting it" + ); + } + + #[test] + fn a_text_newtype_applies_its_mode_through_every_constructor_it_has() { + assert_eq!(Header::new("nel\nolaren\0").as_str(), "nelolaren"); + assert_eq!(Body::new("nel\nolaren\0").as_str(), "nel\nolaren\0"); + assert_eq!( + Column::from_column("nel\nolaren").as_str(), + "nelolaren", + "a type that names its constructor for where the value comes from \ + must still apply its mode" + ); + + let header: Header = serde_json::from_str("\"nel\\nolaren\"").unwrap(); + let body: Body = serde_json::from_str("\"nel\\nolaren\"").unwrap(); + let column: Column = serde_json::from_str("\"nel\\nolaren\"").unwrap(); + assert_eq!( + (header.as_str(), body.as_str(), column.as_str()), + ("nelolaren", "nel\nolaren", "nelolaren"), + "deserialization goes through the constructor, whatever it is named" + ); + assert_eq!(serde_json::to_string(&body).unwrap(), "\"nel\\nolaren\""); + + let map: std::collections::HashMap = + [(Body::new("kelp"), 1)].into_iter().collect(); + assert_eq!(map.get("kelp"), Some(&1), "a text newtype borrows as str"); + } +} diff --git a/knot2/crates/knot-types/src/policy.rs b/knot2/crates/knot-types/src/policy.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-types/src/policy.rs @@ -0,0 +1,9 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum AdmissionPolicy { + #[default] + Closed, + Open, +} diff --git a/knot2/crates/knot-workflow/src/lib.rs b/knot2/crates/knot-workflow/src/lib.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-workflow/src/lib.rs @@ -0,0 +1,593 @@ +use globset::{GlobBuilder, GlobSet, GlobSetBuilder}; +use knot_types::{ChangedFiles, Listing, ParseError, RefName}; +use serde::Deserialize; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct WorkflowName(String); + +impl WorkflowName { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + let valid = + !value.is_empty() && !value.contains('/') && !value.chars().any(char::is_control); + match valid { + true => Ok(Self(value)), + false => Err(ParseError::Invalid { + kind: "workflow name", + value, + }), + } + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +fn engine_is_named(engine: &str) -> Result<(), ParseError> { + match !engine.chars().any(|c| c.is_whitespace() || c.is_control()) { + true => Ok(()), + false => Err(ParseError::Invalid { + kind: "engine reference", + value: engine.to_string(), + }), + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum CloneDepth { + Full, + Limited, +} + +impl CloneDepth { + fn parse(depth: i64) -> Result { + match u32::try_from(depth) { + Ok(0) => Ok(Self::Full), + Ok(_) => Ok(Self::Limited), + Err(_) => Err(ParseError::Invalid { + kind: "clone depth", + value: depth.to_string(), + }), + } + } +} + +pub struct RawWorkflow { + pub name: WorkflowName, + pub contents: Vec, +} + +pub enum Trigger { + Push { ref_name: RefName }, +} + +impl Trigger { + fn kind(&self) -> &'static str { + match self { + Trigger::Push { .. } => "push", + } + } +} + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct Diagnostics { + pub errors: Vec, + pub warnings: Vec, +} + +impl Diagnostics { + fn error(&mut self, path: &str, message: impl AsRef) { + self.errors + .push(format!("error: {path}: {}", message.as_ref())); + } + + fn warning(&mut self, path: &str, kind: &str, reason: &str) { + self.warnings + .push(format!("warning: {path}: {kind}: {reason}")); + } + + fn combine(mut self, other: Diagnostics) -> Diagnostics { + self.errors.extend(other.errors); + self.warnings.extend(other.warnings); + self + } + + pub fn is_empty(&self) -> bool { + self.errors.is_empty() && self.warnings.is_empty() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum PathMatch { + Assumed, + Listed, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CompiledWorkflow { + pub name: WorkflowName, + pub paths: PathMatch, +} + +pub struct Compiled { + pub workflows: Vec, + pub diagnostics: Diagnostics, +} + +impl Compiled { + pub fn any_listed_match(&self) -> bool { + self.workflows + .iter() + .any(|workflow| workflow.paths == PathMatch::Listed) + } +} + +pub fn compile(raw: &[RawWorkflow], trigger: &Trigger, changed: &ChangedFiles) -> Compiled { + let (workflows, diagnostics) = raw + .iter() + .map(|workflow| compile_one(workflow, trigger, changed)) + .fold( + (Vec::new(), Diagnostics::default()), + |(mut workflows, diagnostics), (compiled, diag)| { + workflows.extend(compiled); + (workflows, diagnostics.combine(diag)) + }, + ); + Compiled { + workflows, + diagnostics, + } +} + +fn compile_one( + raw: &RawWorkflow, + trigger: &Trigger, + changed: &ChangedFiles, +) -> (Option, Diagnostics) { + let mut diag = Diagnostics::default(); + let parsed = match parse(&raw.contents) { + Ok(parsed) => parsed, + Err(error) => { + diag.error(raw.name.as_str(), error.to_string()); + return (None, diag); + } + }; + let matched = match workflow_matches(&parsed.when, trigger, changed) { + Ok(matched) => matched, + Err(error) => { + diag.error( + raw.name.as_str(), + format!("failed to execute workflow: {error}"), + ); + return (None, diag); + } + }; + let Some(paths) = matched else { + diag.warning( + raw.name.as_str(), + "workflow skipped", + &format!("didn't match trigger {}", trigger.kind()), + ); + return (None, diag); + }; + let depth = match CloneDepth::parse(parsed.clone.depth) { + Ok(depth) => depth, + Err(error) => { + diag.error(raw.name.as_str(), error.to_string()); + return (None, diag); + } + }; + analyze_clone(&parsed.clone, depth, raw.name.as_str(), &mut diag); + if parsed.engine.is_empty() { + diag.error(raw.name.as_str(), "missing engine"); + return (None, diag); + } + match engine_is_named(&parsed.engine) { + Ok(()) => ( + Some(CompiledWorkflow { + name: raw.name.clone(), + paths, + }), + diag, + ), + Err(error) => { + diag.error(raw.name.as_str(), error.to_string()); + (None, diag) + } + } +} + +fn analyze_clone(clone: &CloneOpts, depth: CloneDepth, path: &str, diag: &mut Diagnostics) { + if !clone.skip { + return; + } + [ + ("tags", clone.tags.is_some()), + ("submodules", clone.submodules.is_some()), + ("depth", depth == CloneDepth::Limited), + ] + .into_iter() + .filter(|(_, set)| *set) + .for_each(|(key, _)| { + diag.warning( + path, + "invalid configuration", + &format!("`clone.{key}` has no effect with `clone.skip`"), + ); + }); +} + +fn workflow_matches( + when: &[Constraint], + trigger: &Trigger, + changed: &ChangedFiles, +) -> Result, String> { + if when.is_empty() { + return Ok(Some(PathMatch::Listed)); + } + when.iter() + .map(|constraint| constraint_matches(constraint, trigger, changed)) + .collect::>, String>>() + .map(|results| results.into_iter().flatten().max()) +} + +fn constraint_matches( + constraint: &Constraint, + trigger: &Trigger, + changed: &ChangedFiles, +) -> Result, String> { + match trigger { + Trigger::Push { ref_name } => { + let event = constraint.event.0.iter().any(|kind| kind == "push"); + let reference = match ref_kind(ref_name.as_str()) { + Some((RefKind::Branch, short)) => glob_set(&constraint.branch.0)?.is_match(short), + Some((RefKind::Tag, short)) => glob_set(&constraint.tag.0)?.is_match(short), + None => false, + }; + let globs = glob_set(&constraint.paths.0)?; + let listed = changed + .paths() + .iter() + .any(|path| globs.is_match(path.as_str())); + // We aren't the one deciding whether this workflow runs, remember, + // spindles are, and spindles will decide from the + // atproto record we will emit. + let paths = match (constraint.paths.0.is_empty(), listed, changed.listing()) { + (true, _, _) | (false, true, _) => Some(PathMatch::Listed), + (false, false, Listing::Truncated) => Some(PathMatch::Assumed), + (false, false, Listing::Complete) => None, + }; + Ok(paths.filter(|_| event && reference)) + } + } +} + +enum RefKind { + Branch, + Tag, +} + +fn ref_kind(reference: &str) -> Option<(RefKind, &str)> { + reference + .strip_prefix("refs/heads/") + .map(|short| (RefKind::Branch, short)) + .or_else(|| { + reference + .strip_prefix("refs/tags/") + .map(|short| (RefKind::Tag, short)) + }) +} + +fn glob_set(patterns: &[String]) -> Result { + patterns + .iter() + .try_fold(GlobSetBuilder::new(), |mut builder, pattern| { + GlobBuilder::new(pattern) + .literal_separator(true) + .build() + .map(|glob| { + builder.add(glob); + builder + }) + .map_err(|error| error.to_string()) + }) + .and_then(|builder| builder.build().map_err(|error| error.to_string())) +} + +fn parse(contents: &[u8]) -> Result { + serde_norway::from_slice(contents) +} + +#[derive(Debug, Default, Deserialize)] +struct WorkflowFile { + #[serde(default)] + engine: String, + #[serde(default)] + when: Vec, + #[serde(default)] + clone: CloneOpts, +} + +#[derive(Debug, Default, Deserialize)] +struct Constraint { + #[serde(default)] + event: StringList, + #[serde(default)] + branch: StringList, + #[serde(default)] + tag: StringList, + #[serde(default)] + paths: StringList, +} + +#[derive(Debug, Default, Deserialize)] +struct CloneOpts { + #[serde(default)] + skip: bool, + #[serde(default)] + depth: i64, + #[serde(default)] + submodules: Option, + #[serde(default)] + tags: Option, +} + +#[derive(Debug, Default)] +struct StringList(Vec); + +impl<'de> Deserialize<'de> for StringList { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + #[derive(Deserialize)] + #[serde(untagged)] + enum OneOrMany { + One(String), + Many(Vec), + } + Ok(match OneOrMany::deserialize(deserializer)? { + OneOrMany::One(value) => StringList(vec![value]), + OneOrMany::Many(values) => StringList(values), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn raw(name: &str, contents: &str) -> RawWorkflow { + RawWorkflow { + name: WorkflowName::new(name).unwrap(), + contents: contents.as_bytes().to_vec(), + } + } + + fn changed(values: &[&str]) -> ChangedFiles { + let mut budget = knot_types::ChangedFilesBudget::new(); + let _ = values + .iter() + .try_for_each(|value| budget.admit(knot_types::RepoPath::new(*value).unwrap())); + budget.finish() + } + + fn push(reference: &str) -> Trigger { + Trigger::Push { + ref_name: RefName::new(reference).unwrap(), + } + } + + #[test] + fn workflow_name_round_trips_through_as_str() { + let name = WorkflowName::new("test.yml").unwrap(); + assert_eq!(name.as_str(), "test.yml"); + assert_eq!(name, WorkflowName::new("test.yml".to_string()).unwrap()); + assert_ne!(name, WorkflowName::new("ci.yml").unwrap()); + } + + #[test] + fn a_matching_branch_push_compiles_the_workflow() { + let workflows = [raw( + "test.yml", + "engine: nixery.dev/x\nwhen:\n - event: push\n branch: [main]\n", + )]; + let compiled = compile(&workflows, &push("refs/heads/main"), &ChangedFiles::none()); + assert_eq!( + compiled + .workflows + .iter() + .map(|workflow| workflow.name.as_str()) + .collect::>(), + vec!["test.yml"] + ); + assert!( + compiled.any_listed_match(), + "a workflow with no paths constraint never depends on the listing" + ); + assert!( + compiled.diagnostics.is_empty(), + "{:?}", + compiled.diagnostics + ); + } + + #[test] + fn a_paths_constraint_matches_changed_files_and_never_their_parent_directories() { + let changed = changed(&["src/deep/main.rs"]); + let cases: &[(&str, usize)] = &[ + ("", 1), + ("\n paths: ['src/**']", 1), + ("\n paths: ['**/main.rs']", 1), + ("\n paths: ['docs/**']", 0), + ("\n paths: ['src']", 0), + ("\n paths: ['*']", 0), + ("\n paths: ['docs/**', 'src/**']", 1), + ]; + cases.iter().for_each(|(constraint, count)| { + let yaml = format!( + "engine: nixery.dev/x\nwhen:\n - event: push\n branch: ['**']{constraint}\n" + ); + let compiled = compile(&[raw("ci.yml", &yaml)], &push("refs/heads/main"), &changed); + assert_eq!(compiled.workflows.len(), *count, "{yaml}"); + }); + + let unmatched_branch = compile( + &[raw( + "ci.yml", + "engine: nixery.dev/x\nwhen:\n - event: push\n branch: [release]\n paths: ['[']\n", + )], + &push("refs/heads/main"), + &changed, + ); + assert!( + !unmatched_branch.diagnostics.errors.is_empty(), + "compile reports a malformed paths glob even when the branch already decided the match: {:?}", + unmatched_branch.diagnostics + ); + } + + #[test] + fn a_truncated_listing_assumes_a_paths_match_and_only_a_listed_hit_promises_the_run() { + let yaml = "engine: nixery.dev/x\nwhen:\n - event: push\n branch: ['**']\n paths: ['src/**']\n"; + assert_eq!( + compile( + &[raw("ci.yml", yaml)], + &push("refs/heads/main"), + &changed(&["docs/only.md"]) + ) + .workflows + .len(), + 0, + "a complete listing that misses the globs skips the workflow" + ); + + let assumed = compile( + &[raw("ci.yml", yaml)], + &push("refs/heads/main"), + &ChangedFiles::unknown(), + ); + assert_eq!( + assumed.workflows.len(), + 1, + "a listing the record couldn't hold rules no glob out" + ); + assert_eq!(assumed.workflows[0].paths, PathMatch::Assumed); + assert!( + !assumed.any_listed_match(), + "spindle reads the same truncated listing and skips this run" + ); + + let mut budget = knot_types::ChangedFilesBudget::new(); + let _ = budget.admit(knot_types::RepoPath::new("src/deep/main.rs").unwrap()); + let _ = budget.truncate(); + let listed = compile( + &[raw("ci.yml", yaml)], + &push("refs/heads/main"), + &budget.finish(), + ); + assert_eq!(listed.workflows[0].paths, PathMatch::Listed); + assert!( + listed.any_listed_match(), + "spindle sees the same listed path and runs this workflow" + ); + + let strongest = compile( + &[raw( + "ci.yml", + "engine: nixery.dev/x\nwhen:\n - event: push\n branch: ['**']\n paths: ['never/**']\n - event: push\n branch: ['**']\n", + )], + &push("refs/heads/main"), + &ChangedFiles::unknown(), + ); + assert_eq!( + strongest.workflows[0].paths, + PathMatch::Listed, + "an assumed constraint never weakens an unconstrained one" + ); + } + + #[test] + fn compile_cases() { + let tag = "engine: nixery.dev/x\nwhen:\n - event: push\n tag: ['v*']\n"; + let single_star = + "engine: nixery.dev/x\nwhen:\n - event: push\n branch: ['feature/*']\n"; + let cases: &[(&str, &str, usize, Option<&str>)] = &[ + ( + "engine: nixery.dev/x\nwhen:\n - event: push\n branch: [dev]\n", + "refs/heads/main", + 0, + Some("workflow skipped"), + ), + ("engine: nixery.dev/x\n", "refs/heads/anything", 1, None), + ( + "when:\n - event: push\n branch: ['*']\n", + "refs/heads/main", + 0, + Some("missing engine"), + ), + ( + "engine: nixery.dev/x\nclone:\n skip: true\n submodules: true\n", + "refs/heads/main", + 1, + Some("`clone.submodules` has no effect with `clone.skip`"), + ), + ( + "engine: nixery.dev/x\nclone:\n skip: true\n submodules: false\n", + "refs/heads/main", + 1, + Some("`clone.submodules` has no effect with `clone.skip`"), + ), + ( + "engine: nixery.dev/x\nclone:\n skip: true\n tags: true\n", + "refs/heads/main", + 1, + Some("`clone.tags` has no effect with `clone.skip`"), + ), + ( + "engine: nixery.dev/x\nclone:\n skip: true\n", + "refs/heads/main", + 1, + None, + ), + ( + "engine: nixery.dev/x\nwhen:\n - event: push\n branch: '**'\n", + "refs/heads/feature/x", + 1, + None, + ), + (tag, "refs/tags/v1.0", 1, None), + (tag, "refs/heads/v1.0", 0, None), + (single_star, "refs/heads/feature/x", 1, None), + (single_star, "refs/heads/feature/x/y", 0, None), + ]; + cases.iter().for_each(|(yaml, reference, count, diag)| { + let compiled = compile( + &[raw("ci.yml", yaml)], + &push(reference), + &ChangedFiles::none(), + ); + assert_eq!(compiled.workflows.len(), *count, "{yaml}"); + let messages: Vec<&String> = compiled + .diagnostics + .errors + .iter() + .chain(compiled.diagnostics.warnings.iter()) + .collect(); + match diag { + Some(sub) => assert!( + messages.iter().any(|message| message.contains(sub)), + "{sub}: {messages:?}" + ), + None => assert!( + compiled.diagnostics.errors.is_empty() + && !messages + .iter() + .any(|message| message.contains("invalid configuration")), + "{yaml}: {messages:?}" + ), + } + }); + } +} diff --git a/knot2/crates/knot-xrpc/src/blocklist.rs b/knot2/crates/knot-xrpc/src/blocklist.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/blocklist.rs @@ -0,0 +1,118 @@ +use std::sync::Arc; + +use axum::body::Bytes; +use axum::extract::State; +use axum::response::Response; +use http::HeaderMap; +use serde::Deserialize; + +use knot_acl::{KnotAcl, can_admin_knot}; +use knot_cob::{CobHome, CobStore}; +use knot_cobs::{BlocklistChange, BlocklistCob, Grant, Removal}; +use knot_git::Repo; +use knot_index::Resolved; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::AccountDid; + +use crate::cob::grant_set_apply; +use crate::error::XrpcError; +use crate::{XrpcState, decode, ok_empty, run_blocking}; + +pub(crate) const BAN_ROUTE: &str = "/xrpc/sh.tangled.knot.ban"; +pub(crate) const UNBAN_ROUTE: &str = "/xrpc/sh.tangled.knot.unban"; + +#[derive(Deserialize)] +struct SubjectInput { + subject: AccountDid, +} + +pub(crate) async fn ban( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + body: Bytes, +) -> Result { + let actor = state.authenticate(&headers, &method).await?; + let acl = KnotAcl::new(&state.admins, state.admission, &state.index); + if !can_admin_knot(&acl, &actor).is_allowed() { + return Err(XrpcError::forbidden("only knot admin may ban accounts")); + } + + let SubjectInput { subject } = decode(&body)?; + if state.admins.contains(&subject) { + return Err(XrpcError::forbidden("admin cannot be banned")); + } + if matches!(state.index.is_blocked(&subject), Resolved::Ready(true)) { + return Ok(ok_empty()); + } + + let now = state.now(); + let grant = Grant { + subject, + added_by: actor, + created_at: now, + }; + let signer = state.secrets.signer(&state.knot_did)?; + let meta_path = state.meta_path.clone(); + let index = Arc::clone(&state.index); + let cob_locks = Arc::clone(&state.cob_locks); + let home = CobHome::from(&state.knot_did); + run_blocking(move || { + let _guard = cob_locks.meta(); + let meta = Repo::open(&meta_path)?; + grant_set_apply::( + &CobStore::new(&meta), + &home, + BlocklistChange::Add(grant), + &signer, + now, + true, + )?; + index.refresh_blocklist().map_err(XrpcError::from) + }) + .await?; + + Ok(ok_empty()) +} + +pub(crate) async fn unban( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + body: Bytes, +) -> Result { + let actor = state.authenticate(&headers, &method).await?; + let acl = KnotAcl::new(&state.admins, state.admission, &state.index); + if !can_admin_knot(&acl, &actor).is_allowed() { + return Err(XrpcError::forbidden("only knot admin may unban accounts")); + } + + let SubjectInput { subject } = decode(&body)?; + if matches!(state.index.is_blocked(&subject), Resolved::Ready(false)) { + return Ok(ok_empty()); + } + + let now = state.now(); + let removal = Removal { subject }; + let signer = state.secrets.signer(&state.knot_did)?; + let meta_path = state.meta_path.clone(); + let index = Arc::clone(&state.index); + let cob_locks = Arc::clone(&state.cob_locks); + let home = CobHome::from(&state.knot_did); + run_blocking(move || { + let _guard = cob_locks.meta(); + let meta = Repo::open(&meta_path)?; + grant_set_apply::( + &CobStore::new(&meta), + &home, + BlocklistChange::Remove(removal), + &signer, + now, + false, + )?; + index.refresh_blocklist().map_err(XrpcError::from) + }) + .await?; + + Ok(ok_empty()) +} diff --git a/knot2/crates/knot-xrpc/src/body.rs b/knot2/crates/knot-xrpc/src/body.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/body.rs @@ -0,0 +1,132 @@ +use serde::Deserialize; +use serde::de::{self, Deserializer}; + +use knot_types::{AtUri, RefName, RepoName}; +use url::Url; + +pub(crate) struct RepoAtUri(AtUri); + +impl RepoAtUri { + pub(crate) fn at_uri(&self) -> &AtUri { + &self.0 + } +} + +impl<'de> Deserialize<'de> for RepoAtUri { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + AtUri::new_owned(raw) + .map(RepoAtUri) + .map_err(|_| de::Error::custom("repo must be an at-uri")) + } +} + +pub(crate) struct RepoNameArg(RepoName); + +impl RepoNameArg { + pub(crate) fn as_str(&self) -> &str { + self.0.as_str() + } +} + +impl<'de> Deserialize<'de> for RepoNameArg { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + RepoName::new(raw) + .map(RepoNameArg) + .map_err(de::Error::custom) + } +} + +#[derive(Clone)] +pub(crate) struct SourceUrl(Url); + +impl SourceUrl { + pub(crate) fn parse(raw: &str) -> Result { + parse_source_url(raw) + } + + pub(crate) fn as_str(&self) -> &str { + self.0.as_str() + } + + pub(crate) fn as_url(&self) -> &Url { + &self.0 + } +} + +impl<'de> Deserialize<'de> for SourceUrl { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + parse_source_url(&raw).map_err(de::Error::custom) + } +} + +fn parse_source_url(raw: &str) -> Result { + let url = Url::parse(raw).map_err(|_| "source must be a valid url")?; + match matches!(url.scheme(), "http" | "https") && url.has_host() { + true => Ok(SourceUrl(url)), + false => Err("source must be an http or https url"), + } +} + +// A sourceless repo is a plain repo not a bad request necessarily, +// so empty string / missing field both have to make `None`. +pub(crate) fn optional_source_url<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + match Option::::deserialize(deserializer)?.as_deref() { + None | Some("") => Ok(None), + Some(raw) => parse_source_url(raw).map(Some).map_err(de::Error::custom), + } +} + +knot_types::text_newtype! { + pub(crate) struct Patch(String) => verbatim; + pub(crate) struct CommitMessage(String) => verbatim; + pub(crate) struct CommitBody(String) => verbatim; +} + +pub(crate) use crate::query::Revspec; + +pub(crate) struct ForkRef(RefName); + +impl ForkRef { + pub(crate) fn hidden_ref(&self, remote: &RemoteRef) -> Option { + RefName::new(format!("{}/{}", self.0.as_str(), remote.as_str())).ok() + } +} + +// Comes as a bare name, +// and `refs/hidden/` is the only place a fork staging ref is allowed to exist, +// so `Deserialize` prepends the prefix & `RefName::new` validates the whole string, +// therefore there's nothing to append after check. +impl<'de> Deserialize<'de> for ForkRef { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + RefName::new(format!("refs/hidden/{raw}")) + .map(ForkRef) + .map_err(|_| de::Error::custom("invalid fork ref")) + } +} + +pub(crate) struct RemoteRef(knot_types::BranchName); + +impl RemoteRef { + pub(crate) fn as_str(&self) -> &str { + self.0.as_str() + } + + pub(crate) fn head_ref(&self) -> RefName { + self.0.head_ref() + } +} + +impl<'de> Deserialize<'de> for RemoteRef { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + knot_types::BranchName::new(raw) + .map(RemoteRef) + .map_err(|_| de::Error::custom("invalid remote ref")) + } +} diff --git a/knot2/crates/knot-xrpc/src/branches.rs b/knot2/crates/knot-xrpc/src/branches.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/branches.rs @@ -0,0 +1,145 @@ +use std::sync::Arc; + +use axum::body::Bytes; +use axum::extract::State; +use axum::response::Response; +use http::HeaderMap; +use serde::Deserialize; + +use knot_events::GitRefUpdate; +use knot_git::{GitError, RefUpdate}; +use knot_index::Resolved; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{AtUri, BranchName, OwnerDid, RepoDid, RepoRkey}; + +use crate::body::RepoAtUri; +use crate::error::XrpcError; +use crate::{XrpcState, decode, ok_empty, run_blocking}; + +pub(crate) const SET_DEFAULT_ROUTE: &str = "/xrpc/sh.tangled.repo.setDefaultBranch"; +pub(crate) const DELETE_ROUTE: &str = "/xrpc/sh.tangled.repo.deleteBranch"; +const REPO_COLLECTION: &str = "sh.tangled.repo"; + +#[derive(Deserialize)] +struct SetDefaultBranchInput { + repo: RepoAtUri, + #[serde(rename = "defaultBranch")] + default_branch: BranchName, +} + +#[derive(Deserialize)] +struct DeleteBranchInput { + repo: RepoAtUri, + branch: BranchName, +} + +const BRANCH_DENIED: &str = "only repository owner or a collaborator may change its branches"; + +pub(crate) fn resolve_at_uri( + state: &XrpcState, + at: &AtUri, +) -> Result { + let owner = OwnerDid::new(at.authority().as_str()) + .map_err(|_| XrpcError::invalid_request("at-uri authority must be a DID"))?; + if at + .collection() + .is_none_or(|collection| collection.as_str() != REPO_COLLECTION) + { + return Err(XrpcError::invalid_request( + "at-uri must address an sh.tangled.repo record", + )); + } + let rkey = at + .rkey() + .ok_or_else(|| XrpcError::invalid_request("at-uri must include a record key"))?; + let rkey = RepoRkey::new(rkey.as_str()) + .map_err(|_| XrpcError::invalid_request("at-uri record key isn't a valid rkey"))?; + match state.index.resolve_repo(&owner, &rkey) { + Resolved::Ready(Some(repo_did)) => Ok(repo_did), + Resolved::Ready(None) => Err(XrpcError::not_found("no such repository on this knot")), + Resolved::Warming => Err(XrpcError::warming("registry projection is still warming")), + } +} + +pub(crate) async fn set_default_branch( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + body: Bytes, +) -> Result { + let actor = state.authenticate(&headers, &method).await?; + let input: SetDefaultBranchInput = decode(&body)?; + let repo_did = resolve_at_uri(&state, input.repo.at_uri())?; + crate::authorize_push(&state, &actor, &repo_did, BRANCH_DENIED).await?; + let refname = input.default_branch.head_ref(); + + let layout = state.layout.clone(); + let target = repo_did.clone(); + let events = Arc::clone(&state.events); + let reservation = run_blocking(move || { + let repo = layout.open(&target)?; + let target_exists = repo.find_ref(&refname)?.is_some(); + let has_branches = !repo.branches()?.is_empty(); + if !target_exists && has_branches { + return Err(XrpcError::not_found("no such branch to set as default")); + } + repo.set_head_sealed(&refname, || events.reserve()) + .map_err(XrpcError::from) + }) + .await?; + + let owner = crate::current_owner(&state, &repo_did); + reservation.fulfill(&GitRefUpdate::new(repo_did, owner, actor)); + + Ok(ok_empty()) +} + +pub(crate) async fn delete_branch( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + body: Bytes, +) -> Result { + let actor = state.authenticate(&headers, &method).await?; + let input: DeleteBranchInput = decode(&body)?; + let repo_did = resolve_at_uri(&state, input.repo.at_uri())?; + crate::authorize_push(&state, &actor, &repo_did, BRANCH_DENIED).await?; + let refname = input.branch.head_ref(); + + let layout = state.layout.clone(); + let target = repo_did.clone(); + let deleted_ref = refname.clone(); + let events = Arc::clone(&state.events); + let (old, reservation, format) = run_blocking(move || { + let repo = layout.open(&target)?; + if repo.default_branch().as_ref() == Some(&refname) { + return Err(XrpcError::invalid_request( + "default branch cannot be deleted until a different default is set", + )); + } + let old = repo + .find_ref(&refname)? + .ok_or_else(|| XrpcError::not_found("no such branch"))?; + let reservation = repo + .update_ref_sealed(&RefUpdate::Delete { name: refname, old }, || { + events.reserve() + }) + .map_err(|error| match error { + GitError::AtomicRefs(_) => { + XrpcError::conflict("branch changed during deletion, retry") + } + other => XrpcError::internal(other.to_string()), + })?; + Ok((old, reservation, repo.object_format())) + }) + .await?; + + let owner = crate::current_owner(&state, &repo_did); + reservation.fulfill(&GitRefUpdate::new(repo_did, owner, actor).on_ref( + deleted_ref, + knot_types::RefTransition::Delete { old }, + format, + )); + + Ok(ok_empty()) +} diff --git a/knot2/crates/knot-xrpc/src/cob.rs b/knot2/crates/knot-xrpc/src/cob.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/cob.rs @@ -0,0 +1,44 @@ +use knot_cob::{ChangePayload, Checkpoint, CobError, CobHome, CobStore, Evaluate}; +use knot_cobs::{GrantChange, Roster}; +use knot_runtime::Signer; +use knot_types::UnixSeconds; + +use crate::error::XrpcError; + +pub(crate) fn grant_set_apply( + store: &CobStore, + home: &CobHome, + change: E::Change, + signer: &dyn Signer, + now: UnixSeconds, + create_if_absent: bool, +) -> Result +where + E: Checkpoint + Evaluate, + E::Change: ChangePayload + Clone + GrantChange, +{ + match store.list::().map_err(XrpcError::from)?.as_slice() { + [] if create_if_absent => { + store + .create(home, &change, signer, now) + .map_err(XrpcError::from)?; + Ok(true) + } + [] => Ok(false), + [object] => store + .update_maybe_checkpointed::(home, *object, signer, now, |roster| { + let redundant = change.adds() == roster.contains(change.subject()); + Ok(if redundant { + None + } else { + Some(change.clone()) + }) + }) + .map(|change_id| change_id.is_some()) + .map_err(XrpcError::from), + many => Err(XrpcError::internal(format!( + "{} collaborative objects of one type share namespace", + many.len() + ))), + } +} diff --git a/knot2/crates/knot-xrpc/src/collaborators.rs b/knot2/crates/knot-xrpc/src/collaborators.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/collaborators.rs @@ -0,0 +1,177 @@ +use std::sync::Arc; + +use axum::body::Bytes; +use axum::extract::State; +use axum::response::Response; +use http::HeaderMap; +use serde::Deserialize; + +use knot_acl::{KnotAcl, can_manage_collaborators}; +use knot_cob::{CobHome, CobStore}; +use knot_cobs::{CollaboratorsChange, CollaboratorsCob, Grant, Removal}; +use knot_events::RepoCollaboratorUpdate; +use knot_index::Resolved; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{AccountDid, RepoDid}; + +use crate::cob::grant_set_apply; +use crate::error::XrpcError; +use crate::{XrpcState, decode, ok_empty, run_blocking}; + +pub(crate) const ADD_ROUTE: &str = "/xrpc/sh.tangled.repo.addCollaborator"; +pub(crate) const REMOVE_ROUTE: &str = "/xrpc/sh.tangled.repo.removeCollaborator"; + +#[derive(Deserialize)] +struct CollaboratorInput { + repo: RepoDid, + subject: AccountDid, +} + +fn require_owner( + state: &XrpcState, + actor: &AccountDid, + repo: &RepoDid, +) -> Result { + let owner = match state.index.owner_of(repo) { + Resolved::Ready(Some(owner)) => owner, + Resolved::Ready(None) => { + return Err(XrpcError::not_found( + "repository isn't registered on this knot", + )); + } + Resolved::Warming => { + return Err(XrpcError::warming("registry projection is still warming")); + } + }; + let acl = KnotAcl::new(&state.admins, state.admission, &state.index); + if can_manage_collaborators(&acl, actor, repo).is_allowed() { + Ok(owner) + } else { + Err(XrpcError::forbidden( + "only repository owner may manage collaborators", + )) + } +} + +pub(crate) async fn add_collaborator( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + body: Bytes, +) -> Result { + let actor = state.authenticate(&headers, &method).await?; + let CollaboratorInput { repo, subject } = decode(&body)?; + let owner = require_owner(&state, &actor, &repo)?; + + crate::fold_collaborators(&state, &repo).await; + if owner.is(&subject) + || matches!( + state.index.is_collaborator(&repo, &subject), + Resolved::Ready(true) + ) + { + return Ok(ok_empty()); + } + + let now = state.now(); + let event_subject = subject.clone(); + let event_repo = repo.clone(); + let grant = Grant { + subject, + added_by: actor, + created_at: now, + }; + let signer = state.secrets.signer(&state.knot_did).map_err(|error| { + XrpcError::internal(format!("knot signing key is unavailable: {error}")) + })?; + let layout = state.layout.clone(); + let index = Arc::clone(&state.index); + let cob_locks = Arc::clone(&state.cob_locks); + let events = Arc::clone(&state.events); + run_blocking(move || { + let _guard = cob_locks.repo(&repo); + owner_unmoved(&index, &repo, &owner)?; + let git = layout.open(&repo)?; + let changed = grant_set_apply::( + &CobStore::new(&git), + &CobHome::from(&repo), + CollaboratorsChange::Add(grant), + &signer, + now, + true, + )?; + index.refresh_collaborators(&repo)?; + if changed { + events.publish(&RepoCollaboratorUpdate::added(event_subject, event_repo)); + } + Ok(()) + }) + .await?; + + Ok(ok_empty()) +} + +fn owner_unmoved( + index: &knot_index::Index, + repo: &RepoDid, + owner: &knot_types::OwnerDid, +) -> Result<(), XrpcError> { + match index.owner_of(repo) { + Resolved::Ready(Some(current)) if current == *owner => Ok(()), + _ => Err(XrpcError::conflict( + "repository is no longer registered to owner who authorized this request", + )), + } +} + +pub(crate) async fn remove_collaborator( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + body: Bytes, +) -> Result { + let actor = state.authenticate(&headers, &method).await?; + let CollaboratorInput { repo, subject } = decode(&body)?; + let owner = require_owner(&state, &actor, &repo)?; + + crate::fold_collaborators(&state, &repo).await; + if matches!( + state.index.is_collaborator(&repo, &subject), + Resolved::Ready(false) + ) { + return Ok(ok_empty()); + } + + let now = state.now(); + let event_subject = subject.clone(); + let event_repo = repo.clone(); + let removal = Removal { subject }; + let signer = state.secrets.signer(&state.knot_did).map_err(|error| { + XrpcError::internal(format!("knot signing key is unavailable: {error}")) + })?; + let layout = state.layout.clone(); + let index = Arc::clone(&state.index); + let cob_locks = Arc::clone(&state.cob_locks); + let events = Arc::clone(&state.events); + run_blocking(move || { + let _guard = cob_locks.repo(&repo); + owner_unmoved(&index, &repo, &owner)?; + let git = layout.open(&repo)?; + let changed = grant_set_apply::( + &CobStore::new(&git), + &CobHome::from(&repo), + CollaboratorsChange::Remove(removal), + &signer, + now, + false, + )?; + index.refresh_collaborators(&repo)?; + if changed { + events.publish(&RepoCollaboratorUpdate::removed(event_subject, event_repo)); + } + Ok(()) + }) + .await?; + + Ok(ok_empty()) +} diff --git a/knot2/crates/knot-xrpc/src/error.rs b/knot2/crates/knot-xrpc/src/error.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/error.rs @@ -0,0 +1,395 @@ +use axum::Json; +use axum::response::{IntoResponse, Response}; +use http::StatusCode; +use serde_json::json; + +#[derive(Debug, Clone)] +pub struct XrpcError { + status: StatusCode, + error: &'static str, + message: String, +} + +impl XrpcError { + fn new(status: StatusCode, error: &'static str, message: impl Into) -> Self { + Self { + status, + error, + message: message.into(), + } + } + + pub fn invalid_request(message: impl Into) -> Self { + Self::new(StatusCode::BAD_REQUEST, "InvalidRequest", message) + } + + pub(crate) fn named( + status: StatusCode, + error: &'static str, + message: impl Into, + ) -> Self { + Self::new(status, error, message) + } + + pub fn auth_required(message: impl Into) -> Self { + Self::new(StatusCode::UNAUTHORIZED, "AuthenticationRequired", message) + } + + pub fn forbidden(message: impl Into) -> Self { + Self::new(StatusCode::FORBIDDEN, "Forbidden", message) + } + + pub fn not_found(message: impl Into) -> Self { + Self::new(StatusCode::NOT_FOUND, "NotFound", message) + } + + pub fn conflict(message: impl Into) -> Self { + Self::new(StatusCode::CONFLICT, "Conflict", message) + } + + pub fn request_too_large(message: impl Into) -> Self { + Self::new(StatusCode::PAYLOAD_TOO_LARGE, "RequestTooLarge", message) + } + + pub fn warming(message: impl Into) -> Self { + Self::new( + StatusCode::SERVICE_UNAVAILABLE, + "ProjectionWarming", + message, + ) + } + + pub fn upstream_unavailable(message: impl Into) -> Self { + Self::new( + StatusCode::SERVICE_UNAVAILABLE, + "UpstreamUnavailable", + message, + ) + } + + pub fn rate_limited(message: impl Into) -> Self { + Self::new(StatusCode::TOO_MANY_REQUESTS, "RateLimitExceeded", message) + } + + pub fn overloaded(message: impl Into) -> Self { + Self::new(StatusCode::SERVICE_UNAVAILABLE, "Overloaded", message) + } + + pub fn bad_gateway(message: impl Into) -> Self { + Self::new(StatusCode::BAD_GATEWAY, "UpstreamFailure", message) + } + + pub fn internal(message: impl Into) -> Self { + Self::new(StatusCode::INTERNAL_SERVER_ERROR, "InternalError", message) + } + + pub(crate) fn status(&self) -> StatusCode { + self.status + } + + pub(crate) fn from_status(status: StatusCode, message: impl Into) -> Self { + let error = match status { + StatusCode::BAD_REQUEST => "InvalidRequest", + StatusCode::UNAUTHORIZED => "AuthenticationRequired", + StatusCode::FORBIDDEN => "Forbidden", + StatusCode::NOT_FOUND => "NotFound", + StatusCode::CONFLICT => "Conflict", + StatusCode::PAYLOAD_TOO_LARGE => "RequestTooLarge", + StatusCode::UNSUPPORTED_MEDIA_TYPE => "UnsupportedMediaType", + StatusCode::TOO_MANY_REQUESTS => "RateLimitExceeded", + StatusCode::SERVICE_UNAVAILABLE => "Overloaded", + StatusCode::BAD_GATEWAY => "UpstreamFailure", + _ => return Self::internal(message), + }; + Self::new(status, error, message) + } +} + +impl std::fmt::Display for XrpcError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}: {}", self.error, self.message) + } +} + +impl From for XrpcError { + fn from(error: knot_git::GitError) -> Self { + use knot_git::GitError; + let message = error.to_string(); + match error { + GitError::AlreadyExists(_) => Self::conflict(message), + GitError::AtomicRefs(_) => Self::conflict(message), + GitError::UnsafeRepoDid(_) | GitError::ReservedDid(_) => Self::invalid_request(message), + GitError::DepthExceeded(_) => Self::invalid_request(message), + GitError::Selection(_) => Self::overloaded(message), + // Every oid passed to the object database here came from a ref this + // knot already resolved or a tree it already read, so a miss means + // the repository is missing an object it references. A handler that + // looks up something the caller named reports its own named error + // before it ever gets an oid. + GitError::ObjectNotFound(_) + | GitError::Open { .. } + | GitError::Create { .. } + | GitError::Remove { .. } + | GitError::Reference { .. } + | GitError::Fsync { .. } + | GitError::Write { .. } + | GitError::RevWalk(_) + | GitError::RemoveObject { .. } + | GitError::Corrupt { .. } + | GitError::ObjectType { .. } + | GitError::Decode(_) + | GitError::Backend(_) + | GitError::Staging(_) + | GitError::Config { .. } + | GitError::Maintenance(_) => Self::internal(message), + } + } +} + +impl From for XrpcError { + fn from(error: knot_git::ApplyError) -> Self { + use knot_git::ApplyError; + match error { + ApplyError::TooLarge => Self::request_too_large(error.to_string()), + ApplyError::Git(inner) => inner.into(), + } + } +} + +impl From for XrpcError { + fn from(error: knot_cob::CobError) -> Self { + use knot_cob::CobError; + match error { + CobError::Contended(_) | CobError::StaleTip { .. } => Self::conflict(error.to_string()), + other => Self::internal(other.to_string()), + } + } +} + +impl From for XrpcError { + fn from(error: knot_index::IndexError) -> Self { + use knot_index::IndexError; + match error { + IndexError::Git(git) => git.into(), + IndexError::Cob(cob) => cob.into(), + other => Self::internal(other.to_string()), + } + } +} + +impl From for XrpcError { + fn from(error: knot_secrets::SecretsError) -> Self { + use knot_secrets::SecretsError; + match error { + SecretsError::Occupied(_) => Self::conflict(error.to_string()), + other => Self::internal(other.to_string()), + } + } +} + +impl From for XrpcError { + fn from(error: knot_cobs::RegistryError) -> Self { + use knot_cobs::RegistryError; + match error { + RegistryError::AlreadyRegistered { .. } + | RegistryError::RepoMismatch { .. } + | RegistryError::RkeyTaken { .. } + | RegistryError::OwnerMoved { .. } => Self::conflict(error.to_string()), + RegistryError::NotRegistered { .. } | RegistryError::NotHosted { .. } => { + Self::not_found(error.to_string()) + } + RegistryError::Cob(cob) => cob.into(), + } + } +} + +impl From for XrpcError { + fn from(error: knot_atproto::AtprotoError) -> Self { + use knot_atproto::AtprotoError; + if error.is_transient() { + return Self::upstream_unavailable(error.to_string()); + } + match error { + AtprotoError::Resolve(_) => Self::invalid_request(error.to_string()), + AtprotoError::PlcSubmit { .. } => Self::bad_gateway(error.to_string()), + other => Self::internal(other.to_string()), + } + } +} + +impl From for XrpcError { + fn from(error: knot_atproto::IdentityError) -> Self { + Self::internal(error.to_string()) + } +} + +impl From for XrpcError { + fn from(error: knot_pack::PackError) -> Self { + Self::from_status(error.http_status(), error.to_string()) + } +} + +impl From for XrpcError { + fn from(error: knot_pack::FetchError) -> Self { + use knot_pack::FetchError; + let message = error.to_string(); + match error { + FetchError::Url(_) => Self::invalid_request(message), + FetchError::Network(_) => Self::upstream_unavailable(message), + FetchError::Status(_) | FetchError::Protocol(_) | FetchError::Remote(_) => { + Self::bad_gateway(message) + } + FetchError::PackTooLarge { .. } => Self::request_too_large(message), + FetchError::Pack(pack) => pack.into(), + } + } +} + +impl IntoResponse for XrpcError { + fn into_response(self) -> Response { + match self.status { + StatusCode::FORBIDDEN | StatusCode::TOO_MANY_REQUESTS => tracing::warn!( + status = self.status.as_u16(), + error = self.error, + message = %self.message, + "request rejected" + ), + StatusCode::UNAUTHORIZED => tracing::debug!( + error = self.error, + message = %self.message, + "request unauthenticated" + ), + _ => {} + } + ( + self.status, + Json(json!({ "error": self.error, "message": self.message })), + ) + .into_response() + } +} + +#[cfg(test)] +mod tests { + use super::XrpcError; + use axum::response::IntoResponse; + use http::StatusCode; + + #[test] + fn each_tag_maps_to_its_status_in_the_class_and_through_into_response() { + let cases: &[(XrpcError, StatusCode, &str)] = &[ + ( + XrpcError::invalid_request("x"), + StatusCode::BAD_REQUEST, + "InvalidRequest", + ), + ( + XrpcError::auth_required("x"), + StatusCode::UNAUTHORIZED, + "AuthenticationRequired", + ), + ( + XrpcError::forbidden("x"), + StatusCode::FORBIDDEN, + "Forbidden", + ), + (XrpcError::not_found("x"), StatusCode::NOT_FOUND, "NotFound"), + (XrpcError::conflict("x"), StatusCode::CONFLICT, "Conflict"), + ( + XrpcError::request_too_large("x"), + StatusCode::PAYLOAD_TOO_LARGE, + "RequestTooLarge", + ), + ( + XrpcError::rate_limited("x"), + StatusCode::TOO_MANY_REQUESTS, + "RateLimitExceeded", + ), + ( + XrpcError::warming("x"), + StatusCode::SERVICE_UNAVAILABLE, + "ProjectionWarming", + ), + ( + XrpcError::upstream_unavailable("x"), + StatusCode::SERVICE_UNAVAILABLE, + "UpstreamUnavailable", + ), + ( + XrpcError::overloaded("x"), + StatusCode::SERVICE_UNAVAILABLE, + "Overloaded", + ), + ( + XrpcError::bad_gateway("x"), + StatusCode::BAD_GATEWAY, + "UpstreamFailure", + ), + ( + XrpcError::internal("x"), + StatusCode::INTERNAL_SERVER_ERROR, + "InternalError", + ), + ]; + cases.iter().for_each(|(error, status, tag)| { + assert_eq!((error.status, error.error), (*status, *tag)); + assert_eq!( + error.clone().into_response().status(), + *status, + "into_response serves the mapped status for {tag}" + ); + }); + } + + #[test] + fn a_domain_error_maps_to_the_status_that_names_whose_fault_it_is() { + let oid = knot_types::Oid::from_hex(&"a".repeat(40)).unwrap(); + let cases: Vec<(XrpcError, StatusCode, &str)> = vec![ + ( + knot_git::GitError::ObjectNotFound(oid).into(), + StatusCode::INTERNAL_SERVER_ERROR, + "these oids come from refs and trees this knot resolved itself, \ + and every read lexicon names its own not-found error for what the caller asked for", + ), + ( + knot_git::GitError::Corrupt { + oid, + message: "truncated".to_string(), + } + .into(), + StatusCode::INTERNAL_SERVER_ERROR, + "a corrupt repository on this knot isn't the caller's fault to fix", + ), + ( + knot_git::GitError::AtomicRefs("lost".to_string()).into(), + StatusCode::CONFLICT, + "losing a ref race is a conflict", + ), + ( + knot_git::GitError::AlreadyExists("/scallop".into()).into(), + StatusCode::CONFLICT, + "creating a repository that exists is a conflict", + ), + ( + knot_pack::FetchError::Remote("gone".to_string()).into(), + StatusCode::BAD_GATEWAY, + "a fetch failure at the upstream during fork sync is the upstream's fault", + ), + ( + knot_git::ApplyError::Git(knot_git::GitError::AtomicRefs("lost".to_string())) + .into(), + StatusCode::CONFLICT, + "wrapping a git error in ApplyError mustn't downgrade it to a generic fault", + ), + ( + knot_index::IndexError::Git(knot_git::GitError::AlreadyExists("/whelk".into())) + .into(), + StatusCode::CONFLICT, + "wrapping a git error in IndexError mustn't downgrade it either", + ), + ]; + cases.iter().for_each(|(error, status, why)| { + assert_eq!(error.status(), *status, "{why}"); + }); + } +} diff --git a/knot2/crates/knot-xrpc/src/events.rs b/knot2/crates/knot-xrpc/src/events.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/events.rs @@ -0,0 +1,158 @@ +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; + +use axum::extract::ws::{CloseFrame, Message, WebSocket, WebSocketUpgrade}; +use axum::extract::{ConnectInfo, Query, State}; +use axum::response::{IntoResponse, Response}; +use futures::stream::SplitSink; +use futures::{SinkExt, StreamExt}; +use http::HeaderMap; +use serde::Deserialize; + +use knot_events::{ + BatchEnd, EventCursor, EventLog, ReplayBounds, ReplayBytes, ReplayEvents, Replayed, +}; +use knot_runtime::{Clock, HttpTransport}; + +use crate::XrpcState; +use crate::error::XrpcError; + +pub(crate) const EVENTS_ROUTE: &str = "/events"; + +const DRAIN_BATCH: usize = 100; +const DRAIN_BYTES: usize = 4 << 20; +const MAX_BATCHES_PER_DRAIN: usize = 1_000; +const KEEPALIVE: Duration = Duration::from_secs(30); +const WRITE_DEADLINE: Duration = Duration::from_secs(10); +const TRY_AGAIN_LATER: u16 = 1013; + +#[derive(Deserialize)] +pub(crate) struct EventsQuery { + cursor: Option, +} + +pub(crate) async fn events( + State(state): State>>, + ConnectInfo(socket_peer): ConnectInfo, + headers: HeaderMap, + Query(query): Query, + upgrade: WebSocketUpgrade, +) -> Response { + let peer = state + .trusted_proxy_header + .as_ref() + .and_then(|header| knot_types::forwarded_peer(&headers, header)) + .unwrap_or_else(|| socket_peer.ip()); + let Some(permit) = state.subscriber_gate.try_admit(peer) else { + return XrpcError::overloaded( + "knot is serving its maximum number of event subscribers, retry shortly", + ) + .into_response(); + }; + let cursor = query + .cursor + .as_deref() + .and_then(|raw| raw.parse::().ok()) + .map(EventCursor::new) + .unwrap_or(EventCursor::START); + let log = Arc::clone(&state.events); + upgrade.on_upgrade(move |socket| async move { + let _permit = permit; + stream_events(socket, log, cursor).await; + }) +} + +enum Drained { + CaughtUp, + Limited, +} + +async fn stream_events(socket: WebSocket, log: Arc>, start: EventCursor) { + let mut head = log.subscribe(); + let (mut sink, mut from_client) = socket.split(); + let mut keepalive = + tokio::time::interval_at(tokio::time::Instant::now() + KEEPALIVE, KEEPALIVE); + let mut cursor = start; + loop { + match drain(&mut sink, &log, &mut cursor).await { + Ok(Drained::CaughtUp) => {} + Ok(Drained::Limited) => { + let close = Message::Close(Some(CloseFrame { + code: TRY_AGAIN_LATER, + reason: "drain limit reached, reconnect to continue".into(), + })); + let _ = tokio::time::timeout(WRITE_DEADLINE, sink.send(close)).await; + return; + } + Err(()) => return, + } + tokio::select! { + changed = head.changed() => { + if changed.is_err() { + return; + } + } + _ = keepalive.tick() => { + let ping = sink.send(Message::Ping(Vec::new().into())); + if !matches!(tokio::time::timeout(WRITE_DEADLINE, ping).await, Ok(Ok(()))) { + return; + } + } + received = from_client.next() => { + match received { + None | Some(Err(_)) | Some(Ok(Message::Close(_))) => return, + Some(Ok(_)) => {} + } + } + } + } +} + +fn drain_bounds() -> ReplayBounds { + ReplayBounds::new( + ReplayEvents::new(DRAIN_BATCH).expect("drain event maximum is nonzero"), + ReplayBytes::new(DRAIN_BYTES).expect("drain byte maximum is nonzero"), + ) +} + +// who up draining they clock +async fn drain( + sink: &mut SplitSink, + log: &EventLog, + cursor: &mut EventCursor, +) -> Result { + let mut batches = 0; + loop { + let Replayed { events, end } = log.replay(*cursor, drain_bounds()); + if let Some(last) = events.last() { + *cursor = last.created; + } + let messages: Vec> = events + .iter() + .map(|event| { + Ok(Message::Text( + serde_json::to_string(event.as_ref()) + .expect("wire event serializes to JSON") + .into(), + )) + }) + .collect(); + drop(events); + let sent = tokio::time::timeout( + WRITE_DEADLINE, + sink.send_all(&mut futures::stream::iter(messages)), + ) + .await; + if !matches!(sent, Ok(Ok(()))) { + return Err(()); + } + if end == BatchEnd::CaughtUp { + return Ok(Drained::CaughtUp); + } + batches += 1; + if batches == MAX_BATCHES_PER_DRAIN { + return Ok(Drained::Limited); + } + } +} diff --git a/knot2/crates/knot-xrpc/src/forks.rs b/knot2/crates/knot-xrpc/src/forks.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/forks.rs @@ -0,0 +1,628 @@ +use std::sync::Arc; + +use axum::Json; +use axum::body::Bytes; +use axum::extract::State; +use axum::response::{IntoResponse, Response}; +use http::HeaderMap; +use serde::{Deserialize, Serialize}; +use url::Url; + +use knot_events::Reservation; +use knot_git::{Filter, GitError, Haves, RefUpdate, Repo, Staging, Wants}; +use knot_index::Resolved; +use knot_pack::{FetchError, HaveOids, PackLimits, UpstreamRefs, WantOids}; +use knot_postreceive::{Actor, Ci}; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{BranchName, Oid, OwnerDid, RefName, RepoDid}; + +use crate::body::{ForkRef, RemoteRef, RepoAtUri, RepoNameArg, Revspec, SourceUrl}; +use crate::branches::resolve_at_uri; +use crate::error::XrpcError; +use crate::{XrpcState, decode, ok_empty, run_blocking}; + +pub(crate) const STATUS_ROUTE: &str = "/xrpc/sh.tangled.repo.forkStatus"; +pub(crate) const SYNC_ROUTE: &str = "/xrpc/sh.tangled.repo.forkSync"; +pub(crate) const HIDDEN_REF_ROUTE: &str = "/xrpc/sh.tangled.repo.hiddenRef"; + +#[derive(Clone)] +pub(crate) enum Upstream { + Local(RepoDid), + Remote(Url), +} + +fn url_authority(url: &Url) -> String { + let host = url.host_str().unwrap_or_default(); + match url.port() { + Some(port) => format!("{host}:{port}"), + None => host.to_string(), + } +} + +fn resolve_local_path( + state: &XrpcState, + url: &Url, +) -> Result { + let segments: Vec<&str> = url + .path_segments() + .map(|segments| segments.filter(|segment| !segment.is_empty()).collect()) + .unwrap_or_default(); + match segments.as_slice() { + [did] => { + let did = RepoDid::new(*did) + .map_err(|_| XrpcError::invalid_request("fork source path isn't a DID"))?; + match state.index.owner_of(&did) { + Resolved::Ready(Some(_)) => Ok(did), + Resolved::Ready(None) => Err(XrpcError::not_found( + "fork source isn't hosted on this knot", + )), + Resolved::Warming => { + Err(XrpcError::warming("registry projection is still warming")) + } + } + } + [owner, name] => { + let owner = OwnerDid::new(*owner) + .map_err(|_| XrpcError::invalid_request("fork source owner segment isn't a DID"))?; + let name = name.strip_suffix(".git").unwrap_or(name); + crate::merge::resolve_by_name(state, &owner, name) + } + _ => Err(XrpcError::invalid_request( + "fork source path must be /did or /owner/name", + )), + } +} + +pub(crate) fn resolve_upstream( + state: &XrpcState, + source: &SourceUrl, +) -> Result { + let url = source.as_url(); + if url_authority(url) == state.knot_authority() { + return resolve_local_path(state, url).map(Upstream::Local); + } + Ok(Upstream::Remote(url.clone())) +} + +fn map_fetch(error: FetchError) -> XrpcError { + match error { + FetchError::PackTooLarge { limit } => XrpcError::request_too_large(format!( + "upstream pack exceeds this knot's fork limit of {limit} bytes" + )), + FetchError::Pack(inner) => { + XrpcError::bad_gateway(format!("upstream sent an unusable pack: {inner}")) + } + other => XrpcError::bad_gateway(other.to_string()), + } +} + +pub(crate) async fn upstream_refs( + state: &XrpcState, + upstream: &Upstream, + prefixes: Vec, +) -> Result { + match upstream { + Upstream::Local(did) => { + let layout = state.layout.clone(); + let did = did.clone(); + run_blocking(move || { + let repo = layout.open(&did)?; + let prefixes: Vec<&str> = prefixes.iter().map(String::as_str).collect(); + knot_pack::local_refs(&repo, &prefixes).map_err(XrpcError::from) + }) + .await + } + Upstream::Remote(url) => { + let prefixes: Vec<&str> = prefixes.iter().map(String::as_str).collect(); + knot_pack::remote_refs(state.git_http.as_ref(), url, &prefixes) + .await + .map_err(map_fetch) + } + } +} + +pub(crate) async fn upstream_pack( + state: &XrpcState, + upstream: &Upstream, + wants: WantOids, + haves: HaveOids, +) -> Result, XrpcError> { + let byte_limit = state.byte_limits.fork_pack.get(); + match upstream { + Upstream::Local(did) => { + let layout = state.layout.clone(); + let did = did.clone(); + run_blocking(move || { + let repo = layout.open(&did)?; + knot_pack::local_pack(&repo, &wants, &haves, byte_limit).map_err( + |error| match error { + FetchError::PackTooLarge { limit } => { + XrpcError::request_too_large(format!( + "fork source pack exceeds this knot's fork limit of {limit} bytes" + )) + } + other => XrpcError::internal(other.to_string()), + }, + ) + }) + .await + } + Upstream::Remote(url) => { + knot_pack::remote_pack(state.git_http.as_ref(), url, &wants, &haves, byte_limit) + .await + .map_err(map_fetch) + } + } +} + +fn connected(repo: &Repo, wants: Wants<'_>, haves: Haves<'_>) -> Result { + repo.select_pack_objects_filtered(wants, haves, Filter::None, knot_pack::selection_budget()) + .map(|selection| selection.send.iter().all(|oid| repo.contains(*oid))) + .map_err(XrpcError::from) +} + +pub(crate) fn populate_fork( + repo: &Repo, + refs: &UpstreamRefs, + pack: &[u8], + origin: &SourceUrl, +) -> Result<(), XrpcError> { + knot_pack::ingest_pack( + &repo.objects_dir(), + pack, + &PackLimits::default(), + repo.object_format().kind(), + ) + .map_err(|error| XrpcError::bad_gateway(format!("fork source pack is unusable: {error}")))?; + if !connected(repo, Wants::new(&refs.tips()), Haves::new(&[]))? { + return Err(XrpcError::bad_gateway( + "fork source sent an incomplete pack", + )); + } + let creates: Vec = refs + .refs + .iter() + .map(|record| RefUpdate::Create { + name: record.name.clone(), + new: record.target, + }) + .collect(); + if !creates.is_empty() { + repo.update_refs(&creates)?; + } + if let Some(head) = refs + .head_symref + .as_ref() + .filter(|head| head.as_str().starts_with("refs/heads/")) + { + repo.set_head(head)?; + } + repo.set_origin_url(origin.as_str()) + .map_err(XrpcError::from) +} + +fn pull_into_live(live: &Repo, pack: &[u8], tip: Oid, haves: &[Oid]) -> Result<(), XrpcError> { + if live.contains(tip) { + return Ok(()); + } + let staging = Staging::new(live)?; + knot_pack::ingest_pack( + &staging.repo().objects_dir(), + pack, + &PackLimits::default(), + live.object_format().kind(), + ) + .map_err(|error| XrpcError::bad_gateway(format!("upstream pack is unusable: {error}")))?; + if !connected(staging.repo(), Wants::new(&[tip]), Haves::new(haves))? { + return Err(XrpcError::bad_gateway("upstream sent an incomplete pack")); + } + staging.migrate_into(live).map_err(XrpcError::from) +} + +const FORCE_REF_ATTEMPTS: usize = 16; + +enum ForceStep { + Done(Option), + Retry, +} + +fn force_ref( + repo: &Repo, + name: &RefName, + new: Oid, + reserve: &dyn Fn() -> Reservation, +) -> Result, XrpcError> { + fn attempt( + repo: &Repo, + name: &RefName, + new: Oid, + reserve: &dyn Fn() -> Reservation, + ) -> Result { + let current = repo.find_ref(name)?; + let update = match current { + Some(old) if old == new => return Ok(ForceStep::Done(None)), + Some(old) => RefUpdate::Update { + name: name.clone(), + old, + new, + }, + None => RefUpdate::Create { + name: name.clone(), + new, + }, + }; + match repo.update_ref_sealed(&update, reserve) { + Ok(reservation) => Ok(ForceStep::Done(Some(reservation))), + Err(GitError::Reference { .. } | GitError::AtomicRefs(_)) => Ok(ForceStep::Retry), + Err(other) => Err(XrpcError::internal(other.to_string())), + } + } + + (0..FORCE_REF_ATTEMPTS) + .find_map(|_| match attempt(repo, name, new, reserve) { + Ok(ForceStep::Done(reservation)) => Some(Ok(reservation)), + Ok(ForceStep::Retry) => None, + Err(error) => Some(Err(error)), + }) + .unwrap_or_else(|| Err(XrpcError::conflict("ref moved during pull, retry"))) +} + +const FORK_DENIED: &str = "only repository owner or a collaborator may operate on this fork"; + +struct ForkState { + origin: SourceUrl, + haves: Vec, +} + +fn load_fork_state(repo: &Repo) -> Result { + let origin = repo.origin_url().ok_or_else(|| { + XrpcError::invalid_request("this repository isn't a fork and has no upstream") + })?; + let origin = SourceUrl::parse(&origin) + .map_err(|reason| XrpcError::internal(format!("stored fork origin: {reason}")))?; + let haves = repo + .references()? + .into_iter() + .map(|record| record.target) + .collect(); + Ok(ForkState { origin, haves }) +} + +pub(crate) struct SyncResult { + old: Option, + new: Oid, + reservation: Option, + lfs_missing: Vec, +} + +#[derive(Clone, Copy)] +struct SourceRef<'a>(&'a RefName); + +impl<'a> SourceRef<'a> { + fn new(reference: &'a RefName) -> Self { + Self(reference) + } + + fn get(self) -> &'a RefName { + self.0 + } +} + +#[derive(Clone, Copy)] +struct TargetRef<'a>(&'a RefName); + +impl<'a> TargetRef<'a> { + fn new(reference: &'a RefName) -> Self { + Self(reference) + } + + fn get(self) -> &'a RefName { + self.0 + } +} + +async fn pull_upstream_branch( + state: &Arc>, + repo_did: &RepoDid, + source: SourceRef<'_>, + target: TargetRef<'_>, +) -> Result { + let branch = source.get(); + let target = target.get(); + let layout = state.layout.clone(); + let opened = repo_did.clone(); + let fork = run_blocking(move || { + let repo = layout.open(&opened)?; + load_fork_state(&repo) + }) + .await?; + + let upstream = resolve_upstream(state, &fork.origin)?; + let refs = upstream_refs(state, &upstream, vec![branch.as_str().to_string()]).await?; + let tip = refs + .find(branch) + .ok_or_else(|| XrpcError::not_found("upstream repository doesn't have that branch"))?; + let pack = upstream_pack( + state, + &upstream, + WantOids::new(vec![tip]), + HaveOids::new(fork.haves.clone()), + ) + .await?; + + let layout = state.layout.clone(); + let opened = repo_did.clone(); + let haves = fork.haves.clone(); + run_blocking(move || { + let repo = layout.open(&opened)?; + pull_into_live(&repo, &pack, tip, &fork.haves) + }) + .await?; + + let lfs_missing = crate::lfs::mirror_fork_objects( + Arc::clone(state), + upstream, + repo_did.clone(), + WantOids::new(vec![tip]), + HaveOids::new(haves), + ) + .await?; + + let layout = state.layout.clone(); + let opened = repo_did.clone(); + let target = target.clone(); + let events = Arc::clone(&state.events); + run_blocking(move || { + let repo = layout.open(&opened)?; + let old = repo.find_ref(&target)?; + let reservation = force_ref(&repo, &target, tip, &|| events.reserve())?; + Ok(SyncResult { + old, + new: tip, + reservation, + lfs_missing, + }) + }) + .await +} + +#[derive(Deserialize)] +struct ForkSyncInput { + did: OwnerDid, + name: RepoNameArg, + branch: BranchName, +} + +pub(crate) async fn fork_sync( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + body: Bytes, +) -> Result { + let actor = state.authenticate(&headers, &method).await?; + let input: ForkSyncInput = decode(&body)?; + let repo_did = crate::merge::resolve_by_name(&state, &input.did, input.name.as_str())?; + crate::authorize_push(&state, &actor, &repo_did, FORK_DENIED).await?; + let branch = input.branch.head_ref(); + let sync = pull_upstream_branch( + &state, + &repo_did, + SourceRef::new(&branch), + TargetRef::new(&branch), + ) + .await?; + if let Some(reservation) = sync.reservation { + let owner = crate::current_owner(&state, &repo_did); + let layout = state.layout.clone(); + let languages_push_budget = state.budgets.languages_push; + let catalog = Arc::clone(&state.catalog); + let event_repo = repo_did.clone(); + let (old, new) = (sync.old, sync.new); + if let Err(error) = run_blocking(move || -> Result<(), XrpcError> { + let repo = layout.open(&event_repo)?; + let update = match old { + Some(old) => RefUpdate::Update { + name: branch, + old, + new, + }, + None => RefUpdate::Create { name: branch, new }, + }; + let post_actor = Actor { + committer: actor, + owner, + repo: event_repo, + }; + knot_postreceive::post_receive( + &repo, + &post_actor, + vec![(update, reservation)], + &Ci::Skip, + &knot_types::PushOptions::default(), + None, + languages_push_budget, + &catalog.push, + ); + Ok(()) + }) + .await + { + tracing::warn!(repo = repo_did.as_str(), %error, "post-receive after fork sync failed"); + } + } + match sync.lfs_missing.is_empty() { + true => Ok(ok_empty()), + false => Ok(( + http::StatusCode::OK, + Json(serde_json::json!({ + "lfsMissing": sync.lfs_missing.iter().map(|oid| oid.as_str()).collect::>(), + })), + ) + .into_response()), + } +} + +#[derive(Deserialize)] +struct HiddenRefInput { + repo: RepoAtUri, + #[serde(rename = "forkRef")] + fork_ref: ForkRef, + #[serde(rename = "remoteRef")] + remote_ref: RemoteRef, +} + +#[derive(Serialize)] +struct HiddenRefOutput { + success: bool, + #[serde(rename = "ref")] + ref_name: RefName, + #[serde(rename = "lfsMissing", skip_serializing_if = "Vec::is_empty")] + lfs_missing: Vec, +} + +pub(crate) async fn hidden_ref( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + body: Bytes, +) -> Result { + let actor = state.authenticate(&headers, &method).await?; + let input: HiddenRefInput = decode(&body)?; + let repo_did = resolve_at_uri(&state, input.repo.at_uri())?; + crate::authorize_push(&state, &actor, &repo_did, FORK_DENIED).await?; + let branch = input.remote_ref.head_ref(); + let target = input + .fork_ref + .hidden_ref(&input.remote_ref) + .ok_or_else(|| { + XrpcError::invalid_request("forkRef and remoteRef don't form a valid ref") + })?; + let sync = pull_upstream_branch( + &state, + &repo_did, + SourceRef::new(&branch), + TargetRef::new(&target), + ) + .await?; + Ok(( + http::StatusCode::OK, + Json(HiddenRefOutput { + success: true, + ref_name: target, + lfs_missing: sync.lfs_missing, + }), + ) + .into_response()) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ForkStatus { + UpToDate, + FastForwardable, + Conflict, +} + +impl ForkStatus { + fn code(self) -> u8 { + match self { + ForkStatus::UpToDate => 0, + ForkStatus::FastForwardable => 1, + ForkStatus::Conflict => 2, + } + } +} + +#[derive(Deserialize)] +struct ForkStatusInput { + did: OwnerDid, + name: Option, + #[serde(default, deserialize_with = "crate::body::optional_source_url")] + source: Option, + branch: Revspec, + #[serde(rename = "hiddenRef")] + hidden_ref: Revspec, +} + +#[derive(Serialize)] +struct ForkStatusOutput { + status: u8, +} + +fn source_basename(source: &Url) -> Option { + source + .path_segments() + .and_then(|mut segments| segments.rfind(|segment| !segment.is_empty())) + .map(str::to_string) +} + +pub(crate) async fn fork_status( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + body: Bytes, +) -> Result { + let actor = state.authenticate(&headers, &method).await?; + let input: ForkStatusInput = decode(&body)?; + let name = input + .name + .as_ref() + .map(|name| name.as_str().to_string()) + .or_else(|| { + input + .source + .as_ref() + .and_then(|source| source_basename(source.as_url())) + }) + .ok_or_else(|| { + XrpcError::invalid_request("neither name nor a source url with path was supplied") + })?; + let repo_did = crate::merge::resolve_by_name(&state, &input.did, &name)?; + crate::authorize_push(&state, &actor, &repo_did, FORK_DENIED).await?; + + let layout = state.layout.clone(); + let status = run_blocking(move || { + let repo = layout.open(&repo_did)?; + let fork = repo + .resolve_revision(input.branch.as_str()) + .ok_or_else(|| { + XrpcError::invalid_request(format!( + "cannot resolve revision {}", + input.branch.as_str() + )) + }) + .and_then(|oid| { + repo.peel_to_commit(oid) + .map_err(|error| XrpcError::invalid_request(error.to_string())) + })?; + let source = repo + .resolve_revision(input.hidden_ref.as_str()) + .ok_or_else(|| { + XrpcError::invalid_request(format!( + "cannot resolve revision {}", + input.hidden_ref.as_str() + )) + }) + .and_then(|oid| { + repo.peel_to_commit(oid) + .map_err(|error| XrpcError::invalid_request(error.to_string())) + })?; + if fork == source { + return Ok(ForkStatus::UpToDate); + } + let base = repo.merge_base(fork, source)?; + Ok(match base { + Some(base) if base == fork => ForkStatus::FastForwardable, + Some(base) if base == source => ForkStatus::UpToDate, + _ => ForkStatus::Conflict, + }) + }) + .await?; + + Ok(( + http::StatusCode::OK, + Json(ForkStatusOutput { + status: status.code(), + }), + ) + .into_response()) +} diff --git a/knot2/crates/knot-xrpc/src/lfs.rs b/knot2/crates/knot-xrpc/src/lfs.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/lfs.rs @@ -0,0 +1,1168 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use axum::Router; +use axum::body::{Body, Bytes}; +use axum::extract::rejection::BytesRejection; +use axum::extract::{DefaultBodyLimit, Path, Request, State}; +use axum::http::{HeaderMap, HeaderValue, StatusCode, header}; +use axum::response::{IntoResponse, Response}; +use axum::routing::{get, post}; +use knot_lfs::{ + BATCH_MEDIA_TYPE, BatchAction, BatchActions, BatchObject, BatchObjectError, BatchOperation, + BatchRequest, BatchResponse, BatchResponseObject, ClaimedSize, LfsError, LfsHandle, LfsOid, + LfsSize, LfsStore, MAX_BATCH_OBJECTS, UploadAdmission, +}; +use knot_pack::{HaveOids, SocketPeer, WantOids}; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{AccountDid, HttpStatus, KnotServiceUrl, RepoDid}; +use serde_json::json; +use tokio::sync::{OwnedSemaphorePermit, Semaphore, watch}; +use tower::ServiceExt; +use tower_http::services::ServeFile; +use url::Url; + +use crate::XrpcState; +use crate::forks::Upstream; + +pub const MAX_BATCH_BYTES: usize = 1024 * 1024; + +const IMMUTABLE_CACHE: &str = "public, max-age=31536000, immutable"; + +const READINESS_PROBE_TIMEOUT: Duration = Duration::from_secs(5); + +struct Readiness { + result: watch::Sender>, + probing: Mutex, +} + +pub struct LfsWeb { + pub handle: LfsHandle, + downloads: Arc, + readiness: Arc, +} + +impl LfsWeb { + pub fn new(handle: LfsHandle, max_downloads: usize) -> Self { + let (result, _rx) = watch::channel(None); + Self { + handle, + downloads: Arc::new(Semaphore::new(max_downloads)), + readiness: Arc::new(Readiness { + result, + probing: Mutex::new(false), + }), + } + } + + pub async fn ready(&self) -> bool { + let mut rx = self.readiness.result.subscribe(); + let launch = { + let mut probing = self.readiness.probing.lock().unwrap(); + match *probing { + false => { + *probing = true; + self.readiness.result.send_replace(None); + true + } + true => false, + } + }; + if launch { + Self::spawn_probe(Arc::clone(&self.readiness), Arc::clone(&self.handle.store)); + } + let settled = async { + match rx.wait_for(Option::is_some).await { + Ok(seen) => seen.unwrap_or(false), + Err(_) => false, + } + }; + tokio::time::timeout(READINESS_PROBE_TIMEOUT + Duration::from_secs(1), settled) + .await + .unwrap_or(false) + } + + fn spawn_probe(readiness: Arc, store: Arc) { + tokio::spawn(async move { + let mut guard = ProbeGuard { + readiness, + outcome: false, + }; + let joined = tokio::task::spawn_blocking(move || store.probe_ready()).await; + let ok = matches!(joined, Ok(Ok(()))); + if !ok { + tracing::warn!("lfs store readiness probe failed, reporting unready"); + } + guard.outcome = ok; + }); + } +} + +struct ProbeGuard { + readiness: Arc, + outcome: bool, +} + +impl Drop for ProbeGuard { + fn drop(&mut self) { + *self.readiness.probing.lock().unwrap() = false; + self.readiness.result.send_replace(Some(self.outcome)); + } +} + +pub(crate) fn routes() -> Router>> { + let batch = Router::new() + .route( + "/{did}/{name}/info/lfs/objects/batch", + post(batch_named::), + ) + .route("/{did}/info/lfs/objects/batch", post(batch_did::)) + .layer(DefaultBodyLimit::max(MAX_BATCH_BYTES)); + let objects = Router::new() + .route( + "/{did}/{name}/info/lfs/objects/{oid}", + get(object_named::).put(object_upload_named::), + ) + .route( + "/{did}/info/lfs/objects/{oid}", + get(object_did::).put(object_upload_did::), + ) + .layer(DefaultBodyLimit::disable()); + batch.merge(objects) +} + +fn fail(status: StatusCode, message: &str) -> Response { + ( + status, + [( + header::CONTENT_TYPE, + HeaderValue::from_static(BATCH_MEDIA_TYPE), + )], + json!({ "message": message }).to_string(), + ) + .into_response() +} + +fn not_enabled() -> Response { + fail(StatusCode::NOT_FOUND, "LFS isn't enabled on this knot") +} + +fn lfs_error(error: crate::XrpcError) -> Box { + Box::new(fail(error.status(), &error.to_string())) +} + +fn resolve_did( + state: &XrpcState, + segment: &crate::RepoDidSegment, +) -> Result> { + crate::resolve_repo_did(state, segment).map_err(lfs_error) +} + +async fn resolve_named( + state: &XrpcState, + owner: &crate::OwnerSegment, + name: &crate::RepoNameSegment, +) -> Result> { + crate::resolve_repo_named(state, owner, name) + .await + .map_err(lfs_error) +} + +#[derive(serde::Deserialize)] +#[serde(transparent)] +struct OidSegment(String); + +impl OidSegment { + fn as_str(&self) -> &str { + &self.0 + } +} + +#[derive(serde::Deserialize)] +struct RepoObjectParams { + did: crate::OwnerSegment, + name: crate::RepoNameSegment, + oid: OidSegment, +} + +#[derive(serde::Deserialize)] +struct DidObjectParams { + did: crate::RepoDidSegment, + oid: OidSegment, +} + +async fn batch_named( + State(state): State>>, + Path(crate::RepoPathParams { did, name }): Path, + peer: SocketPeer, + headers: HeaderMap, + body: Result, +) -> Response { + let repo = match resolve_named(&state, &did, &name).await { + Ok(repo) => repo, + Err(response) => return *response, + }; + serve_batch( + &state, + repo, + &format!("{}/{}", did.as_str(), name.as_str()), + peer, + &headers, + body, + ) + .await +} + +async fn batch_did( + State(state): State>>, + Path(did): Path, + peer: SocketPeer, + headers: HeaderMap, + body: Result, +) -> Response { + let repo = match resolve_did(&state, &did) { + Ok(repo) => repo, + Err(response) => return *response, + }; + serve_batch(&state, repo, did.as_str(), peer, &headers, body).await +} + +async fn serve_batch( + state: &XrpcState, + repo: RepoDid, + path_prefix: &str, + peer: SocketPeer, + headers: &HeaderMap, + body: Result, +) -> Response { + let Some(lfs) = state.lfs.as_ref() else { + return not_enabled(); + }; + let body = match body { + Ok(body) => body, + Err(rejection) => return fail(rejection.status(), &rejection.body_text()), + }; + let request: BatchRequest = match serde_json::from_slice(&body) { + Ok(request) => request, + Err(error) => { + return fail( + StatusCode::UNPROCESSABLE_ENTITY, + &format!("invalid batch request: {error}"), + ); + } + }; + if request.objects.len() > MAX_BATCH_OBJECTS { + return fail( + StatusCode::UNPROCESSABLE_ENTITY, + &format!("batch exceeds {MAX_BATCH_OBJECTS} objects"), + ); + } + if !request.transfers.is_empty() + && !request + .transfers + .iter() + .any(knot_lfs::TransferAdapter::is_basic) + { + return fail( + StatusCode::UNPROCESSABLE_ENTITY, + "no mutually supported transfer adapter, this server serves basic", + ); + } + if let Some(algo) = &request.hash_algo + && !algo.is_sha256() + { + return fail( + StatusCode::CONFLICT, + &format!("unsupported hash algorithm {:?}", algo.as_str()), + ); + } + let base = &state.knot_service_url; + let objects: Result, Box> = match request.operation { + BatchOperation::Upload => match authorized_pusher(state, peer, headers, &repo).await { + Ok(_actor) => match probe_all(lfs, &repo, &request.objects).await { + Ok(stored) => request + .objects + .iter() + .zip(stored) + .map(|(object, present)| upload_action(base, path_prefix, object, present)) + .collect(), + Err(response) => Err(response), + }, + Err(response) => return *response, + }, + BatchOperation::Download => match probe_all(lfs, &repo, &request.objects).await { + Ok(stored) => request + .objects + .iter() + .zip(stored) + .map(|(object, stored)| downloadable(base, path_prefix, object, stored)) + .collect(), + Err(response) => Err(response), + }, + }; + let objects = match objects { + Ok(objects) => objects, + Err(response) => return *response, + }; + let response = BatchResponse { + transfer: knot_lfs::TransferAdapter::Basic, + objects, + hash_algo: Some(knot_lfs::HashAlgo::Sha256), + }; + ( + StatusCode::OK, + [( + header::CONTENT_TYPE, + HeaderValue::from_static(BATCH_MEDIA_TYPE), + )], + serde_json::to_string(&response).expect("batch response always serializes"), + ) + .into_response() +} + +async fn authorized_pusher( + state: &XrpcState, + peer: SocketPeer, + headers: &HeaderMap, + repo: &RepoDid, +) -> Result> { + crate::authenticate_and_authorize_push( + state, + peer, + headers, + repo, + "you aren't authorized to push to this repository", + ) + .await + .map_err(|error| Box::new(challenge(error))) +} + +fn challenge(error: crate::XrpcError) -> Response { + if error.status() == StatusCode::UNAUTHORIZED { + unauthorized(&error.to_string()) + } else { + error.into_response() + } +} + +fn unauthorized(message: &str) -> Response { + ( + StatusCode::UNAUTHORIZED, + [ + (header::WWW_AUTHENTICATE, crate::BASIC_CHALLENGE), + ( + header::CONTENT_TYPE, + HeaderValue::from_static(BATCH_MEDIA_TYPE), + ), + ], + json!({ "message": message }).to_string(), + ) + .into_response() +} + +fn upload_action( + base: &KnotServiceUrl, + path_prefix: &str, + object: &BatchObject, + present: Option, +) -> Result> { + Ok(match present { + Some(size) => BatchResponseObject { + oid: object.oid.clone(), + size: ClaimedSize::new(size.get()), + authenticated: Some(true), + actions: None, + error: None, + }, + None => BatchResponseObject { + oid: object.oid.clone(), + size: object.size, + // I know what you're thinking about putting `authenticated: true`, + // but trust me TM git-lfs thinks + // "the href already has credentials on it" + // and omits the Authorization header from the following PUT req. + // PUT needs auth so the client would 401, re-run the batch, + // repeat. + // + // Having this be `None` makes git-lfs re-send the header it + // used on the batch-call in the first place. + authenticated: None, + actions: Some(BatchActions { + download: None, + upload: Some(BatchAction { + href: object_href(base, path_prefix, &object.oid)?, + }), + }), + error: None, + }, + }) +} + +async fn probe_all( + lfs: &LfsWeb, + repo: &RepoDid, + objects: &[BatchObject], +) -> Result>, Box> { + let store = Arc::clone(&lfs.handle.store); + let target = repo.clone(); + let oids: Vec = objects.iter().map(|object| object.oid.clone()).collect(); + tokio::task::spawn_blocking(move || { + oids.iter() + .map(|oid| store.probe(&target, oid)) + .collect::, _>>() + }) + .await + .map_err(|_| { + Box::new(fail( + StatusCode::INTERNAL_SERVER_ERROR, + "store probe failed", + )) + })? + .map_err(|error| { + tracing::warn!(repo = repo.as_str(), %error, "lfs store probe failed"); + Box::new(fail( + StatusCode::INTERNAL_SERVER_ERROR, + "store probe failed", + )) + }) +} + +fn downloadable( + base: &KnotServiceUrl, + path_prefix: &str, + object: &BatchObject, + stored: Option, +) -> Result> { + Ok(match stored { + Some(size) => BatchResponseObject { + oid: object.oid.clone(), + size: ClaimedSize::new(size.get()), + authenticated: Some(true), + actions: Some(BatchActions { + download: Some(BatchAction { + href: object_href(base, path_prefix, &object.oid)?, + }), + upload: None, + }), + error: None, + }, + None => BatchResponseObject { + oid: object.oid.clone(), + size: object.size, + authenticated: None, + actions: None, + error: Some(BatchObjectError { + code: HttpStatus::new(404), + message: "object not found".to_string(), + }), + }, + }) +} + +fn object_href( + base: &KnotServiceUrl, + path_prefix: &str, + oid: &LfsOid, +) -> Result> { + Url::parse(&format!( + "{}/{path_prefix}/info/lfs/objects/{oid}", + base.as_str() + )) + .map_err(|error| { + Box::new(fail( + StatusCode::INTERNAL_SERVER_ERROR, + &format!("cannot derive object href: {error}"), + )) + }) +} + +async fn object_named( + State(state): State>>, + Path(RepoObjectParams { did, name, oid }): Path, + request: Request, +) -> Response { + let repo = match resolve_named(&state, &did, &name).await { + Ok(repo) => repo, + Err(response) => return *response, + }; + serve_object(&state, repo, &oid, request).await +} + +async fn object_did( + State(state): State>>, + Path(DidObjectParams { did, oid }): Path, + request: Request, +) -> Response { + let repo = match resolve_did(&state, &did) { + Ok(repo) => repo, + Err(response) => return *response, + }; + serve_object(&state, repo, &oid, request).await +} + +async fn serve_object( + state: &XrpcState, + repo: RepoDid, + oid_raw: &OidSegment, + request: Request, +) -> Response { + let Some(lfs) = state.lfs.as_ref() else { + return not_enabled(); + }; + let Ok(oid) = LfsOid::new(oid_raw.as_str()) else { + return fail(StatusCode::NOT_FOUND, "object not found"); + }; + let located = { + let store = Arc::clone(&lfs.handle.store); + let target = repo.clone(); + let oid = oid.clone(); + tokio::task::spawn_blocking(move || store.object_file(&target, &oid)).await + }; + let (size, path) = match located { + Ok(Ok(Some((size, path)))) => (size, path), + Ok(Ok(None)) => return fail(StatusCode::NOT_FOUND, "object not found"), + Ok(Err(error)) => { + tracing::warn!(repo = repo.as_str(), oid = oid.as_str(), %error, "lfs store read failed"); + return fail(StatusCode::INTERNAL_SERVER_ERROR, "store read failed"); + } + Err(_) => return fail(StatusCode::INTERNAL_SERVER_ERROR, "store read failed"), + }; + let etag = format!("\"{oid}\""); + if client_holds_current(request.headers().get(header::IF_NONE_MATCH), &etag) { + return not_modified(&etag); + } + let request = honor_if_range(request, &etag); + let permit = match Arc::clone(&lfs.downloads).acquire_owned().await { + Ok(permit) => permit, + Err(_) => { + return fail( + StatusCode::SERVICE_UNAVAILABLE, + "server is shutting down, retry shortly", + ); + } + }; + let served = ServeFile::new(path) + .oneshot(request) + .await + .map(|response| response.map(Body::new)); + let mut response = match served { + Ok(response) => response, + Err(error) => match error {}, + }; + tracing::info!( + repo = repo.as_str(), + oid = oid.as_str(), + size = size.get(), + status = response.status().as_u16(), + "lfs object served over http" + ); + if response.status().is_success() { + let headers = response.headers_mut(); + headers.insert( + header::CONTENT_TYPE, + HeaderValue::from_static("application/octet-stream"), + ); + headers.insert( + header::CACHE_CONTROL, + HeaderValue::from_static(IMMUTABLE_CACHE), + ); + if let Ok(value) = HeaderValue::from_str(&etag) { + headers.insert(header::ETAG, value); + } + } + response.map(|body| { + Body::new(PermitBody { + body, + _permit: permit, + }) + }) +} + +async fn object_upload_named( + State(state): State>>, + Path(RepoObjectParams { did, name, oid }): Path, + peer: SocketPeer, + request: Request, +) -> Response { + let repo = match resolve_named(&state, &did, &name).await { + Ok(repo) => repo, + Err(response) => return *response, + }; + serve_object_upload(&state, repo, &oid, peer, request).await +} + +async fn object_upload_did( + State(state): State>>, + Path(DidObjectParams { did, oid }): Path, + peer: SocketPeer, + request: Request, +) -> Response { + let repo = match resolve_did(&state, &did) { + Ok(repo) => repo, + Err(response) => return *response, + }; + serve_object_upload(&state, repo, &oid, peer, request).await +} + +async fn serve_object_upload( + state: &XrpcState, + repo: RepoDid, + oid_raw: &OidSegment, + peer: SocketPeer, + request: Request, +) -> Response { + let Some(lfs) = state.lfs.as_ref() else { + return not_enabled(); + }; + let Ok(oid) = LfsOid::new(oid_raw.as_str()) else { + return fail(StatusCode::NOT_FOUND, "object not found"); + }; + if let Err(response) = authorized_pusher(state, peer, request.headers(), &repo).await { + return *response; + } + let Some(size) = content_length(request.headers()) else { + return fail( + StatusCode::LENGTH_REQUIRED, + "content-length is required for an lfs object upload", + ); + }; + let permit = match lfs.handle.admission.admit(size) { + Ok(permit) => permit, + Err(error) => return store_fault(&repo, &oid, error), + }; + let store = Arc::clone(&lfs.handle.store); + let target = repo.clone(); + let object = oid.clone(); + let landed = { + use futures::TryStreamExt; + let reader = tokio_util::io::StreamReader::new( + request + .into_body() + .into_data_stream() + .map_err(std::io::Error::other), + ); + tokio::task::spawn_blocking(move || { + let mut body = tokio_util::io::SyncIoBridge::new(reader); + let outcome = store.put(&target, &object, size, &mut body); + drop(permit); + outcome + }) + .await + }; + match landed { + Ok(Ok(())) => { + tracing::info!( + repo = repo.as_str(), + oid = oid.as_str(), + size = size.get(), + "lfs object stored over http" + ); + StatusCode::OK.into_response() + } + Ok(Err(error)) => store_fault(&repo, &oid, error), + Err(_) => fail(StatusCode::INTERNAL_SERVER_ERROR, "upload task died"), + } +} + +fn content_length(headers: &HeaderMap) -> Option { + headers + .get(header::CONTENT_LENGTH)? + .to_str() + .ok()? + .trim() + .parse::() + .ok() + .map(ClaimedSize::new) +} + +fn store_fault(repo: &RepoDid, oid: &LfsOid, error: LfsError) -> Response { + let status = match &error { + LfsError::HashMismatch { .. } | LfsError::SizeMismatch { .. } => { + StatusCode::UNPROCESSABLE_ENTITY + } + LfsError::SizeLimitExceeded { .. } => StatusCode::PAYLOAD_TOO_LARGE, + LfsError::FreeSpaceDenied { .. } => StatusCode::INSUFFICIENT_STORAGE, + LfsError::BodyRead { .. } => StatusCode::BAD_REQUEST, + _ => StatusCode::INTERNAL_SERVER_ERROR, + }; + if status.is_server_error() { + tracing::warn!(repo = repo.as_str(), oid = oid.as_str(), %error, "lfs object upload failed"); + } + fail(status, &error.to_string()) +} + +fn client_holds_current(if_none_match: Option<&HeaderValue>, etag: &str) -> bool { + if_none_match + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| { + value + .split(',') + .map(str::trim) + .any(|candidate| candidate == "*" || candidate.trim_start_matches("W/") == etag) + }) +} + +fn not_modified(etag: &str) -> Response { + let mut response = StatusCode::NOT_MODIFIED.into_response(); + let headers = response.headers_mut(); + headers.insert( + header::CACHE_CONTROL, + HeaderValue::from_static(IMMUTABLE_CACHE), + ); + if let Ok(value) = HeaderValue::from_str(etag) { + headers.insert(header::ETAG, value); + } + response +} + +fn honor_if_range(mut request: Request, etag: &str) -> Request { + let Some(if_range) = request.headers().get(header::IF_RANGE) else { + return request; + }; + let matches = if_range + .to_str() + .map(|value| value == etag) + .unwrap_or(false); + let headers = request.headers_mut(); + headers.remove(header::IF_RANGE); + if !matches { + headers.remove(header::RANGE); + } + request +} + +pub(crate) fn mirror_fork_objects( + state: Arc>, + upstream: Upstream, + fork: RepoDid, + wants: WantOids, + haves: HaveOids, +) -> futures::future::BoxFuture<'static, Result, crate::XrpcError>> { + Box::pin(mirror_fork_objects_inner( + state, upstream, fork, wants, haves, + )) +} + +async fn mirror_fork_objects_inner( + state: Arc>, + upstream: Upstream, + fork: RepoDid, + wants: WantOids, + haves: HaveOids, +) -> Result, crate::XrpcError> { + let state = &state; + let upstream = &upstream; + let fork = ⋔ + let Some(lfs) = state.lfs.as_ref() else { + return Ok(Vec::new()); + }; + let store = Arc::clone(&lfs.handle.store); + let admission = Arc::clone(&lfs.handle.admission); + + let needed = { + let layout = state.layout.clone(); + let fork = fork.clone(); + let store = Arc::clone(&store); + tokio::task::spawn_blocking(move || -> Result, String> { + let repo = layout.open(&fork).map_err(|error| error.to_string())?; + knot_lfs::scan_pointers(&repo, wants.wants(), haves.haves()) + .map_err(|error| error.to_string())? + .into_iter() + .map(|(oid, size)| match store.probe(&fork, &oid) { + Ok(None) => Ok(Some((oid, size))), + Ok(Some(_)) => Ok(None), + Err(fault) => Err(fault.to_string()), + }) + .filter_map(Result::transpose) + .collect() + }) + .await + }; + let needed = match needed { + Ok(Ok(needed)) => needed, + Ok(Err(fault)) => { + tracing::warn!( + repo = fork.as_str(), + fault, + "lfs pointer scan failed on fork" + ); + return Err(crate::XrpcError::internal("lfs pointer scan failed")); + } + Err(_) => { + return Err(crate::XrpcError::internal("lfs pointer scan task died")); + } + }; + if needed.is_empty() { + return Ok(Vec::new()); + } + + let missing = match upstream { + Upstream::Local(source) => { + let source = source.clone(); + let fork = fork.clone(); + let store = Arc::clone(&store); + let admission = Arc::clone(&admission); + tokio::task::spawn_blocking(move || { + needed + .into_iter() + .filter_map(|(oid, size)| { + let copied = admission.admit(size).and_then(|_permit| { + store + .read(&source, &oid) + .and_then(|mut body| store.put(&fork, &oid, size, &mut body)) + }); + match copied { + Ok(()) => None, + Err(fault) => { + tracing::warn!( + source = source.as_str(), + oid = oid.as_str(), + %fault, + "lfs fork copy skipped an object" + ); + Some(oid) + } + } + }) + .collect() + }) + .await + .map_err(|_| crate::XrpcError::internal("lfs fork copy task died"))? + } + Upstream::Remote(url) => match remote_batch_url(url) { + Some(batch_url) => { + use futures::StreamExt; + let chunks: Vec> = needed + .chunks(REMOTE_BATCH_CHUNK) + .map(<[(LfsOid, ClaimedSize)]>::to_vec) + .collect(); + futures::stream::iter(chunks) + .then(|chunk| { + fetch_remote_chunk( + Arc::clone(state), + Arc::clone(&store), + Arc::clone(&admission), + fork.clone(), + batch_url.clone(), + chunk, + ) + }) + .concat() + .await + } + None => needed.into_iter().map(|(oid, _)| oid).collect(), + }, + }; + if !missing.is_empty() { + tracing::warn!( + repo = fork.as_str(), + count = missing.len(), + "fork upstream couldn't serve every referenced lfs object" + ); + } + Ok(missing) +} + +const REMOTE_BATCH_CHUNK: usize = 100; + +fn remote_batch_url(origin: &Url) -> Option { + let mut origin = origin.clone(); + origin.set_query(None); + origin.set_fragment(None); + let base = origin.as_str().trim_end_matches('/'); + let base = match base.ends_with(".git") { + true => base.to_string(), + false => format!("{base}.git"), + }; + Url::parse(&format!("{base}/info/lfs/objects/batch")).ok() +} + +async fn fetch_remote_chunk( + state: Arc>, + store: Arc, + admission: Arc, + fork: RepoDid, + batch_url: Url, + chunk: Vec<(LfsOid, ClaimedSize)>, +) -> Vec { + use futures::StreamExt; + let all_missing = || chunk.iter().map(|(oid, _)| oid.clone()).collect::>(); + let request_body = BatchRequest { + operation: BatchOperation::Download, + transfers: vec![knot_lfs::TransferAdapter::Basic], + reference: None, + objects: chunk + .iter() + .map(|(oid, size)| knot_lfs::BatchObject { + oid: oid.clone(), + size: *size, + }) + .collect(), + hash_algo: Some(knot_lfs::HashAlgo::Sha256), + }; + let body = match serde_json::to_vec(&request_body) { + Ok(body) => body, + Err(_) => return all_missing(), + }; + let mut request = knot_runtime::HttpRequest::post(batch_url.clone(), body.into()); + request.headers.insert( + header::CONTENT_TYPE, + HeaderValue::from_static(BATCH_MEDIA_TYPE), + ); + request + .headers + .insert(header::ACCEPT, HeaderValue::from_static(BATCH_MEDIA_TYPE)); + let response = match state.git_http.execute(request).await { + Ok(response) if response.status.is_success() => response, + Ok(response) => { + tracing::warn!( + url = batch_url.as_str(), + status = response.status.as_u16(), + "upstream lfs batch refused" + ); + return all_missing(); + } + Err(fault) => { + tracing::warn!(url = batch_url.as_str(), %fault, "upstream lfs batch failed"); + return all_missing(); + } + }; + let parsed: BatchResponse = match serde_json::from_slice(&response.body) { + Ok(parsed) => parsed, + Err(fault) => { + tracing::warn!(url = batch_url.as_str(), %fault, "upstream lfs batch unparsable"); + return all_missing(); + } + }; + let declared: BTreeMap = chunk.into_iter().collect(); + let unanswered = unanswered_oids(&declared, &parsed.objects); + let tagged: Vec<(BatchResponseObject, ClaimedSize)> = parsed + .objects + .into_iter() + .filter_map(|object| { + declared + .get(&object.oid) + .copied() + .map(|size| (object, size)) + }) + .collect(); + let failed: Vec = futures::stream::iter(tagged) + .then(|(object, size)| { + fetch_remote_object( + Arc::clone(&state), + Arc::clone(&store), + Arc::clone(&admission), + fork.clone(), + object, + size, + ) + }) + .filter_map(std::future::ready) + .collect() + .await; + unanswered.into_iter().chain(failed).collect() +} + +fn unanswered_oids( + declared: &BTreeMap, + answered: &[BatchResponseObject], +) -> Vec { + let answered: BTreeSet<&LfsOid> = answered.iter().map(|object| &object.oid).collect(); + declared + .keys() + .filter(|oid| !answered.contains(oid)) + .cloned() + .collect() +} + +fn href_is_fetchable(url: &Url) -> bool { + let scheme_ok = matches!(url.scheme(), "http" | "https"); + let host_ok = match url.host() { + Some(url::Host::Ipv4(ip)) => !knot_runtime::is_blocked_ip(ip.into()), + Some(url::Host::Ipv6(ip)) => !knot_runtime::is_blocked_ip(ip.into()), + Some(url::Host::Domain(_)) => true, + None => false, + }; + scheme_ok && host_ok +} + +async fn fetch_remote_object( + state: Arc>, + store: Arc, + admission: Arc, + fork: RepoDid, + object: BatchResponseObject, + declared: ClaimedSize, +) -> Option { + let Some(action) = object.actions.and_then(|actions| actions.download) else { + return Some(object.oid); + }; + if !href_is_fetchable(&action.href) { + tracing::warn!( + oid = object.oid.as_str(), + href = action.href.as_str(), + "lfs fork download href isn't a public http target" + ); + return Some(object.oid); + } + let permit = match admission.admit(declared) { + Ok(permit) => permit, + Err(fault) => { + tracing::warn!(oid = object.oid.as_str(), %fault, "lfs fork download refused by admission"); + return Some(object.oid); + } + }; + let streamed = match state + .git_http + .execute_streamed(knot_runtime::HttpRequest::get(action.href)) + .await + { + Ok(streamed) if streamed.status.is_success() => streamed, + _ => return Some(object.oid), + }; + let landed = { + use futures::TryStreamExt; + let reader = + tokio_util::io::StreamReader::new(streamed.body.map_err(std::io::Error::other)); + let oid = object.oid.clone(); + tokio::task::spawn_blocking(move || { + let mut body = tokio_util::io::SyncIoBridge::new(reader); + let outcome = store.put(&fork, &oid, declared, &mut body); + drop(permit); + outcome + }) + .await + }; + match landed { + Ok(Ok(())) => None, + Ok(Err(fault)) => { + tracing::warn!(oid = object.oid.as_str(), %fault, "lfs fork download failed"); + Some(object.oid) + } + Err(_) => Some(object.oid), + } +} + +struct PermitBody { + body: Body, + _permit: OwnedSemaphorePermit, +} + +impl http_body::Body for PermitBody { + type Data = Bytes; + type Error = axum::Error; + + fn poll_frame( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll, Self::Error>>> { + std::pin::Pin::new(&mut self.body).poll_frame(cx) + } + + fn is_end_stream(&self) -> bool { + self.body.is_end_stream() + } + + fn size_hint(&self) -> http_body::SizeHint { + self.body.size_hint() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_remote_batch_endpoint_matches_git_lfs_derivation() { + let plain = Url::parse("https://nel.pet/did:web:witchcraft.systems/anemone").unwrap(); + assert_eq!( + remote_batch_url(&plain).unwrap().as_str(), + "https://nel.pet/did:web:witchcraft.systems/anemone.git/info/lfs/objects/batch" + ); + let suffixed = Url::parse("https://nel.pet/did:plc:cuttle.git").unwrap(); + assert_eq!( + remote_batch_url(&suffixed).unwrap().as_str(), + "https://nel.pet/did:plc:cuttle.git/info/lfs/objects/batch" + ); + let trailing = Url::parse("https://nel.pet/did:plc:cuttle/").unwrap(); + assert_eq!( + remote_batch_url(&trailing).unwrap().as_str(), + "https://nel.pet/did:plc:cuttle.git/info/lfs/objects/batch" + ); + let decorated = Url::parse("https://nel.pet/did:plc:cuttle?ref=main#readme").unwrap(); + assert_eq!( + remote_batch_url(&decorated).unwrap().as_str(), + "https://nel.pet/did:plc:cuttle.git/info/lfs/objects/batch" + ); + } + + #[test] + fn oids_the_upstream_batch_never_answers_count_as_missing() { + use sha2::{Digest, Sha256}; + let held = LfsOid::from_digest(Sha256::digest(b"held").into()); + let ignored = LfsOid::from_digest(Sha256::digest(b"ignored").into()); + let declared: BTreeMap = [ + (held.clone(), ClaimedSize::new(4)), + (ignored.clone(), ClaimedSize::new(7)), + ] + .into_iter() + .collect(); + let answered = vec![BatchResponseObject { + oid: held, + size: ClaimedSize::new(4), + authenticated: None, + actions: None, + error: None, + }]; + assert_eq!(unanswered_oids(&declared, &answered), vec![ignored.clone()]); + assert_eq!( + unanswered_oids(&declared, &[]).len(), + 2, + "an empty upstream response must leave every oid missing" + ); + } + + #[test] + fn a_stale_if_range_drops_the_range_for_a_full_response() { + use axum::http::Request as HttpRequest; + let etag = "\"6c17f2007cbe934aee6e309b28b2fba3c119d98be6ea4156da3aa3173456ad16\""; + + let matching = HttpRequest::builder() + .header(header::IF_RANGE, etag) + .header(header::RANGE, "bytes=0-9") + .body(Body::empty()) + .unwrap(); + let kept = honor_if_range(matching, etag); + assert!(kept.headers().get(header::IF_RANGE).is_none()); + assert!( + kept.headers().get(header::RANGE).is_some(), + "a matching validator keeps the range for a 206" + ); + + let stale = HttpRequest::builder() + .header(header::IF_RANGE, "\"stale\"") + .header(header::RANGE, "bytes=0-9") + .body(Body::empty()) + .unwrap(); + let full = honor_if_range(stale, etag); + assert!(full.headers().get(header::IF_RANGE).is_none()); + assert!( + full.headers().get(header::RANGE).is_none(), + "a stale validator drops the range so the client gets the whole object" + ); + } + + #[test] + fn revalidation_matches_the_oid_etag() { + let etag = "\"6c17f2007cbe934aee6e309b28b2fba3c119d98be6ea4156da3aa3173456ad16\""; + let holds = + |value: &str| client_holds_current(Some(&HeaderValue::from_str(value).unwrap()), etag); + assert!(holds(etag)); + assert!(holds(&format!("W/{etag}"))); + assert!(holds(&format!("\"other\", {etag}"))); + assert!(holds("*")); + assert!(!holds("\"other\"")); + assert!(!client_holds_current(None, etag)); + } +} diff --git a/knot2/crates/knot-xrpc/src/lib.rs b/knot2/crates/knot-xrpc/src/lib.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/lib.rs @@ -0,0 +1,632 @@ +mod blocklist; +mod body; +mod branches; +mod cob; +mod collaborators; +mod error; +mod events; +mod forks; +mod lfs; +mod lists; +mod locks; +mod members; +mod merge; +mod patchtext; +mod query; +mod reads; +mod receive; +mod repos; +mod reservations; +mod service; +mod sniff; +mod wire; + +#[cfg(test)] +mod tests; + +pub use error::XrpcError; +pub use knot_pack::MaxWireBytes; +pub use knot_postreceive::LanguagesPushBudget; +pub use knot_resource::{ + Burst, GlobalInflight, LimitConfig, PerPeerInflight, PreAuthLimiter, RateLimit, RefillMicros, +}; +pub use lfs::LfsWeb; +pub use locks::CobLocks; +pub use merge::Committer; +pub use receive::advertiser as receive_advertiser; +pub use reservations::{GlobalQuota, PerActorQuota, ReservationTtl, Reservations}; + +use std::collections::BTreeSet; +use std::net::IpAddr; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use axum::Json; +use axum::Router; +use axum::body::Bytes; +use axum::extract::{DefaultBodyLimit, FromRequestParts, MatchedPath, Request, State}; +use axum::middleware::{Next, from_fn_with_state}; +use axum::response::{IntoResponse, Response}; +use axum::routing::{get, post}; +use http::request::Parts; +use http::{HeaderMap, HeaderValue, StatusCode, header::AUTHORIZATION}; +use serde::de::DeserializeOwned; +use serde_json::json; + +use knot_atproto::{Atproto, AtprotoError, ServiceJwt}; +use knot_events::{EventLog, SubscriberGate}; +use knot_git::Layout; +use knot_index::{Index, Resolved}; +use knot_maintenance::MaintenanceHandle; +use knot_resource::Slots; +use knot_runtime::{Clock, Entropy, HttpTransport}; +use knot_secrets::SealedStore; +use knot_types::{ + AccountDid, AdmissionPolicy, AppviewEndpoint, CiLogsAddr, KnotHostname, KnotId, KnotServiceUrl, + Nsid, OwnerDid, OwnerRef, RepoDid, RepoRkey, UnixSeconds, +}; + +use base64::Engine; +use knot_pack::SocketPeer; +use knot_resource::{AdmitGuard, Refusal}; + +pub(crate) const PUSH_NSID: &str = "sh.tangled.repo.push"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ReadBudget { + Within(Duration), + Unbounded, +} + +impl ReadBudget { + pub fn deadline(self) -> Option { + match self { + ReadBudget::Within(budget) => Some(Instant::now() + budget), + ReadBudget::Unbounded => None, + } + } +} + +// `XrpcState` keeps a bunch of these side by side, +// some usize & some u64. +// Within each group every one of them typechecked in every other one's slot. +knot_types::scalar_newtype! { + pub struct BodyLimit(usize); + pub struct PatchLimit(usize); + pub struct PatchDecompressedLimit(u64); + pub struct ResponseLimit(usize); + pub struct ArchiveLimit(u64); + pub struct ForkPackLimit(u64); + pub struct TreeReadBudget(ReadBudget); + pub struct BlobReadBudget(ReadBudget); + pub struct LanguagesReadBudget(ReadBudget); +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ByteLimits { + pub body: BodyLimit, + pub patch: PatchLimit, + pub patch_decompressed: PatchDecompressedLimit, + pub response: ResponseLimit, + pub archive: ArchiveLimit, + pub fork_pack: ForkPackLimit, + pub pack: MaxWireBytes, +} + +impl Default for ByteLimits { + fn default() -> Self { + Self { + body: BodyLimit::new(64 * 1024), + patch: PatchLimit::new(16 * 1024 * 1024), + patch_decompressed: PatchDecompressedLimit::new(128 * 1024 * 1024), + response: ResponseLimit::new(5 * 1024 * 1024), + archive: ArchiveLimit::new(1024 * 1024 * 1024), + fork_pack: ForkPackLimit::new(1024 * 1024 * 1024), + pack: MaxWireBytes::new(8 * 1024 * 1024 * 1024), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Budgets { + pub tree_last_commit: TreeReadBudget, + pub blob_last_commit: BlobReadBudget, + pub languages: LanguagesReadBudget, + pub languages_push: LanguagesPushBudget, +} + +impl Default for Budgets { + fn default() -> Self { + Self { + tree_last_commit: TreeReadBudget::new(ReadBudget::Within(Duration::from_millis(300))), + blob_last_commit: BlobReadBudget::new(ReadBudget::Within(Duration::from_millis(2_000))), + languages: LanguagesReadBudget::new(ReadBudget::Within(Duration::from_millis(1_000))), + languages_push: LanguagesPushBudget::new(Duration::from_millis(2_000)), + } + } +} + +pub struct XrpcState { + pub layout: Layout, + pub index: Arc, + pub atproto: Arc>, + pub secrets: Arc, + pub entropy: Arc, + pub admins: BTreeSet, + pub admission: AdmissionPolicy, + pub knot_did: KnotId, + pub knot_hostname: KnotHostname, + pub ci_logs: Option, + pub meta_path: PathBuf, + pub knot_service_url: KnotServiceUrl, + pub limiter: Arc, + pub cob_locks: Arc, + pub reservations: Arc, + pub trusted_proxy_header: Option, + pub committer: Committer, + pub byte_limits: ByteLimits, + pub budgets: Budgets, + pub git_http: Arc, + pub pack_limits: knot_pack::PackLimits, + pub service_owner: AccountDid, + pub events: Arc>, + pub subscriber_gate: Arc, + pub maintenance: MaintenanceHandle, + pub appview: AppviewEndpoint, + pub slots: Slots, + pub lfs: Option, + pub catalog: Arc, +} + +impl XrpcState { + pub fn now(&self) -> UnixSeconds { + UnixSeconds::new((self.atproto.now().get() / 1_000_000) as i64) + } + + pub(crate) fn knot_authority(&self) -> &str { + self.knot_service_url.authority() + } + + pub(crate) async fn authenticate( + &self, + headers: &HeaderMap, + method: &Method, + ) -> Result { + let token = bearer(headers)?; + self.atproto + .verify_service_jwt(&token, method.nsid()) + .await + .map_err(map_verify_error) + } + + pub(crate) async fn authenticate_push( + &self, + headers: &HeaderMap, + ) -> Result { + let token = push_credential(headers)?; + let method = Nsid::new_owned(PUSH_NSID).expect("push nsid is always a valid nsid"); + self.atproto + .verify_service_jwt_guarded( + &token, + &method, + knot_atproto::ReplayGuard::ReusableUntilExpiry, + ) + .await + .map_err(map_verify_error) + } +} + +fn map_verify_error(error: AtprotoError) -> XrpcError { + if error.is_transient() { + XrpcError::upstream_unavailable(error.to_string()) + } else { + XrpcError::auth_required(error.to_string()) + } +} + +pub(crate) struct Method(Nsid); + +impl Method { + fn nsid(&self) -> &Nsid { + &self.0 + } + + #[cfg(test)] + pub(crate) fn from_nsid(nsid: &str) -> Self { + Self(Nsid::new_owned(nsid).expect("test route nsid parses")) + } +} + +impl FromRequestParts for Method { + type Rejection = XrpcError; + + async fn from_request_parts(parts: &mut Parts, state: &S) -> Result { + let matched = MatchedPath::from_request_parts(parts, state) + .await + .map_err(|_| XrpcError::internal("xrpc handler reached without a matched route"))?; + let nsid = matched + .as_str() + .strip_prefix("/xrpc/") + .ok_or_else(|| XrpcError::internal("xrpc route paths are prefixed with /xrpc/"))?; + Nsid::new_owned(nsid) + .map(Self) + .map_err(|_| XrpcError::internal("route nsid is always a valid nsid")) + } +} + +pub fn router(state: Arc>) -> Router { + let merge_routes = Router::new() + .route(merge::MERGE_ROUTE, post(merge::merge::)) + .route(merge::MERGE_CHECK_ROUTE, post(merge::merge_check::)) + .layer(DefaultBodyLimit::max(state.byte_limits.patch.get())); + Router::new() + .merge(merge_routes) + .route(members::ADD_ROUTE, post(members::add_member::)) + .route(members::REMOVE_ROUTE, post(members::remove_member::)) + .route(blocklist::BAN_ROUTE, post(blocklist::ban::)) + .route(blocklist::UNBAN_ROUTE, post(blocklist::unban::)) + .route( + collaborators::ADD_ROUTE, + post(collaborators::add_collaborator::), + ) + .route( + collaborators::REMOVE_ROUTE, + post(collaborators::remove_collaborator::), + ) + .route(repos::CREATE_ROUTE, post(repos::create_repo::)) + .route(repos::DELETE_ROUTE, post(repos::delete_repo::)) + .route(repos::RENAME_ROUTE, post(repos::rename_repo::)) + .route(repos::RESERVE_ROUTE, post(repos::reserve_key::)) + .route( + branches::SET_DEFAULT_ROUTE, + post(branches::set_default_branch::), + ) + .route( + branches::DELETE_ROUTE, + post(branches::delete_branch::), + ) + .route(forks::STATUS_ROUTE, post(forks::fork_status::)) + .route(forks::SYNC_ROUTE, post(forks::fork_sync::)) + .route(forks::HIDDEN_REF_ROUTE, post(forks::hidden_ref::)) + .route(reads::TREE_ROUTE, get(reads::repo_tree::)) + .route(reads::LOG_ROUTE, get(reads::repo_log::)) + .route(reads::BRANCHES_ROUTE, get(reads::repo_branches::)) + .route(reads::BRANCH_ROUTE, get(reads::repo_branch::)) + .route(reads::TAGS_ROUTE, get(reads::repo_tags::)) + .route(reads::TAG_ROUTE, get(reads::repo_tag::)) + .route(reads::BLOB_ROUTE, get(reads::repo_blob::)) + .route(reads::DIFF_ROUTE, get(reads::repo_diff::)) + .route(reads::COMPARE_ROUTE, get(reads::repo_compare::)) + .route(reads::ARCHIVE_ROUTE, get(reads::repo_archive::)) + .route(reads::LANGUAGES_ROUTE, get(reads::repo_languages::)) + .route( + reads::GET_DEFAULT_BRANCH_ROUTE, + get(reads::repo_get_default_branch::), + ) + .route( + reads::DESCRIBE_REPO_ROUTE, + get(reads::repo_describe_repo::), + ) + .route(reads::LIST_REFS_ROUTE, get(reads::git_list_refs::)) + .route(reads::LIST_REPOS_ROUTE, get(reads::sync_list_repos::)) + .route(lists::LIST_MEMBERS_ROUTE, get(lists::list_members::)) + .route( + lists::LIST_COLLABORATORS_ROUTE, + get(lists::list_collaborators::), + ) + .route(service::VERSION_ROUTE, get(service::version)) + .route(service::OWNER_ROUTE, get(service::owner::)) + .layer(DefaultBodyLimit::max(state.byte_limits.body.get())) + .layer(from_fn_with_state( + Arc::clone(&state), + enforce_pre_auth_limit::, + )) + .merge(lfs::routes::()) + .merge(receive::routes::()) + .route(service::HEALTH_ROUTE, get(service::health::)) + .route(events::EVENTS_ROUTE, get(events::events::)) + .with_state(state) +} + +async fn enforce_pre_auth_limit( + State(state): State>>, + socket: SocketPeer, + request: Request, + next: Next, +) -> Response { + let peer = effective_peer(&state, socket, request.headers()); + match admit_pre_auth(&state, peer) { + Ok(guard) => { + let response = next.run(request).await; + drop(guard); + response + } + Err(error) => error.into_response(), + } +} + +pub(crate) fn effective_peer( + state: &XrpcState, + socket: SocketPeer, + headers: &HeaderMap, +) -> Option { + state + .trusted_proxy_header + .as_ref() + .and_then(|header| knot_types::forwarded_peer(headers, header)) + .or(socket.ip()) +} + +pub(crate) fn admit_pre_auth( + state: &XrpcState, + peer: Option, +) -> Result { + state + .limiter + .admit(peer, state.atproto.now()) + .map_err(|refusal| match refusal { + Refusal::RateLimited => { + XrpcError::rate_limited("too many pre-authentication requests, retry shortly") + } + Refusal::Saturated => { + XrpcError::overloaded("knot is shedding pre-authentication load, retry shortly") + } + }) +} + +pub(crate) const BASIC_CHALLENGE: HeaderValue = HeaderValue::from_static("Basic realm=\"knot\""); + +fn strip_bearer(value: &str) -> Option<&str> { + let (scheme, rest) = value.split_once(' ')?; + scheme.eq_ignore_ascii_case("Bearer").then_some(rest) +} + +fn bearer(headers: &HeaderMap) -> Result { + headers + .get(AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .and_then(strip_bearer) + .map(str::trim) + .and_then(|token| ServiceJwt::new(token).ok()) + .ok_or_else(|| XrpcError::auth_required("missing or malformed Bearer authorization header")) +} + +fn strip_basic(value: &str) -> Option { + let (scheme, rest) = value.split_once(' ')?; + if !scheme.eq_ignore_ascii_case("Basic") { + return None; + } + let decoded = base64::engine::general_purpose::STANDARD + .decode(rest.trim()) + .ok()?; + let text = String::from_utf8(decoded).ok()?; + let (_user, password) = text.split_once(':')?; + (!password.is_empty()).then(|| password.to_string()) +} + +fn push_credential(headers: &HeaderMap) -> Result { + let value = headers + .get(AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .ok_or_else(|| XrpcError::auth_required("missing authorization header"))?; + strip_bearer(value) + .map(str::trim) + .map(str::to_string) + .or_else(|| strip_basic(value)) + .and_then(|token| ServiceJwt::new(token).ok()) + .ok_or_else(|| { + XrpcError::auth_required("authorization isn't a bearer token or basic credential") + }) +} + +pub(crate) fn decode(body: &Bytes) -> Result { + serde_json::from_slice(body) + .map_err(|error| XrpcError::invalid_request(format!("invalid request body: {error}"))) +} + +pub(crate) fn ok_empty() -> Response { + (StatusCode::OK, Json(json!({}))).into_response() +} + +pub(crate) fn current_owner( + state: &XrpcState, + repo: &RepoDid, +) -> Option { + match state.index.owner_of(repo) { + Resolved::Ready(owner) => owner, + Resolved::Warming => None, + } +} + +pub(crate) async fn fold_collaborators( + state: &XrpcState, + repo: &RepoDid, +) { + let index = Arc::clone(&state.index); + let target = repo.clone(); + let _ = run_blocking(move || Ok(index.ensure_collaborators(&target))).await; +} + +pub(crate) async fn authorize_push( + state: &XrpcState, + actor: &AccountDid, + repo: &RepoDid, + denied: &str, +) -> Result<(), XrpcError> { + fold_collaborators(state, repo).await; + let acl = knot_acl::KnotAcl::new(&state.admins, state.admission, &state.index); + if knot_acl::can_push(&acl, actor, repo).is_allowed() { + Ok(()) + } else { + Err(XrpcError::forbidden(denied)) + } +} + +pub(crate) async fn authenticate_and_authorize_push( + state: &XrpcState, + socket: SocketPeer, + headers: &HeaderMap, + repo: &RepoDid, + denied: &str, +) -> Result { + let peer = effective_peer(state, socket, headers); + let guard = admit_pre_auth(state, peer)?; + let actor = state.authenticate_push(headers).await?; + guard.refund(); + authorize_push(state, &actor, repo, denied).await?; + Ok(actor) +} + +pub(crate) async fn run_blocking(task: F) -> Result +where + F: FnOnce() -> Result + Send + 'static, + T: Send + 'static, +{ + match tokio::task::spawn_blocking(task).await { + Ok(result) => result, + Err(_) => Err(XrpcError::internal("blocking task failed to complete")), + } +} + +#[derive(serde::Deserialize)] +#[serde(transparent)] +pub(crate) struct OwnerSegment(String); + +#[derive(serde::Deserialize)] +#[serde(transparent)] +pub(crate) struct RepoNameSegment(String); + +impl OwnerSegment { + pub(crate) fn as_str(&self) -> &str { + &self.0 + } +} + +impl RepoNameSegment { + pub(crate) fn as_str(&self) -> &str { + &self.0 + } +} + +#[derive(serde::Deserialize)] +#[serde(transparent)] +pub(crate) struct RepoDidSegment(String); + +impl RepoDidSegment { + pub(crate) fn as_str(&self) -> &str { + &self.0 + } +} + +#[derive(serde::Deserialize)] +pub(crate) struct RepoPathParams { + pub(crate) did: OwnerSegment, + pub(crate) name: RepoNameSegment, +} + +pub(crate) fn resolve_repo_did( + state: &XrpcState, + segment: &RepoDidSegment, +) -> Result { + let raw = segment.as_str(); + let trimmed = raw.strip_suffix(".git").unwrap_or(raw); + let did = RepoDid::new(trimmed).map_err(|_| XrpcError::not_found("repository not found"))?; + match state.index.owner_of(&did) { + Resolved::Ready(Some(_)) => Ok(did), + Resolved::Ready(None) => Err(XrpcError::not_found("repository not found")), + Resolved::Warming => Err(XrpcError::warming( + "registry projection is still warming, retry shortly", + )), + } +} + +pub(crate) async fn resolve_repo_named( + state: &XrpcState, + owner: &OwnerSegment, + name: &RepoNameSegment, +) -> Result { + let owner = resolve_owner_segment(state, owner).await?; + RepoRkey::clone_path_candidates(name.as_str()) + .find_map(|rkey| match state.index.resolve_repo(&owner, &rkey) { + Resolved::Ready(Some(did)) => Some(Ok(did)), + Resolved::Ready(None) => None, + Resolved::Warming => Some(Err(XrpcError::warming( + "registry projection is still warming, retry shortly", + ))), + }) + .unwrap_or_else(|| Err(XrpcError::not_found("repository not found"))) +} + +async fn resolve_owner_segment( + state: &XrpcState, + owner: &OwnerSegment, +) -> Result { + let not_found = || XrpcError::not_found("repository not found"); + match OwnerRef::parse(owner.as_str()).ok_or_else(not_found)? { + OwnerRef::Did(did) => Ok(did), + OwnerRef::Handle(handle) => state + .atproto + .resolve_handle_to_did(&handle) + .await + .map(OwnerDid::from) + .map_err(|_| not_found()), + } +} + +#[cfg(test)] +mod credential_tests { + use super::push_credential; + use base64::Engine; + use http::{HeaderMap, HeaderValue, header::AUTHORIZATION}; + + fn with(value: &str) -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert(AUTHORIZATION, HeaderValue::from_str(value).unwrap()); + headers + } + + fn basic(user_pass: &str) -> String { + format!( + "Basic {}", + base64::engine::general_purpose::STANDARD.encode(user_pass) + ) + } + + #[test] + fn a_bearer_token_is_taken_verbatim() { + assert_eq!( + push_credential(&with("Bearer jwt.abc.def")) + .unwrap() + .as_str(), + "jwt.abc.def" + ); + assert_eq!( + push_credential(&with("bearer jwt.abc.def")) + .unwrap() + .as_str(), + "jwt.abc.def" + ); + } + + #[test] + fn a_basic_credential_yields_the_password_after_the_first_colon() { + assert_eq!( + push_credential(&with(&basic("x-tangled-token:jwt.abc.def"))) + .unwrap() + .as_str(), + "jwt.abc.def", + "RFC 7617 puts the token in the password half, so the username stays colon-free" + ); + } + + #[test] + fn malformed_or_empty_credentials_are_rejected() { + assert!(push_credential(&HeaderMap::new()).is_err()); + assert!(push_credential(&with("Bearer ")).is_err()); + assert!(push_credential(&with(&basic("x-tangled-token:"))).is_err()); + assert!(push_credential(&with(&basic("no-colon"))).is_err()); + assert!(push_credential(&with("Basic !!!not-base64")).is_err()); + assert!(push_credential(&with("Digest whatever")).is_err()); + } +} diff --git a/knot2/crates/knot-xrpc/src/lists.rs b/knot2/crates/knot-xrpc/src/lists.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/lists.rs @@ -0,0 +1,186 @@ +use std::sync::Arc; + +use axum::Json; +use axum::extract::{FromRequestParts, Query, State}; +use axum::response::{IntoResponse, Response}; +use http::StatusCode; +use http::request::Parts; +use serde::{Deserialize, Serialize}; + +use knot_cobs::Grant; +use knot_index::Resolved; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{AccountDid, RepoDid}; + +use crate::XrpcState; +use crate::error::XrpcError; +use crate::query::{Limit, Offset, Order, Total, ValidatedQuery, next_cursor}; +use crate::wire::rfc3339; + +pub(crate) const LIST_MEMBERS_ROUTE: &str = "/xrpc/sh.tangled.knot.listMembers"; +pub(crate) const LIST_COLLABORATORS_ROUTE: &str = "/xrpc/sh.tangled.repo.listCollaborators"; + +const DEFAULT_LIMIT: usize = 50; +const MAX_LIMIT: usize = 1000; + +#[derive(Deserialize)] +pub(crate) struct Paging { + #[serde(default)] + limit: Limit, + #[serde(default)] + cursor: Offset, + #[serde(default)] + order: Order, +} + +struct Window { + offset: Offset, + limit: Limit, + descending: bool, +} + +impl Paging { + fn window(self) -> Window { + Window { + offset: self.cursor, + limit: self.limit, + descending: self.order.descending(), + } + } +} + +#[derive(Deserialize)] +struct SubjectQuery { + subject: Option, +} + +fn subject_param(parts: &Parts) -> Result { + Query::::try_from_uri(&parts.uri) + .map_err(|rejection| XrpcError::invalid_request(rejection.body_text()))? + .0 + .subject + .filter(|raw| !raw.is_empty()) + .ok_or_else(|| XrpcError::invalid_request("missing subject parameter")) +} + +pub(crate) struct MemberSubject; + +impl FromRequestParts for MemberSubject { + type Rejection = XrpcError; + + async fn from_request_parts(parts: &mut Parts, _state: &S) -> Result { + let subject = subject_param(parts)?; + AccountDid::new(subject) + .map(|_| MemberSubject) + .map_err(|_| { + XrpcError::named( + StatusCode::BAD_REQUEST, + "InvalidSubject", + "subject must be an account DID", + ) + }) + } +} + +pub(crate) struct CollaboratorRepo(RepoDid); + +impl FromRequestParts for CollaboratorRepo { + type Rejection = XrpcError; + + async fn from_request_parts(parts: &mut Parts, _state: &S) -> Result { + let subject = subject_param(parts)?; + RepoDid::new(subject).map(CollaboratorRepo).map_err(|_| { + XrpcError::named( + StatusCode::BAD_REQUEST, + "InvalidRepo", + "subject must be a repo DID", + ) + }) + } +} + +#[derive(Serialize)] +struct ItemWire { + subject: AccountDid, + #[serde(rename = "addedBy")] + added_by: AccountDid, + #[serde(rename = "createdAt")] + created_at: String, +} + +#[derive(Serialize)] +struct PageWire { + items: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + cursor: Option, +} + +fn respond(mut entries: Vec, window: Window) -> Response { + entries.sort_by(|a, b| { + let by_time = a.created_at.cmp(&b.created_at); + let by_time = match window.descending { + true => by_time.reverse(), + false => by_time, + }; + by_time.then_with(|| a.subject.cmp(&b.subject)) + }); + let total = entries.len(); + let items = entries + .into_iter() + .skip(window.offset.get()) + .take(window.limit.get()) + .map(|grant| ItemWire { + subject: grant.subject, + added_by: grant.added_by, + created_at: rfc3339(grant.created_at.get(), 0), + }) + .collect(); + let cursor = next_cursor(window.offset, window.limit, Total::new(total)); + Json(PageWire { items, cursor }).into_response() +} + +fn members_warming() -> XrpcError { + XrpcError::warming("members projection is still warming") +} + +fn collaborators_warming() -> XrpcError { + XrpcError::warming("collaborators projection is still warming") +} + +pub(crate) async fn list_members( + State(state): State>>, + _subject: MemberSubject, + ValidatedQuery(paging): ValidatedQuery, +) -> Result { + let window = paging.window(); + match state.index.member_entries() { + Resolved::Warming => Err(members_warming()), + Resolved::Ready(entries) => Ok(respond(entries, window)), + } +} + +pub(crate) async fn list_collaborators( + State(state): State>>, + CollaboratorRepo(repo): CollaboratorRepo, + ValidatedQuery(paging): ValidatedQuery, +) -> Result { + let window = paging.window(); + match state.index.owner_of(&repo) { + Resolved::Warming => return Err(crate::reads::warming()), + Resolved::Ready(None) => return Ok(respond(Vec::new(), window)), + Resolved::Ready(Some(_)) => {} + } + let index = Arc::clone(&state.index); + let target = repo.clone(); + let entries = crate::run_blocking(move || { + index + .ensure_collaborators(&target) + .map_err(|_| collaborators_warming())?; + match index.collaborator_entries(&target) { + Resolved::Warming => Err(collaborators_warming()), + Resolved::Ready(entries) => Ok(entries), + } + }) + .await?; + Ok(respond(entries, window)) +} diff --git a/knot2/crates/knot-xrpc/src/locks.rs b/knot2/crates/knot-xrpc/src/locks.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/locks.rs @@ -0,0 +1,38 @@ +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; +use std::sync::{Mutex, MutexGuard}; + +use knot_types::RepoDid; + +const REPO_SHARDS: usize = 64; + +pub struct CobLocks { + meta: Mutex<()>, + repos: Vec>, +} + +impl Default for CobLocks { + fn default() -> Self { + Self { + meta: Mutex::new(()), + repos: (0..REPO_SHARDS).map(|_| Mutex::new(())).collect(), + } + } +} + +impl CobLocks { + pub fn meta(&self) -> MutexGuard<'_, ()> { + self.meta + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + pub fn repo(&self, repo: &RepoDid) -> MutexGuard<'_, ()> { + let mut hasher = DefaultHasher::new(); + repo.as_str().hash(&mut hasher); + let shard = (hasher.finish() as usize) % self.repos.len(); + self.repos[shard] + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } +} diff --git a/knot2/crates/knot-xrpc/src/members.rs b/knot2/crates/knot-xrpc/src/members.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/members.rs @@ -0,0 +1,130 @@ +use std::sync::Arc; + +use axum::body::Bytes; +use axum::extract::State; +use axum::response::Response; +use http::HeaderMap; +use serde::Deserialize; + +use knot_acl::{KnotAcl, can_admin_knot}; +use knot_cob::{CobHome, CobStore}; +use knot_cobs::{Grant, MembersChange, MembersCob, Removal}; +use knot_events::KnotMemberUpdate; +use knot_git::Repo; +use knot_index::Resolved; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::AccountDid; + +use crate::cob::grant_set_apply; +use crate::error::XrpcError; +use crate::{XrpcState, decode, ok_empty, run_blocking}; + +pub(crate) const ADD_ROUTE: &str = "/xrpc/sh.tangled.knot.addMember"; +pub(crate) const REMOVE_ROUTE: &str = "/xrpc/sh.tangled.knot.removeMember"; + +#[derive(Deserialize)] +struct SubjectInput { + subject: AccountDid, +} + +pub(crate) async fn add_member( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + body: Bytes, +) -> Result { + let actor = state.authenticate(&headers, &method).await?; + let acl = KnotAcl::new(&state.admins, state.admission, &state.index); + if !can_admin_knot(&acl, &actor).is_allowed() { + return Err(XrpcError::forbidden("only knot admin may add members")); + } + + let SubjectInput { subject } = decode(&body)?; + if state.admins.contains(&subject) + || matches!(state.index.is_member(&subject), Resolved::Ready(true)) + { + return Ok(ok_empty()); + } + + let now = state.now(); + let event_subject = subject.clone(); + let grant = Grant { + subject, + added_by: actor, + created_at: now, + }; + let signer = state.secrets.signer(&state.knot_did)?; + let meta_path = state.meta_path.clone(); + let index = Arc::clone(&state.index); + let cob_locks = Arc::clone(&state.cob_locks); + let events = Arc::clone(&state.events); + let home = CobHome::from(&state.knot_did); + run_blocking(move || { + let _guard = cob_locks.meta(); + let meta = Repo::open(&meta_path)?; + let changed = grant_set_apply::( + &CobStore::new(&meta), + &home, + MembersChange::Add(grant), + &signer, + now, + true, + )?; + index.refresh_members()?; + if changed { + events.publish(&KnotMemberUpdate::added(event_subject)); + } + Ok(()) + }) + .await?; + + Ok(ok_empty()) +} + +pub(crate) async fn remove_member( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + body: Bytes, +) -> Result { + let actor = state.authenticate(&headers, &method).await?; + let acl = KnotAcl::new(&state.admins, state.admission, &state.index); + if !can_admin_knot(&acl, &actor).is_allowed() { + return Err(XrpcError::forbidden("only knot admin may remove members")); + } + + let SubjectInput { subject } = decode(&body)?; + if matches!(state.index.is_member(&subject), Resolved::Ready(false)) { + return Ok(ok_empty()); + } + + let now = state.now(); + let event_subject = subject.clone(); + let removal = Removal { subject }; + let signer = state.secrets.signer(&state.knot_did)?; + let meta_path = state.meta_path.clone(); + let index = Arc::clone(&state.index); + let cob_locks = Arc::clone(&state.cob_locks); + let events = Arc::clone(&state.events); + let home = CobHome::from(&state.knot_did); + run_blocking(move || { + let _guard = cob_locks.meta(); + let meta = Repo::open(&meta_path)?; + let changed = grant_set_apply::( + &CobStore::new(&meta), + &home, + MembersChange::Remove(removal), + &signer, + now, + false, + )?; + index.refresh_members()?; + if changed { + events.publish(&KnotMemberUpdate::removed(event_subject)); + } + Ok(()) + }) + .await?; + + Ok(ok_empty()) +} diff --git a/knot2/crates/knot-xrpc/src/merge.rs b/knot2/crates/knot-xrpc/src/merge.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/merge.rs @@ -0,0 +1,533 @@ +use std::sync::Arc; + +use axum::Json; +use axum::body::Bytes; +use axum::extract::State; +use axum::response::{IntoResponse, Response}; +use http::{HeaderMap, StatusCode}; +use serde::{Deserialize, Serialize}; + +use knot_events::Reservation; +use knot_git::{ + ApplyError, ApplyOutcome, Conflict, Identity, NewCommit, ParsedFile, PatchApplier, + PatchParseError, RefUpdate, Repo, StagedChange, Staging, is_format_patch, + parse_mailbox_bounded, parse_patch_bounded, +}; +use knot_index::Resolved; +use knot_postreceive::{Actor, Ci}; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{ + AuthorName, BranchName, Email, Oid, OwnerDid, RefName, RepoDid, RepoRkey, UnixSeconds, +}; + +use crate::body::{CommitBody, CommitMessage, Patch, RepoNameArg}; +use crate::error::XrpcError; +use crate::reads::{open, repo_not_found, warming}; +use crate::{XrpcState, decode, ok_empty, run_blocking}; + +pub(crate) const MERGE_ROUTE: &str = "/xrpc/sh.tangled.repo.merge"; +pub(crate) const MERGE_CHECK_ROUTE: &str = "/xrpc/sh.tangled.repo.mergeCheck"; +const MERGE_RETRIES: u32 = 3; +const CONFLICT_MESSAGE: &str = "patch cannot be applied cleanly"; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Committer { + pub name: AuthorName, + pub email: Email, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct MergeInput { + did: OwnerDid, + name: RepoNameArg, + patch: Patch, + branch: BranchName, + author_name: Option, + author_email: Option, + commit_message: Option, + commit_body: Option, +} + +#[derive(Deserialize)] +struct MergeCheckInput { + did: OwnerDid, + name: RepoNameArg, + patch: Patch, + branch: BranchName, +} + +#[derive(Serialize)] +struct ConflictWire { + filename: String, + reason: String, +} + +#[derive(Serialize)] +struct MergeCheckOutput { + is_conflicted: bool, + #[serde(skip_serializing_if = "Option::is_none")] + conflicts: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + message: Option, + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, +} + +impl MergeCheckOutput { + fn clean() -> Self { + Self { + is_conflicted: false, + conflicts: None, + message: None, + error: None, + } + } + + fn conflicted(conflicts: Vec) -> Self { + Self { + is_conflicted: true, + conflicts: Some( + conflicts + .into_iter() + .map(|conflict| ConflictWire { + filename: conflict.path, + reason: conflict.reason.as_str().to_string(), + }) + .collect(), + ), + message: Some(CONFLICT_MESSAGE.to_string()), + error: None, + } + } + + fn broken(error: String) -> Self { + Self { + is_conflicted: true, + conflicts: None, + message: None, + error: Some(error), + } + } +} + +struct CommitSpec { + files: Vec, + author: Option, + message: String, + change_id: Option, +} + +struct MailAuthor { + name: AuthorName, + email: Email, + date: String, +} + +fn parse_specs( + patch: &str, + message: String, + author: Option, + max_bytes: u64, +) -> Result, PatchParseError> { + match is_format_patch(patch) { + true => Ok(parse_mailbox_bounded(patch, max_bytes)? + .into_iter() + .map(|mail| CommitSpec { + message: mail.commit_message(), + author: Some(MailAuthor { + name: mail.author_name, + email: mail.author_email, + date: mail.date, + }), + change_id: mail.change_id, + files: mail.files, + }) + .collect()), + false => Ok(vec![CommitSpec { + files: parse_patch_bounded(patch, max_bytes)?, + author, + message, + change_id: None, + }]), + } +} + +pub(crate) fn resolve_by_name( + state: &XrpcState, + owner: &OwnerDid, + name: &str, +) -> Result { + let rkey = RepoRkey::new(name).map_err(|_| repo_not_found())?; + match state.index.resolve_repo(owner, &rkey) { + Resolved::Ready(found) => found.ok_or_else(repo_not_found), + Resolved::Warming => Err(warming()), + } +} + +fn branch_tip(repo: &Repo, refname: &RefName) -> Result { + repo.find_ref(refname)? + .ok_or_else(|| XrpcError::invalid_request("no such branch to merge into")) +} + +fn mail_time(date: &str, now: UnixSeconds) -> (UnixSeconds, i32) { + let trimmed = date.trim(); + chrono::DateTime::parse_from_rfc2822(trimmed) + .or_else(|_| chrono::DateTime::parse_from_rfc3339(trimmed)) + .map(|parsed| { + ( + UnixSeconds::new(parsed.timestamp()), + parsed.offset().local_minus_utc(), + ) + }) + .unwrap_or((now, 0)) +} + +fn spec_identities( + spec: &CommitSpec, + fallback: &Identity, + now: UnixSeconds, +) -> (Identity, Vec<(String, Vec)>) { + let author = match &spec.author { + Some(mail) => { + let (time, offset_seconds) = mail_time(&mail.date, now); + Identity { + name: mail.name.clone(), + email: mail.email.clone(), + time, + offset_seconds, + } + } + None => fallback.clone(), + }; + let extra_headers = spec + .change_id + .iter() + .map(|change_id| { + ( + "change-id".to_string(), + change_id.as_str().as_bytes().to_vec(), + ) + }) + .collect(); + (author, extra_headers) +} + +enum StageStop { + Conflict(Vec), + Apply(ApplyError), +} + +fn stage_all( + repo: &Repo, + tip: Oid, + specs: &[CommitSpec], +) -> Result>, Vec>, ApplyError> { + let mut applier = PatchApplier::new(repo, tip); + let staged = specs.iter().try_fold(Vec::new(), |mut clean, spec| { + match applier.step(&spec.files) { + Ok(ApplyOutcome::Clean(staged)) => { + clean.push(staged); + Ok(clean) + } + Ok(ApplyOutcome::Conflicted(conflicts)) => Err(StageStop::Conflict(conflicts)), + Err(error) => Err(StageStop::Apply(error)), + } + }); + match staged { + Ok(clean) => Ok(Ok(clean)), + Err(StageStop::Conflict(conflicts)) => Ok(Err(conflicts)), + Err(StageStop::Apply(error)) => Err(error), + } +} + +enum MergeAttempt { + Done { + old: Oid, + new: Oid, + reservation: Reservation, + }, + Conflicted(Vec), + Raced, +} + +enum Merged { + Done { + old: Oid, + new: Oid, + reservation: Reservation, + }, + Conflicted(Vec), +} + +fn attempt_merge( + repo: &Repo, + refname: &RefName, + specs: &[CommitSpec], + committer: &Committer, + now: UnixSeconds, + reserve: &dyn Fn() -> Reservation, +) -> Result { + if specs.iter().any(|spec| spec.message.trim().is_empty()) { + return Err(XrpcError::invalid_request("commit message is required")); + } + let tip = branch_tip(repo, refname)?; + let staged = match stage_all(repo, tip, specs).map_err(XrpcError::from)? { + Ok(staged) => staged, + Err(conflicts) => return Ok(MergeAttempt::Conflicted(conflicts)), + }; + let committer_identity = Identity { + name: committer.name.clone(), + email: committer.email.clone(), + time: now, + offset_seconds: 0, + }; + let staging = Staging::new(repo).map_err(XrpcError::from)?; + let work = staging.repo(); + let base_tree = work.find_commit(tip).map_err(XrpcError::from)?.tree; + let new_tip = specs.iter().zip(staged).try_fold( + (base_tree, tip), + |(tree, parent), (spec, staged)| -> Result<(Oid, Oid), XrpcError> { + let next_tree = work + .write_staged_tree(tree, &staged) + .map_err(XrpcError::from)?; + let (author, extra_headers) = spec_identities(spec, &committer_identity, now); + let commit = work + .write_commit(&NewCommit { + tree: next_tree, + parents: vec![parent], + author, + committer: committer_identity.clone(), + message: spec.message.clone(), + extra_headers, + }) + .map_err(XrpcError::from)?; + Ok((next_tree, commit)) + }, + )?; + match repo.find_ref(refname).map_err(XrpcError::from)? { + Some(current) if current == tip => {} + _ => return Ok(MergeAttempt::Raced), + } + staging.migrate_into(repo).map_err(XrpcError::from)?; + match repo.update_ref_sealed( + &RefUpdate::Update { + name: refname.clone(), + old: tip, + new: new_tip.1, + }, + reserve, + ) { + Ok(reservation) => Ok(MergeAttempt::Done { + old: tip, + new: new_tip.1, + reservation, + }), + Err(error) => match repo.find_ref(refname) { + Ok(Some(current)) if current != tip => Ok(MergeAttempt::Raced), + _ => Err(error.into()), + }, + } +} + +fn merge_with_retry( + repo: &Repo, + refname: &RefName, + specs: &[CommitSpec], + committer: &Committer, + now: UnixSeconds, + attempts: u32, + reserve: &dyn Fn() -> Reservation, +) -> Result { + match attempt_merge(repo, refname, specs, committer, now, reserve)? { + MergeAttempt::Done { + old, + new, + reservation, + } => Ok(Merged::Done { + old, + new, + reservation, + }), + MergeAttempt::Conflicted(conflicts) => Ok(Merged::Conflicted(conflicts)), + MergeAttempt::Raced if attempts > 1 => { + merge_with_retry(repo, refname, specs, committer, now, attempts - 1, reserve) + } + MergeAttempt::Raced => Err(XrpcError::conflict("branch moved during the merge, retry")), + } +} + +fn merge_conflict(conflicts: &[Conflict]) -> XrpcError { + let detail = conflicts + .first() + .map(|conflict| { + format!( + "{CONFLICT_MESSAGE}: {} {}", + conflict.path, + conflict.reason.as_str() + ) + }) + .unwrap_or_else(|| CONFLICT_MESSAGE.to_string()); + XrpcError::named( + StatusCode::CONFLICT, + "MergeConflict", + format!("Merge failed due to conflicts: {detail}"), + ) +} + +pub(crate) async fn merge( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + body: Bytes, +) -> Result { + let actor = state.authenticate(&headers, &method).await?; + let input: MergeInput = decode(&body)?; + let repo_did = resolve_by_name(&state, &input.did, input.name.as_str())?; + crate::authorize_push( + &state, + &actor, + &repo_did, + "only repository owner or a collaborator may merge", + ) + .await?; + let refname = input.branch.head_ref(); + let committer = state.committer.clone(); + let now = state.now(); + let layout = state.layout.clone(); + let max_patch_bytes = state.byte_limits.patch_decompressed.get(); + let event_repo = repo_did.clone(); + let event_ref = refname.clone(); + + let events = Arc::clone(&state.events); + let outcome = run_blocking(move || { + let specs = parse_specs( + input.patch.as_str(), + unified_message(&input), + unified_author(&input), + max_patch_bytes, + ) + .map_err(|error| XrpcError::invalid_request(error.to_string()))?; + let repo = open(&layout, &repo_did)?; + let reserve = || events.reserve(); + merge_with_retry( + &repo, + &refname, + &specs, + &committer, + now, + MERGE_RETRIES, + &reserve, + ) + }) + .await?; + + match outcome { + Merged::Conflicted(conflicts) => Err(merge_conflict(&conflicts)), + Merged::Done { + old, + new, + reservation, + } => { + let owner = crate::current_owner(&state, &event_repo); + let layout = state.layout.clone(); + let languages_push_budget = state.budgets.languages_push; + let catalog = Arc::clone(&state.catalog); + let repo_label = event_repo.as_str().to_string(); + if let Err(error) = run_blocking(move || -> Result<(), XrpcError> { + let repo = open(&layout, &event_repo)?; + let update = RefUpdate::Update { + name: event_ref, + old, + new, + }; + let post_actor = Actor { + committer: actor, + owner, + repo: event_repo, + }; + knot_postreceive::post_receive( + &repo, + &post_actor, + vec![(update, reservation)], + &Ci::Skip, + &knot_types::PushOptions::default(), + None, + languages_push_budget, + &catalog.push, + ); + Ok(()) + }) + .await + { + tracing::warn!(repo = %repo_label, %error, "post-receive after merge failed"); + } + Ok(ok_empty()) + } + } +} + +fn unified_message(input: &MergeInput) -> String { + let message = input + .commit_message + .as_ref() + .map(|message| message.as_str().to_string()) + .unwrap_or_default(); + match input + .commit_body + .as_ref() + .map(|body| body.as_str()) + .filter(|body| !body.is_empty()) + { + Some(body) => format!("{message}\n\n{body}"), + None => message, + } +} + +fn unified_author(input: &MergeInput) -> Option { + match (input.author_name.as_ref(), input.author_email.as_ref()) { + (Some(name), Some(email)) if !name.as_str().is_empty() && !email.as_str().is_empty() => { + Some(MailAuthor { + name: name.clone(), + email: email.clone(), + date: String::new(), + }) + } + _ => None, + } +} + +pub(crate) async fn merge_check( + State(state): State>>, + body: Bytes, +) -> Result { + let input: MergeCheckInput = decode(&body)?; + let repo_did = resolve_by_name(&state, &input.did, input.name.as_str())?; + let refname = input.branch.head_ref(); + let layout = state.layout.clone(); + let max_patch_bytes = state.byte_limits.patch_decompressed.get(); + + let output = run_blocking(move || { + let specs = match parse_specs(input.patch.as_str(), String::new(), None, max_patch_bytes) { + Ok(specs) => specs, + Err(error) => return Ok(MergeCheckOutput::broken(error.to_string())), + }; + let repo = open(&layout, &repo_did)?; + let tip = branch_tip(&repo, &refname)?; + match stage_all(&repo, tip, &specs) { + Ok(Ok(_)) => Ok(MergeCheckOutput::clean()), + Ok(Err(conflicts)) => Ok(MergeCheckOutput::conflicted(conflicts)), + Err(ApplyError::TooLarge) => { + Ok(MergeCheckOutput::broken(ApplyError::TooLarge.to_string())) + } + Err(ApplyError::Git(error)) => Err(error.into()), + } + }) + .await?; + + Ok(check_response(output)) +} + +fn check_response(output: MergeCheckOutput) -> Response { + (StatusCode::OK, Json(output)).into_response() +} diff --git a/knot2/crates/knot-xrpc/src/patchtext.rs b/knot2/crates/knot-xrpc/src/patchtext.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/patchtext.rs @@ -0,0 +1,215 @@ +use knot_git::{Commit, FilePatch, Hunk, LineCount, LineNumber, LineOp, PatchStatus}; + +use crate::wire::{entry_mode_octal, fold_subject, message_body, rfc2822}; + +const GRAPH_WIDTH: usize = 60; + +fn span(start: LineNumber, lines: LineCount) -> String { + match lines.get() { + 1 => format!("{}", start.get()), + _ => format!("{},{}", start.get(), lines.get()), + } +} + +fn render_hunk(out: &mut String, hunk: &Hunk) { + out.push_str(&format!( + "@@ -{} +{} @@\n", + span(hunk.old_start, hunk.old_lines), + span(hunk.new_start, hunk.new_lines) + )); + hunk.lines.iter().for_each(|line| { + out.push(match line.op { + LineOp::Context => ' ', + LineOp::Delete => '-', + LineOp::Add => '+', + }); + out.push_str(&String::from_utf8_lossy(&line.text)); + if !line.text.ends_with(b"\n") { + out.push_str("\n\\ No newline at end of file\n"); + } + }); +} + +fn render_file(out: &mut String, patch: &FilePatch) { + let (a, b) = (&patch.path, &patch.path); + out.push_str(&format!("diff --git a/{a} b/{b}\n")); + match patch.status { + PatchStatus::Added => { + let mode = patch.new_kind.map(entry_mode_octal).unwrap_or_default(); + out.push_str(&format!("new file mode {mode}\n")); + out.push_str(&format!( + "index {}..{}\n", + patch.old_oid.to_hex(), + patch.new_oid.to_hex() + )); + } + PatchStatus::Deleted => { + let mode = patch.old_kind.map(entry_mode_octal).unwrap_or_default(); + out.push_str(&format!("deleted file mode {mode}\n")); + out.push_str(&format!( + "index {}..{}\n", + patch.old_oid.to_hex(), + patch.new_oid.to_hex() + )); + } + PatchStatus::Modified => { + if patch.old_kind == patch.new_kind { + let mode = patch.old_kind.map(entry_mode_octal).unwrap_or_default(); + out.push_str(&format!( + "index {}..{} {mode}\n", + patch.old_oid.to_hex(), + patch.new_oid.to_hex() + )); + } else { + let old = patch.old_kind.map(entry_mode_octal).unwrap_or_default(); + let new = patch.new_kind.map(entry_mode_octal).unwrap_or_default(); + out.push_str(&format!("old mode {old}\nnew mode {new}\n")); + out.push_str(&format!( + "index {}..{}\n", + patch.old_oid.to_hex(), + patch.new_oid.to_hex() + )); + } + } + } + let old_label = match patch.status { + PatchStatus::Added => "/dev/null".to_string(), + _ => format!("a/{a}"), + }; + let new_label = match patch.status { + PatchStatus::Deleted => "/dev/null".to_string(), + _ => format!("b/{b}"), + }; + if patch.is_binary { + out.push_str(&format!( + "Binary files {old_label} and {new_label} differ\n" + )); + return; + } + if patch.hunks.is_empty() { + return; + } + out.push_str(&format!("--- {old_label}\n+++ {new_label}\n")); + patch.hunks.iter().for_each(|hunk| render_hunk(out, hunk)); +} + +pub(crate) fn render_patches(patches: &[FilePatch]) -> String { + patches.iter().fold(String::new(), |mut out, patch| { + render_file(&mut out, patch); + out + }) +} + +fn stat_counts(patch: &FilePatch) -> (usize, usize) { + patch.hunks.iter().fold((0, 0), |(added, deleted), hunk| { + ( + added + hunk.added().get() as usize, + deleted + hunk.deleted().get() as usize, + ) + }) +} + +fn graph(added: usize, deleted: usize) -> String { + let total = added + deleted; + let (added, deleted) = if total > GRAPH_WIDTH { + (added * GRAPH_WIDTH / total, deleted * GRAPH_WIDTH / total) + } else { + (added, deleted) + }; + format!("{}{}", "+".repeat(added), "-".repeat(deleted)) +} + +fn diffstat(patches: &[FilePatch]) -> String { + let width = patches + .iter() + .map(|patch| patch.path.as_str().len()) + .max() + .unwrap_or(0); + let rows: String = patches + .iter() + .map(|patch| { + if patch.is_binary { + format!(" {: 0 { + summary.push_str(&format!( + ", {added} insertion{}(+)", + if added == 1 { "" } else { "s" } + )); + } + if deleted > 0 { + summary.push_str(&format!( + ", {deleted} deletion{}(-)", + if deleted == 1 { "" } else { "s" } + )); + } + summary.push('\n'); + let created: String = patches + .iter() + .filter(|patch| patch.status == PatchStatus::Added) + .map(|patch| { + format!( + " create mode {} {}\n", + patch.new_kind.map(entry_mode_octal).unwrap_or_default(), + patch.path + ) + }) + .collect(); + let deleted_rows: String = patches + .iter() + .filter(|patch| patch.status == PatchStatus::Deleted) + .map(|patch| { + format!( + " delete mode {} {}\n", + patch.old_kind.map(entry_mode_octal).unwrap_or_default(), + patch.path + ) + }) + .collect(); + format!("{rows}{summary}{created}{deleted_rows}") +} + +pub(crate) fn render_format_patch(commit: &Commit, patches: &[FilePatch]) -> String { + let subject = fold_subject(&commit.message); + let body = message_body(&commit.message); + let mut out = format!("From {} Mon Sep 17 00:00:00 2001\n", commit.id.to_hex()); + out.push_str(&format!( + "From: {} <{}>\n", + commit.author.name, commit.author.email + )); + out.push_str(&format!( + "Date: {}\n", + rfc2822(commit.author.time.get(), commit.author.offset_seconds) + )); + out.push_str(&format!("Subject: [PATCH] {subject}\n")); + if let Some(change_id) = commit.change_id() { + out.push_str(&format!("Change-Id: {change_id}\n")); + } + out.push('\n'); + if !body.is_empty() { + out.push_str(&body); + out.push('\n'); + } + out.push_str("---\n"); + out.push_str(&diffstat(patches)); + out.push('\n'); + out.push_str(&render_patches(patches)); + out.push_str("-- \nknot\n\n"); + out +} diff --git a/knot2/crates/knot-xrpc/src/query.rs b/knot2/crates/knot-xrpc/src/query.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/query.rs @@ -0,0 +1,273 @@ +use axum::extract::{FromRequestParts, Query}; +use http::request::Parts; +use knot_types::{OwnerDid, RepoDid, RepoPath, RepoRkey}; +use serde::de::{self, Deserialize, DeserializeOwned, Deserializer}; + +use crate::error::XrpcError; + +pub(crate) struct ValidatedQuery(pub(crate) T); + +impl FromRequestParts for ValidatedQuery +where + T: DeserializeOwned, + S: Send + Sync, +{ + type Rejection = XrpcError; + + async fn from_request_parts(parts: &mut Parts, _state: &S) -> Result { + Query::::try_from_uri(&parts.uri) + .map(|query| ValidatedQuery(query.0)) + .map_err(|rejection| XrpcError::invalid_request(rejection.body_text())) + } +} + +// Each route's default and its limit are included in the type, +// such that a limit that is ok on one endpoint +// can't be spent on another one with a lower roof. +#[derive(Clone, Copy)] +pub(crate) struct Limit(usize); + +impl Limit { + pub(crate) fn get(self) -> usize { + self.0 + } +} + +impl Default for Limit { + fn default() -> Self { + Limit(DEFAULT) + } +} + +impl<'de, const DEFAULT: usize, const MAX: usize> Deserialize<'de> for Limit { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + if raw.is_empty() { + return Ok(Limit(DEFAULT)); + } + let value = raw + .parse::() + .map_err(|_| de::Error::custom("limit must be an integer"))?; + Ok(Limit(usize::try_from(value).unwrap_or(0).min(MAX).max(1))) + } +} + +knot_types::scalar_newtype! { + #[derive(Default)] + pub(crate) struct Offset(usize); +} + +impl<'de> Deserialize<'de> for Offset { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + if raw.is_empty() { + return Ok(Offset::new(0)); + } + raw.parse::() + .map(Offset::new) + .map_err(|_| de::Error::custom("cursor must be an integer")) + } +} + +knot_types::scalar_newtype! { + pub(crate) struct Total(usize); +} + +pub(crate) fn next_cursor( + offset: Offset, + limit: Limit, + total: Total, +) -> Option { + offset + .get() + .checked_add(limit.get()) + .filter(|&end| end < total.get()) + .map(|end| end.to_string()) +} + +#[derive(Clone, Copy, Default, PartialEq, Eq)] +pub(crate) enum Order { + #[default] + Desc, + Asc, +} + +impl Order { + pub(crate) fn descending(self) -> bool { + matches!(self, Order::Desc) + } +} + +impl<'de> Deserialize<'de> for Order { + fn deserialize>(deserializer: D) -> Result { + match String::deserialize(deserializer)?.as_str() { + "" | "desc" => Ok(Order::Desc), + "asc" => Ok(Order::Asc), + _ => Err(de::Error::custom("order must be 'asc' or 'desc'")), + } + } +} + +pub(crate) enum RepoArg { + Did(RepoDid), + OwnerRkey { owner: OwnerDid, rkey: RepoRkey }, +} + +impl RepoArg { + pub(crate) fn basename(&self) -> &str { + match self { + RepoArg::Did(did) => did.as_str(), + RepoArg::OwnerRkey { rkey, .. } => rkey.as_str(), + } + } + + pub(crate) fn to_param(&self) -> String { + match self { + RepoArg::Did(did) => did.as_str().to_string(), + RepoArg::OwnerRkey { owner, rkey } => format!("{}/{}", owner.as_str(), rkey.as_str()), + } + } +} + +impl<'de> Deserialize<'de> for RepoArg { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + if !raw.starts_with("did:") { + return Err(de::Error::custom( + "missing or invalid repo parameter, expected repo DID", + )); + } + Ok(match raw.split_once('/') { + None => RepoArg::Did(RepoDid::new(raw).map_err(de::Error::custom)?), + Some((owner, rkey)) => RepoArg::OwnerRkey { + owner: OwnerDid::new(owner).map_err(de::Error::custom)?, + rkey: RepoRkey::new(rkey).map_err(de::Error::custom)?, + }, + }) + } +} + +const MAX_REVSPEC_BYTES: usize = 4096; + +#[derive(Clone, Default)] +pub(crate) struct Revspec(String); + +impl Revspec { + pub(crate) fn as_str(&self) -> &str { + &self.0 + } +} + +impl<'de> Deserialize<'de> for Revspec { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + match raw.len() <= MAX_REVSPEC_BYTES && !raw.chars().any(char::is_control) { + true => Ok(Self(raw)), + false => Err(de::Error::custom("invalid revision")), + } + } +} + +#[derive(Default)] +pub(crate) struct BranchArg(Option); + +impl BranchArg { + pub(crate) fn get(&self) -> Option<&knot_types::BranchName> { + self.0.as_ref() + } +} + +impl<'de> serde::Deserialize<'de> for BranchArg { + fn deserialize>(deserializer: D) -> Result { + match String::deserialize(deserializer)? { + raw if raw.is_empty() => Ok(Self(None)), + raw => knot_types::BranchName::new(raw) + .map(|name| Self(Some(name))) + .map_err(de::Error::custom), + } + } +} + +#[derive(Default)] +pub(crate) struct TagArg(Option); + +impl TagArg { + pub(crate) fn get(&self) -> Option<&knot_types::TagName> { + self.0.as_ref() + } +} + +impl<'de> serde::Deserialize<'de> for TagArg { + fn deserialize>(deserializer: D) -> Result { + match String::deserialize(deserializer)? { + raw if raw.is_empty() => Ok(Self(None)), + raw => { + let short = raw.strip_prefix("refs/tags/").unwrap_or(&raw); + knot_types::TagName::new(short) + .map(|name| Self(Some(name))) + .map_err(de::Error::custom) + } + } + } +} + +#[derive(Default)] +pub(crate) enum TreePath { + #[default] + Root, + At(RepoPath), + Outside(String), +} + +impl TreePath { + pub(crate) fn as_str(&self) -> &str { + match self { + TreePath::Root => "", + TreePath::At(path) => path.as_str(), + TreePath::Outside(raw) => raw, + } + } + + pub(crate) fn dir(&self) -> Option> { + match self { + TreePath::Root => Some(None), + TreePath::At(path) => Some(Some(path)), + TreePath::Outside(_) => None, + } + } + + pub(crate) fn file(&self) -> Option<&RepoPath> { + match self { + TreePath::At(path) => Some(path), + TreePath::Root | TreePath::Outside(_) => None, + } + } +} + +impl<'de> Deserialize<'de> for TreePath { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + Ok(match raw.is_empty() { + true => TreePath::Root, + false => match RepoPath::new(&raw) { + Ok(path) => TreePath::At(path), + Err(_) => TreePath::Outside(raw), + }, + }) + } +} + +#[derive(Default)] +pub(crate) struct RawFlag(bool); + +impl RawFlag { + pub(crate) fn requested(&self) -> bool { + self.0 + } +} + +impl<'de> Deserialize<'de> for RawFlag { + fn deserialize>(deserializer: D) -> Result { + Ok(RawFlag(String::deserialize(deserializer)? == "true")) + } +} diff --git a/knot2/crates/knot-xrpc/src/reads.rs b/knot2/crates/knot-xrpc/src/reads.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/reads.rs @@ -0,0 +1,1715 @@ +use std::collections::BTreeMap; +use std::io::{Seek, SeekFrom}; +use std::sync::Arc; + +use axum::body::Body; +use axum::extract::{Query, Request, State}; +use axum::response::{IntoResponse, Response}; +use http::{HeaderMap, HeaderValue, StatusCode, header}; +use serde::de::{self, Deserializer}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use tower::ServiceExt; +use tower_http::services::ServeFile; + +use knot_git::{ + ArchiveFormat, Commit, CommitRange, EntryKind, Layout, LogLimit, LogSkip, Repo, SizedEntry, + is_public_ref, screens_reserved, +}; +use knot_index::{Coverage, Resolved}; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{AuthorName, Email, Oid, OwnerDid, RepoDid, RepoPath, RepoRkey}; + +use crate::error::XrpcError; +use crate::patchtext::{render_format_patch, render_patches}; +use crate::query::{ + BranchArg, Limit, Offset, Order, RawFlag, RepoArg, Revspec, TagArg, Total, TreePath, + ValidatedQuery, next_cursor, +}; +use crate::wire::{ + BranchWire, CommitWire, FileWire, FormatPatchWire, PatchIdentityWire, TagWire, ZERO_TIME, + fold_subject, message_body, nice_diff, normalize_message_section, rfc2822, rfc3339, +}; +use crate::{XrpcState, run_blocking, sniff}; + +pub(crate) const TREE_ROUTE: &str = "/xrpc/sh.tangled.repo.tree"; +pub(crate) const LOG_ROUTE: &str = "/xrpc/sh.tangled.repo.log"; +pub(crate) const BRANCHES_ROUTE: &str = "/xrpc/sh.tangled.repo.branches"; +pub(crate) const BRANCH_ROUTE: &str = "/xrpc/sh.tangled.repo.branch"; +pub(crate) const TAGS_ROUTE: &str = "/xrpc/sh.tangled.repo.tags"; +pub(crate) const TAG_ROUTE: &str = "/xrpc/sh.tangled.repo.tag"; +pub(crate) const BLOB_ROUTE: &str = "/xrpc/sh.tangled.repo.blob"; +pub(crate) const DIFF_ROUTE: &str = "/xrpc/sh.tangled.repo.diff"; +pub(crate) const COMPARE_ROUTE: &str = "/xrpc/sh.tangled.repo.compare"; +pub(crate) const ARCHIVE_ROUTE: &str = "/xrpc/sh.tangled.repo.archive"; +pub(crate) const LANGUAGES_ROUTE: &str = "/xrpc/sh.tangled.repo.languages"; +pub(crate) const GET_DEFAULT_BRANCH_ROUTE: &str = "/xrpc/sh.tangled.repo.getDefaultBranch"; +pub(crate) const DESCRIBE_REPO_ROUTE: &str = "/xrpc/sh.tangled.repo.describeRepo"; +pub(crate) const LIST_REFS_ROUTE: &str = "/xrpc/sh.tangled.git.listRefs"; +pub(crate) const LIST_REPOS_ROUTE: &str = "/xrpc/sh.tangled.sync.listRepos"; + +const DEFAULT_PAGE: usize = 50; +const MAX_PAGE: usize = 100; +const LIST_REFS_DEFAULT: usize = 100; +const LIST_REFS_MAX: usize = 1000; +const LIST_REPOS_DEFAULT: usize = 50; +const LIST_REPOS_MAX: usize = 1000; +const MAX_BLOB_BYTES: u64 = 25 * 1024 * 1024; +const MAX_COMPARE_COMMITS: usize = 500; +const ARCHIVE_CAP_MESSAGE: &str = "archive exceeds configured maximum size"; +const RAW_CSP: &str = "default-src 'none'; style-src 'unsafe-inline'; sandbox"; + +pub(crate) fn repo_not_found() -> XrpcError { + XrpcError::named( + StatusCode::NOT_FOUND, + "RepoNotFound", + "repository not found on this knot", + ) +} + +fn ref_not_found() -> XrpcError { + XrpcError::named( + StatusCode::NOT_FOUND, + "RefNotFound", + "git reference not found", + ) +} + +fn blob_too_large() -> XrpcError { + XrpcError::named( + StatusCode::PAYLOAD_TOO_LARGE, + "BlobTooLarge", + "file is too large to serve", + ) +} + +fn blob_serving_limit(raw: bool, response_limit: usize) -> u64 { + match raw { + true => MAX_BLOB_BYTES, + false => MAX_BLOB_BYTES.min(response_limit as u64 / 4 * 3), + } +} + +fn readme_serving_limit(response_limit: usize) -> u64 { + response_limit as u64 / 8 +} + +fn names_reserved(refspec: &str) -> bool { + screens_reserved(refspec) || screens_reserved(&format!("refs/{refspec}")) +} + +pub(crate) fn warming() -> XrpcError { + XrpcError::warming("registry projection is still warming") +} + +fn resolve_repo( + state: &XrpcState, + repo: &RepoArg, +) -> Result { + match repo { + RepoArg::Did(did) => match state.index.owner_of(did) { + Resolved::Ready(Some(_)) => Ok(did.clone()), + Resolved::Ready(None) => Err(repo_not_found()), + Resolved::Warming => Err(warming()), + }, + RepoArg::OwnerRkey { owner, rkey } => match state.index.resolve_repo(owner, rkey) { + Resolved::Ready(Some(did)) => Ok(did), + Resolved::Ready(None) => Err(repo_not_found()), + Resolved::Warming => Err(warming()), + }, + } +} + +pub(crate) fn open(layout: &Layout, did: &RepoDid) -> Result { + layout + .open(did) + .map_err(|error| XrpcError::internal(format!("cannot open repository: {error}"))) +} + +fn commit_for(repo: &Repo, refspec: &Revspec) -> Result { + let refspec = refspec.as_str(); + if names_reserved(refspec) { + return Err(ref_not_found()); + } + let oid = match refspec.is_empty() { + true => repo.head().map(|head| head.target), + false => repo.resolve_revision(refspec), + } + .ok_or_else(ref_not_found)?; + let commit = repo.peel_to_commit(oid).map_err(|_| ref_not_found())?; + match repo.reachable_from_public(commit) { + Ok(true) => Ok(commit), + Ok(false) => Err(ref_not_found()), + Err(error) => Err(error.into()), + } +} + +struct LimitWriter { + buf: Vec, + limit: usize, +} + +impl std::io::Write for LimitWriter { + fn write(&mut self, data: &[u8]) -> std::io::Result { + if self.buf.len() + data.len() > self.limit { + return Err(std::io::Error::new( + std::io::ErrorKind::WriteZero, + "response exceeds configured maximum size", + )); + } + self.buf.extend_from_slice(data); + Ok(data.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +fn json(value: impl Serialize, limit: usize) -> Result { + let mut writer = LimitWriter { + buf: Vec::new(), + limit, + }; + match serde_json::to_writer(&mut writer, &value) { + Ok(()) => Ok(( + StatusCode::OK, + [(header::CONTENT_TYPE, "application/json")], + writer.buf, + ) + .into_response()), + Err(error) if error.is_io() => Err(XrpcError::request_too_large( + "response exceeds configured maximum size", + )), + Err(error) => Err(XrpcError::internal(format!( + "failed to serialize response: {error}" + ))), + } +} + +#[derive(Deserialize)] +pub(crate) struct TreeParams { + repo: RepoArg, + #[serde(rename = "ref", default)] + refspec: Revspec, + #[serde(default)] + path: TreePath, +} + +#[derive(Serialize)] +struct SignatureOut { + name: AuthorName, + email: Email, + when: String, +} + +#[derive(Serialize)] +struct LastCommitOut { + hash: Oid, + message: String, + when: String, + #[serde(skip_serializing_if = "Option::is_none")] + author: Option, +} + +#[derive(Serialize)] +struct TreeEntryOut { + name: String, + mode: String, + size: i64, + #[serde(skip_serializing_if = "Option::is_none")] + last_commit: Option, +} + +#[derive(Serialize)] +struct ReadmeOut { + filename: String, + contents: String, +} + +#[derive(Serialize)] +struct TreeOut { + #[serde(rename = "ref")] + refspec: String, + #[serde(skip_serializing_if = "Option::is_none")] + parent: Option, + #[serde(skip_serializing_if = "Option::is_none")] + dotdot: Option, + files: Vec, + #[serde(rename = "lastCommit", skip_serializing_if = "Option::is_none")] + last_commit: Option, + readme: ReadmeOut, +} + +fn is_readme(entry: &SizedEntry) -> bool { + let lower = entry.name.to_ascii_lowercase(); + entry.kind.is_file() + && (lower == "readme" + || lower + .strip_prefix("readme.") + .is_some_and(|extension| !extension.is_empty() && !extension.contains('.'))) +} + +fn readme_of( + repo: &Repo, + commit: Oid, + dir: Option<&RepoPath>, + entries: &[SizedEntry], + response_limit: usize, +) -> ReadmeOut { + entries + .iter() + .filter(|entry| is_readme(entry)) + .find_map(|entry| { + let path = match dir { + None => RepoPath::new(entry.name.as_str()).ok()?, + Some(dir) => RepoPath::new(format!("{dir}/{}", entry.name)).ok()?, + }; + let target = repo.entry_at(commit, &path).ok().flatten()?; + if repo.blob_size(target.oid).ok()? > readme_serving_limit(response_limit) { + return None; + } + let contents = repo.read_blob(target.oid).ok()?; + String::from_utf8(contents).ok().map(|contents| ReadmeOut { + filename: entry.name.clone(), + contents, + }) + }) + .unwrap_or(ReadmeOut { + filename: String::new(), + contents: String::new(), + }) +} + +pub(crate) async fn repo_tree( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + let did = resolve_repo(&state, ¶ms.repo)?; + let layout = state.layout.clone(); + let limit = state.byte_limits.response.get(); + let tree_deadline = state.budgets.tree_last_commit.get().deadline(); + run_blocking(move || { + let repo = open(&layout, &did)?; + let commit = commit_for(&repo, ¶ms.refspec)?; + let path_not_found = || { + XrpcError::named( + StatusCode::NOT_FOUND, + "PathNotFound", + "path not found in repository tree", + ) + }; + let dir = params.path.dir().ok_or_else(path_not_found)?; + let path = params.path.as_str(); + let entries = repo + .tree_entries_at(commit, dir)? + .ok_or_else(path_not_found)?; + let names: Vec = entries.iter().map(|entry| entry.name.clone()).collect(); + let attributed = repo + .last_commits(commit, dir, &names, tree_deadline) + .unwrap_or_default(); + let files: Vec = entries + .iter() + .map(|entry| TreeEntryOut { + name: entry.name.clone(), + mode: entry.kind.mode_octal().to_string(), + size: entry.size as i64, + last_commit: attributed.get(&entry.name).map(|last| LastCommitOut { + hash: last.id, + message: last.subject.clone(), + when: rfc3339(last.time.get(), 0), + author: None, + }), + }) + .collect(); + let newest = attributed.values().max_by_key(|last| (last.time, last.id)); + let last_commit = newest.map(|last| LastCommitOut { + hash: last.id, + message: last.subject.clone(), + when: rfc3339(last.time.get(), 0), + author: repo.find_commit(last.id).ok().map(|commit| SignatureOut { + name: commit.author.name, + email: commit.author.email, + when: String::new(), + }), + }); + let readme = readme_of(&repo, commit, dir, &entries, limit); + let parent = (!path.is_empty()).then(|| path.to_string()); + let dotdot = (!path.is_empty()) + .then(|| path.rsplit_once('/').map(|(parent, _)| parent.to_string())) + .flatten(); + json( + TreeOut { + refspec: params.refspec.as_str().to_string(), + parent, + dotdot, + files, + last_commit, + readme, + }, + limit, + ) + }) + .await +} + +#[derive(Deserialize)] +pub(crate) struct LogParams { + repo: RepoArg, + #[serde(rename = "ref", default)] + refspec: Revspec, + #[serde(default)] + path: TreePath, + #[serde(default)] + limit: Limit, + #[serde(default)] + cursor: Offset, +} + +#[derive(Serialize)] +struct LogOut { + #[serde(skip_serializing_if = "Vec::is_empty")] + commits: Vec, + #[serde(rename = "ref", skip_serializing_if = "String::is_empty")] + refspec: String, + #[serde(skip_serializing_if = "String::is_empty")] + description: String, + log: bool, + #[serde(skip_serializing_if = "is_zero")] + total: usize, + page: usize, + per_page: usize, +} + +fn is_zero(value: &usize) -> bool { + *value == 0 +} + +pub(crate) async fn repo_log( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + let did = resolve_repo(&state, ¶ms.repo)?; + let offset = params.cursor.get(); + let limit = params.limit.get(); + let layout = state.layout.clone(); + let response_limit = state.byte_limits.response.get(); + run_blocking(move || { + let repo = open(&layout, &did)?; + let start = commit_for(&repo, ¶ms.refspec)?; + let (commits, total) = + repo.log_window(start, LogSkip::new(offset), LogLimit::new(limit))?; + json( + LogOut { + commits: commits.iter().map(CommitWire::of).collect(), + refspec: params.refspec.as_str().to_string(), + description: params.path.as_str().to_string(), + log: true, + total, + page: (offset / limit) + 1, + per_page: limit, + }, + response_limit, + ) + }) + .await +} + +#[derive(Deserialize)] +pub(crate) struct BranchesParams { + repo: RepoArg, + #[serde(default)] + limit: Limit, + #[serde(default)] + cursor: Offset, +} + +#[derive(Serialize)] +struct BranchesOut { + #[serde(skip_serializing_if = "Vec::is_empty")] + branches: Vec, +} + +pub(crate) async fn repo_branches( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + let did = resolve_repo(&state, ¶ms.repo)?; + let offset = params.cursor.get(); + let limit = params.limit.get(); + let layout = state.layout.clone(); + let response_limit = state.byte_limits.response.get(); + run_blocking(move || { + let repo = open(&layout, &did)?; + let mut branches = repo.branch_list()?; + branches.sort_by(|a, b| { + b.tip + .created_at() + .cmp(&a.tip.created_at()) + .then_with(|| a.name.cmp(&b.name)) + }); + let default = repo + .default_branch() + .map(|name| name.as_str().trim_start_matches("refs/heads/").to_string()); + let absent = repo.object_format().null_oid(); + let window: Vec = branches + .iter() + .skip(offset) + .take(limit) + .map(|branch| { + BranchWire::of( + branch, + default.as_deref() == Some(branch.name.as_str()), + absent, + ) + }) + .rev() + .collect(); + json(BranchesOut { branches: window }, response_limit) + }) + .await +} + +#[derive(Deserialize)] +pub(crate) struct BranchParams { + repo: RepoArg, + #[serde(default)] + name: BranchArg, +} + +#[derive(Serialize)] +struct BranchOut { + name: String, + hash: String, + #[serde(rename = "shortHash")] + short_hash: String, + when: String, + #[serde(skip_serializing_if = "Option::is_none")] + message: Option, + author: SignatureOut, + #[serde(rename = "isDefault")] + is_default: bool, +} + +pub(crate) async fn repo_branch( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + let did = resolve_repo(&state, ¶ms.repo)?; + let Some(name) = params.name.get().cloned() else { + return Err(XrpcError::invalid_request("missing name parameter")); + }; + let layout = state.layout.clone(); + let limit = state.byte_limits.response.get(); + run_blocking(move || { + let repo = open(&layout, &did)?; + let branch_not_found = + || XrpcError::named(StatusCode::NOT_FOUND, "BranchNotFound", "branch not found"); + let target = repo + .find_ref(&name.head_ref()) + .ok() + .flatten() + .ok_or_else(branch_not_found)?; + let commit = repo.find_commit(target).map_err(|_| branch_not_found())?; + let default = repo + .default_branch() + .map(|name| name.as_str().trim_start_matches("refs/heads/").to_string()); + let hash = target.to_hex(); + json( + BranchOut { + name: name.to_string(), + short_hash: hash[..7].to_string(), + hash, + when: rfc3339(commit.author.time.get(), commit.author.offset_seconds), + message: (!commit.message.is_empty()).then(|| commit.message.clone()), + author: SignatureOut { + name: commit.author.name.clone(), + email: commit.author.email.clone(), + when: rfc3339(commit.author.time.get(), commit.author.offset_seconds), + }, + is_default: default.as_deref() == Some(name.as_str()), + }, + limit, + ) + }) + .await +} + +#[derive(Deserialize)] +pub(crate) struct TagsParams { + repo: RepoArg, + #[serde(default)] + limit: Limit, + #[serde(default)] + cursor: Offset, +} + +#[derive(Serialize)] +struct TagsOut { + #[serde(skip_serializing_if = "Vec::is_empty")] + tags: Vec, +} + +pub(crate) async fn repo_tags( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + let did = resolve_repo(&state, ¶ms.repo)?; + let offset = params.cursor.get(); + let limit = params.limit.get(); + let layout = state.layout.clone(); + let response_limit = state.byte_limits.response.get(); + run_blocking(move || { + let repo = open(&layout, &did)?; + let mut tags = repo.tag_list()?; + tags.sort_by(|a, b| { + b.created_at + .cmp(&a.created_at) + .then_with(|| a.name.cmp(&b.name)) + }); + let window: Vec = tags + .iter() + .skip(offset) + .take(limit) + .map(TagWire::of) + .collect(); + json(TagsOut { tags: window }, response_limit) + }) + .await +} + +#[derive(Deserialize)] +pub(crate) struct TagParams { + repo: RepoArg, + #[serde(default)] + tag: TagArg, +} + +#[derive(Serialize)] +struct TagOut { + tag: TagWire, +} + +pub(crate) async fn repo_tag( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + let did = resolve_repo(&state, ¶ms.repo)?; + let Some(name) = params.tag.get().cloned() else { + return Err(XrpcError::invalid_request("missing tag parameter")); + }; + let layout = state.layout.clone(); + let limit = state.byte_limits.response.get(); + run_blocking(move || { + let repo = open(&layout, &did)?; + let info = repo + .tag_list()? + .into_iter() + .find(|tag| tag.name == name) + .ok_or_else(|| { + XrpcError::named(StatusCode::BAD_REQUEST, "TagNotFound", "tag not found") + })?; + json( + TagOut { + tag: TagWire::of(&info), + }, + limit, + ) + }) + .await +} + +#[derive(Deserialize)] +pub(crate) struct BlobParams { + repo: RepoArg, + #[serde(rename = "ref", default)] + refspec: Revspec, + #[serde(default)] + path: TreePath, + #[serde(default)] + raw: RawFlag, +} + +#[derive(Serialize)] +struct SubmoduleOut { + name: String, + url: String, + branch: String, +} + +#[derive(Serialize)] +struct BlobOut { + #[serde(rename = "ref")] + refspec: String, + path: String, + #[serde(skip_serializing_if = "Option::is_none")] + content: Option, + #[serde(skip_serializing_if = "Option::is_none")] + encoding: Option<&'static str>, + #[serde(skip_serializing_if = "Option::is_none")] + size: Option, + #[serde(rename = "isBinary", skip_serializing_if = "Option::is_none")] + is_binary: Option, + #[serde(rename = "mimeType", skip_serializing_if = "Option::is_none")] + mime_type: Option<&'static str>, + #[serde(rename = "lastCommit", skip_serializing_if = "Option::is_none")] + last_commit: Option, + #[serde(skip_serializing_if = "Option::is_none")] + submodule: Option, +} + +fn etag_matches(headers: &HeaderMap, etag: &str) -> bool { + headers + .get_all(header::IF_NONE_MATCH) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(',')) + .map(str::trim) + .any(|candidate| { + candidate == "*" || candidate.strip_prefix("W/").unwrap_or(candidate) == etag + }) +} + +fn quoted_etag(digest: &[u8]) -> String { + format!("\"{}\"", knot_types::lowercase_hex(digest)) +} + +fn serve_raw( + headers: &HeaderMap, + mime: &'static str, + contents: Vec, +) -> Result { + if mime.starts_with("image/") || mime.starts_with("video/") { + let etag = quoted_etag(&Sha256::digest(&contents)); + if etag_matches(headers, &etag) { + return Ok(StatusCode::NOT_MODIFIED.into_response()); + } + return Ok(( + StatusCode::OK, + [ + (header::ETAG, etag), + (header::CONTENT_TYPE, mime.to_string()), + (header::X_CONTENT_TYPE_OPTIONS, "nosniff".to_string()), + (header::CONTENT_SECURITY_POLICY, RAW_CSP.to_string()), + ], + contents, + ) + .into_response()); + } + if sniff::is_textual_mime(mime) { + return Ok(( + StatusCode::OK, + [ + (header::CACHE_CONTROL, "public, no-cache".to_string()), + ( + header::CONTENT_TYPE, + "text/plain; charset=utf-8".to_string(), + ), + (header::X_CONTENT_TYPE_OPTIONS, "nosniff".to_string()), + (header::CONTENT_SECURITY_POLICY, RAW_CSP.to_string()), + ], + contents, + ) + .into_response()); + } + Err(XrpcError::named( + StatusCode::FORBIDDEN, + "InvalidRequest", + "only image, video, and text files can be accessed directly", + )) +} + +pub(crate) async fn repo_blob( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, + headers: HeaderMap, +) -> Result { + let did = resolve_repo(&state, ¶ms.repo)?; + if params.path.as_str().is_empty() { + return Err(XrpcError::invalid_request("missing path parameter")); + } + let layout = state.layout.clone(); + let limit = state.byte_limits.response.get(); + let blob_deadline = state.budgets.blob_last_commit.get().deadline(); + run_blocking(move || { + let refspec = params.refspec.as_str().to_string(); + let path = params.path.as_str().to_string(); + let raw = params.raw.requested(); + let repo = open(&layout, &did)?; + let commit = commit_for(&repo, ¶ms.refspec)?; + let submodule = repo + .submodules(commit) + .unwrap_or_default() + .into_iter() + .find(|submodule| submodule.path.as_str() == path); + if let Some(submodule) = submodule { + return json( + BlobOut { + refspec, + path, + content: None, + encoding: None, + size: None, + is_binary: None, + mime_type: None, + last_commit: None, + submodule: Some(SubmoduleOut { + name: submodule.name, + url: submodule.url, + branch: submodule + .branch + .map(|branch| branch.to_string()) + .unwrap_or_default(), + }), + }, + limit, + ); + } + let file_not_found = || { + XrpcError::named( + StatusCode::NOT_FOUND, + "FileNotFound", + "file not found at specified path", + ) + }; + let file_path = params.path.file().ok_or_else(file_not_found)?; + let entry = repo + .entry_at(commit, file_path)? + .filter(|entry| { + matches!( + entry.kind, + EntryKind::Blob | EntryKind::BlobExecutable | EntryKind::Link + ) + }) + .ok_or_else(file_not_found)?; + if repo.blob_size(entry.oid).map_err(|_| file_not_found())? > blob_serving_limit(raw, limit) + { + return Err(blob_too_large()); + } + let contents = repo.read_blob(entry.oid).map_err(|_| file_not_found())?; + let mime = sniff::override_by_extension(&path, sniff::detect_content_type(&contents)); + + if raw { + return serve_raw(&headers, mime, contents); + } + + let is_binary = !sniff::is_textual_mime(mime); + let size = contents.len() as i64; + let (content, encoding) = match is_binary { + true => ( + base64::Engine::encode(&base64::engine::general_purpose::STANDARD, &contents), + "base64", + ), + false => (String::from_utf8_lossy(&contents).into_owned(), "utf-8"), + }; + let dir = file_path.parent(); + let name = file_path.file_name().to_string(); + let last_commit = repo + .last_commits( + commit, + dir.as_ref(), + std::slice::from_ref(&name), + blob_deadline, + ) + .ok() + .and_then(|attributed| attributed.get(&name).cloned()) + .map(|last| LastCommitOut { + hash: last.id, + message: last.subject, + when: rfc3339(last.time.get(), 0), + author: repo.find_commit(last.id).ok().map(|commit| SignatureOut { + name: commit.author.name, + email: commit.author.email, + when: String::new(), + }), + }); + json( + BlobOut { + refspec, + path, + content: Some(content), + encoding: Some(encoding), + size: Some(size), + is_binary: Some(is_binary), + mime_type: Some(mime), + last_commit, + submodule: None, + }, + limit, + ) + }) + .await +} + +#[derive(Deserialize)] +pub(crate) struct DiffParams { + repo: RepoArg, + #[serde(rename = "ref", default)] + refspec: Revspec, +} + +#[derive(Serialize)] +struct DiffOut { + #[serde(rename = "ref", skip_serializing_if = "String::is_empty")] + refspec: String, + diff: crate::wire::NiceDiffWire, +} + +pub(crate) async fn repo_diff( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + let did = resolve_repo(&state, ¶ms.repo)?; + let layout = state.layout.clone(); + let limit = state.byte_limits.response.get(); + run_blocking(move || { + let repo = open(&layout, &did)?; + let target = commit_for(&repo, ¶ms.refspec)?; + let commit = repo.find_commit(target)?; + let patches = repo.commit_patches(knot_git::PatchRange { + base: commit.parents.first().copied(), + head: target, + })?; + json( + DiffOut { + refspec: params.refspec.as_str().to_string(), + diff: nice_diff(&commit, &patches), + }, + limit, + ) + }) + .await +} + +#[derive(Deserialize)] +pub(crate) struct CompareParams { + repo: RepoArg, + #[serde(default)] + rev1: Revspec, + #[serde(default)] + rev2: Revspec, +} + +#[derive(Serialize)] +struct CompareOut { + rev1: String, + rev2: String, + #[serde(skip_serializing_if = "Vec::is_empty")] + format_patch: Vec, + #[serde(rename = "patch", skip_serializing_if = "String::is_empty")] + patch_raw: String, + #[serde(skip_serializing_if = "Option::is_none")] + combined_patch: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + combined_patch_raw: Option, +} + +fn format_patch_entry( + commit: &Commit, + patches: &[knot_git::FilePatch], + raw: &str, +) -> FormatPatchWire { + let title = fold_subject(&commit.message); + let mut raw_headers: BTreeMap> = BTreeMap::from([ + ( + "From".to_string(), + vec![format!("{} <{}>", commit.author.name, commit.author.email)], + ), + ( + "Date".to_string(), + vec![rfc2822( + commit.author.time.get(), + commit.author.offset_seconds, + )], + ), + ("Subject".to_string(), vec![format!("[PATCH] {title}")]), + ]); + if let Some(change_id) = commit.change_id() { + raw_headers.insert("Change-Id".to_string(), vec![change_id.to_string()]); + } + let files: Vec = patches.iter().map(FileWire::of).collect(); + FormatPatchWire { + files: (!files.is_empty()).then_some(files), + sha: commit.id, + author: Some(PatchIdentityWire { + name: commit.author.name.clone(), + email: commit.author.email.clone(), + }), + author_date: rfc3339(commit.author.time.get(), commit.author.offset_seconds), + committer: None, + committer_date: ZERO_TIME.to_string(), + title, + body: normalize_message_section(message_body(&commit.message).lines()), + subject_prefix: "[PATCH] ".to_string(), + body_appendix: normalize_message_section(appendix_lines(raw)), + raw_headers: Some(raw_headers), + raw: raw.trim().to_string(), + } +} + +fn appendix_lines(raw: &str) -> impl Iterator { + raw.split_once("\n---\n") + .map(|(_, rest)| rest) + .unwrap_or_default() + .split("\ndiff --git ") + .next() + .unwrap_or_default() + .lines() +} + +pub(crate) async fn repo_compare( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + let did = resolve_repo(&state, ¶ms.repo)?; + let rev1 = params.rev1.as_str().to_string(); + if rev1.is_empty() { + return Err(XrpcError::invalid_request("missing rev1 parameter")); + } + let rev2 = params.rev2.as_str().to_string(); + if rev2.is_empty() { + return Err(XrpcError::invalid_request("missing rev2 parameter")); + } + let layout = state.layout.clone(); + let limit = state.byte_limits.response.get(); + run_blocking(move || { + let repo = open(&layout, &did)?; + let resolve = |rev: &str| { + let revision_not_found = || { + XrpcError::named( + StatusCode::BAD_REQUEST, + "RevisionNotFound", + format!("error resolving revision {rev}"), + ) + }; + if names_reserved(rev) { + return Err(revision_not_found()); + } + let commit = repo + .resolve_revision(rev) + .and_then(|oid| repo.peel_to_commit(oid).ok()) + .ok_or_else(revision_not_found)?; + match repo.reachable_from_public(commit) { + Ok(true) => Ok(commit), + Ok(false) => Err(revision_not_found()), + Err(error) => Err(error.into()), + } + }; + let base = resolve(&rev1)?; + let head = resolve(&rev2)?; + let compare_error = |error: knot_git::GitError| { + XrpcError::named( + StatusCode::BAD_REQUEST, + "CompareError", + format!("error comparing revisions: {error}"), + ) + }; + let between = repo + .commits_between( + CommitRange { base, head }, + LogLimit::new(MAX_COMPARE_COMMITS + 1), + ) + .map_err(compare_error)?; + if between.len() > MAX_COMPARE_COMMITS { + return Err(XrpcError::named( + StatusCode::BAD_REQUEST, + "CompareError", + format!("comparison spans more than maximum of {MAX_COMPARE_COMMITS} commits"), + )); + } + let commits: Vec = between + .into_iter() + .map(|oid| repo.find_commit(oid)) + .collect::, _>>() + .map_err(compare_error)? + .into_iter() + .rev() + .filter(|commit| commit.parents.len() <= 1) + .collect(); + let entries: Vec<(FormatPatchWire, String)> = commits + .iter() + .map(|commit| { + repo.commit_patches(knot_git::PatchRange { + base: commit.parents.first().copied(), + head: commit.id, + }) + .map(|patches| { + let raw = render_format_patch(commit, &patches); + (format_patch_entry(commit, &patches, &raw), raw) + }) + }) + .collect::, _>>() + .map_err(compare_error)?; + let patch_raw: String = entries.iter().map(|(_, raw)| format!("{raw}\n")).collect(); + let (combined_patch, combined_patch_raw) = match entries.len() >= 2 { + true => repo + .merge_base(base, head) + .ok() + .flatten() + .and_then(|merge_base| { + repo.commit_patches(knot_git::PatchRange { + base: Some(merge_base), + head, + }) + .ok() + .map(|patches| { + ( + Some(patches.iter().map(FileWire::of).collect::>()), + Some(render_patches(&patches)), + ) + }) + }) + .unwrap_or((None, None)), + false => (None, None), + }; + json( + CompareOut { + rev1: base.to_hex(), + rev2: head.to_hex(), + format_patch: entries.into_iter().map(|(entry, _)| entry).collect(), + patch_raw, + combined_patch, + combined_patch_raw, + }, + limit, + ) + }) + .await +} + +#[derive(Clone, Copy)] +struct ArchiveFormatArg(ArchiveFormat); + +impl Default for ArchiveFormatArg { + fn default() -> Self { + ArchiveFormatArg(ArchiveFormat::TarGz) + } +} + +impl ArchiveFormatArg { + fn format(self) -> ArchiveFormat { + self.0 + } + + fn name(self) -> &'static str { + match self.0 { + ArchiveFormat::Zip => "zip", + _ => "tar.gz", + } + } + + fn content_type(self) -> &'static str { + match self.0 { + ArchiveFormat::Zip => "application/zip", + _ => "application/gzip", + } + } +} + +impl<'de> Deserialize<'de> for ArchiveFormatArg { + fn deserialize>(deserializer: D) -> Result { + match String::deserialize(deserializer)?.as_str() { + "" | "tar.gz" => Ok(ArchiveFormatArg(ArchiveFormat::TarGz)), + "zip" => Ok(ArchiveFormatArg(ArchiveFormat::Zip)), + _ => Err(de::Error::custom( + "only tar.gz and zip formats are supported", + )), + } + } +} + +#[derive(Default)] +struct ArchivePrefixArg(Option); + +impl<'de> Deserialize<'de> for ArchivePrefixArg { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + match raw.is_empty() { + true => Ok(Self(None)), + false => knot_git::ArchivePrefix::new(raw) + .map(|prefix| Self(Some(prefix))) + .map_err(|_| de::Error::custom("archive prefix mustn't escape archive root")), + } + } +} + +#[derive(Deserialize)] +pub(crate) struct ArchiveParams { + repo: RepoArg, + #[serde(rename = "ref", default)] + refspec: Revspec, + #[serde(default)] + format: ArchiveFormatArg, + #[serde(default)] + prefix: ArchivePrefixArg, +} + +fn short_ref(refspec: &str) -> String { + refspec + .trim_start_matches("refs/heads/") + .trim_start_matches("refs/tags/") + .trim_start_matches("refs/remotes/") + .replace('/', "-") +} + +fn sanitize_filename(name: &str) -> String { + name.chars() + .map(|c| match c.is_ascii_control() || matches!(c, '"' | '\\') { + true => '-', + false => c, + }) + .collect() +} + +fn rfc5987_encode(name: &str) -> String { + name.bytes() + .map(|byte| match byte { + b'A'..=b'Z' + | b'a'..=b'z' + | b'0'..=b'9' + | b'!' + | b'#' + | b'$' + | b'&' + | b'+' + | b'-' + | b'.' + | b'^' + | b'_' + | b'`' + | b'|' + | b'~' => String::from(byte as char), + _ => format!("%{byte:02X}"), + }) + .collect() +} + +fn content_disposition(filename: &str) -> String { + let safe = sanitize_filename(filename); + let ascii: String = safe + .chars() + .map(|c| match c.is_ascii() { + true => c, + false => '-', + }) + .collect(); + match safe == ascii { + true => format!("attachment; filename=\"{ascii}\""), + false => format!( + "attachment; filename=\"{ascii}\"; filename*=UTF-8''{}", + rfc5987_encode(&safe) + ), + } +} + +struct BoundedSpool { + file: std::fs::File, + position: u64, + limit: u64, + tripped: bool, +} + +impl std::io::Write for BoundedSpool { + fn write(&mut self, data: &[u8]) -> std::io::Result { + if self.position.saturating_add(data.len() as u64) > self.limit { + self.tripped = true; + return Err(std::io::Error::new( + std::io::ErrorKind::WriteZero, + ARCHIVE_CAP_MESSAGE, + )); + } + let written = self.file.write(data)?; + self.position += written as u64; + Ok(written) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.file.flush() + } +} + +impl Seek for BoundedSpool { + fn seek(&mut self, pos: SeekFrom) -> std::io::Result { + let position = self.file.seek(pos)?; + self.position = position; + Ok(position) + } +} + +fn archive_etag(did: &RepoDid, commit: Oid, format: ArchiveFormat, prefix: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(did.as_str().as_bytes()); + hasher.update(b"\0"); + hasher.update(commit.to_hex().as_bytes()); + hasher.update(b"\0"); + hasher.update(ArchiveFormatArg(format).name().as_bytes()); + hasher.update(b"\0"); + hasher.update(prefix.as_bytes()); + quoted_etag(&hasher.finalize()) +} + +fn pinned_modified(modified_secs: i64) -> std::time::SystemTime { + std::time::UNIX_EPOCH + std::time::Duration::from_secs(modified_secs.max(0) as u64) +} + +fn reconcile_if_range(request: &mut Request, etag: &str, modified_secs: i64) { + let Some(value) = request + .headers() + .get(header::IF_RANGE) + .and_then(|value| value.to_str().ok()) + .map(str::trim) + .map(str::to_string) + else { + return; + }; + let resumes = match value.starts_with('"') || value.starts_with("W/") { + true => value.starts_with('"') && value == etag, + false => httpdate::parse_http_date(&value) + .is_ok_and(|client| client == pinned_modified(modified_secs)), + }; + let headers = request.headers_mut(); + headers.remove(header::IF_RANGE); + if !resumes { + headers.remove(header::RANGE); + } +} + +pub(crate) async fn repo_archive( + State(state): State>>, + mut request: Request, +) -> Result { + let params = Query::::try_from_uri(request.uri()) + .map_err(|rejection| XrpcError::invalid_request(rejection.body_text()))? + .0; + let did = resolve_repo(&state, ¶ms.repo)?; + let format = params.format; + let format_name = format.name(); + let repo_name = params.repo.basename().to_string(); + let safe_ref = short_ref(params.refspec.as_str()); + let archive_prefix = match ¶ms.prefix.0 { + None => format!("{repo_name}-{safe_ref}"), + Some(prefix) => prefix.as_str().to_string(), + }; + + let (resolved, modified_secs) = run_blocking({ + let layout = state.layout.clone(); + let did = did.clone(); + let refspec = params.refspec.clone(); + move || { + let repo = open(&layout, &did)?; + let commit = commit_for(&repo, &refspec)?; + let modified_secs = repo + .find_commit(commit) + .map(|commit| commit.committer.time.get()) + .unwrap_or(0); + Ok((commit, modified_secs)) + } + }) + .await?; + + let etag = archive_etag(&did, resolved, format.format(), &archive_prefix); + if etag_matches(request.headers(), &etag) { + return Ok(( + StatusCode::NOT_MODIFIED, + [ + (header::ETAG, etag), + (header::CACHE_CONTROL, "no-cache".to_string()), + ], + ) + .into_response()); + } + + let temp = run_blocking({ + let layout = state.layout.clone(); + let did = did.clone(); + let archive_limit = state.byte_limits.archive.get(); + let tree_prefix = knot_git::ArchivePrefix::new(format!("{archive_prefix}/")) + .expect("validated prefix with trailing slash stays valid"); + move || { + let repo = open(&layout, &did)?; + let tree = repo.peel_to_tree(resolved)?; + let temp = tempfile::NamedTempFile::new() + .map_err(|error| XrpcError::internal(format!("cannot spool archive: {error}")))?; + let file = temp + .reopen() + .map_err(|error| XrpcError::internal(format!("cannot spool archive: {error}")))?; + let mut spool = BoundedSpool { + file, + position: 0, + limit: archive_limit, + tripped: false, + }; + repo.write_archive(tree, format.format(), Some(&tree_prefix), &mut spool) + .map_err(|error| match spool.tripped { + true => XrpcError::request_too_large(ARCHIVE_CAP_MESSAGE), + false => XrpcError::named( + StatusCode::BAD_REQUEST, + "ArchiveError", + format!("failed to create archive: {error}"), + ), + })?; + temp.as_file() + .set_modified(pinned_modified(modified_secs)) + .map_err(|error| XrpcError::internal(error.to_string()))?; + Ok(temp) + } + }) + .await?; + + let immutable = { + let mut query = url::form_urlencoded::Serializer::new(String::new()); + query.append_pair("format", format_name); + query.append_pair("prefix", &archive_prefix); + query.append_pair("ref", &resolved.to_hex()); + query.append_pair("repo", ¶ms.repo.to_param()); + format!( + "{}/xrpc/sh.tangled.repo.archive?{}", + state.knot_service_url.as_str(), + query.finish() + ) + }; + let content_type = format.content_type(); + let disposition = content_disposition(&format!("{repo_name}-{safe_ref}.{format_name}")); + + reconcile_if_range(&mut request, &etag, modified_secs); + let serve_response = ServeFile::new(temp.path()) + .oneshot(request) + .await + .unwrap_or_else(|error| match error {}); + drop(temp); + + let mut response = serve_response.map(Body::new); + let headers = response.headers_mut(); + headers.insert(header::CONTENT_TYPE, HeaderValue::from_static(content_type)); + let header_value = |value: &str| { + HeaderValue::from_str(value).map_err(|error| XrpcError::internal(error.to_string())) + }; + headers.insert(header::CONTENT_DISPOSITION, header_value(&disposition)?); + headers.insert( + header::LINK, + header_value(&format!("<{immutable}>; rel=\"immutable\""))?, + ); + headers.insert(header::ETAG, header_value(&etag)?); + headers.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-cache")); + Ok(response) +} + +#[derive(Deserialize)] +pub(crate) struct LanguagesParams { + repo: RepoArg, + #[serde(rename = "ref", default)] + refspec: Revspec, +} + +#[derive(Serialize)] +struct LanguageOut { + name: knot_types::LanguageName, + size: knot_types::LanguageBytes, + percentage: i64, +} + +#[derive(Serialize)] +struct LanguagesOut { + #[serde(rename = "ref")] + refspec: String, + languages: Option>, + #[serde(rename = "totalSize", skip_serializing_if = "Option::is_none")] + total_size: Option, + #[serde(rename = "totalFiles", skip_serializing_if = "Option::is_none")] + total_files: Option, +} + +pub(crate) async fn repo_languages( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + let did = resolve_repo(&state, ¶ms.repo)?; + let layout = state.layout.clone(); + let limit = state.byte_limits.response.get(); + let languages_deadline = state.budgets.languages.get().deadline(); + run_blocking(move || { + let repo = open(&layout, &did)?; + let commit = commit_for(&repo, ¶ms.refspec)?; + let sizes = knot_langs::analyze(&repo, commit, languages_deadline)?; + let total: u64 = sizes.values().map(|size| size.get()).sum(); + let mut languages: Vec = sizes + .iter() + .filter(|(_, size)| size.get() > 0) + .map(|(name, size)| LanguageOut { + name: *name, + size: *size, + percentage: ((size.get() as f64) / (total as f64) * 100.0).round() as i64, + }) + .collect(); + languages.sort_by(|a, b| b.size.cmp(&a.size).then_with(|| a.name.cmp(&b.name))); + let count = languages.len() as i64; + json( + LanguagesOut { + refspec: params.refspec.as_str().to_string(), + languages: (!languages.is_empty()).then_some(languages), + total_size: (total > 0).then_some(total), + total_files: (total > 0).then_some(count), + }, + limit, + ) + }) + .await +} + +#[derive(Deserialize)] +pub(crate) struct DefaultBranchParams { + repo: RepoArg, +} + +#[derive(Serialize)] +struct DefaultBranchOut { + name: String, + hash: String, + when: String, +} + +pub(crate) async fn repo_get_default_branch( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + let did = resolve_repo(&state, ¶ms.repo)?; + let layout = state.layout.clone(); + let limit = state.byte_limits.response.get(); + run_blocking(move || { + let repo = open(&layout, &did)?; + let name = repo + .default_branch() + .map(|name| name.as_str().trim_start_matches("refs/heads/").to_string()) + .ok_or_else(|| { + XrpcError::named( + StatusCode::INTERNAL_SERVER_ERROR, + "InvalidRequest", + "failed to get default branch", + ) + })?; + json( + DefaultBranchOut { + name, + hash: String::new(), + when: rfc3339(0, 0), + }, + limit, + ) + }) + .await +} + +#[derive(Deserialize)] +pub(crate) struct DescribeRepoParams { + #[serde(rename = "repoDid")] + repo_did: RepoDid, +} + +#[derive(Serialize)] +struct DescribeRepoOut { + #[serde(rename = "repoDid")] + repo_did: RepoDid, + #[serde(rename = "ownerDid")] + owner_did: OwnerDid, + rkey: RepoRkey, +} + +pub(crate) async fn repo_describe_repo( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + let did = params.repo_did; + let owner = match state.index.owner_of(&did) { + Resolved::Ready(Some(owner)) => owner, + Resolved::Ready(None) => return Err(repo_not_found()), + Resolved::Warming => return Err(warming()), + }; + let rkey = match state.index.rkey_of(&did) { + Resolved::Ready(Some(rkey)) => rkey, + Resolved::Ready(None) => return Err(repo_not_found()), + Resolved::Warming => return Err(warming()), + }; + json( + DescribeRepoOut { + repo_did: did, + owner_did: owner, + rkey, + }, + state.byte_limits.response.get(), + ) +} + +#[derive(Serialize)] +struct DefaultBranchWire { + #[serde(rename = "ref")] + name: String, + #[serde(skip_serializing_if = "Option::is_none")] + head: Option, +} + +#[derive(Deserialize)] +pub(crate) struct ListRefsParams { + repo: RepoArg, + #[serde(default)] + limit: Limit, + #[serde(default)] + cursor: Offset, +} + +#[derive(Serialize)] +struct RefWire { + #[serde(rename = "ref")] + name: String, + sha: Oid, +} + +#[derive(Serialize)] +struct ListRefsOut { + refs: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + cursor: Option, + #[serde(rename = "defaultBranch", skip_serializing_if = "Option::is_none")] + default_branch: Option, +} + +pub(crate) async fn git_list_refs( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + let did = resolve_repo(&state, ¶ms.repo)?; + let offset = params.cursor; + let limit = params.limit; + let layout = state.layout.clone(); + let response_limit = state.byte_limits.response.get(); + run_blocking(move || { + let repo = open(&layout, &did)?; + let mut refs: Vec<_> = repo + .references()? + .into_iter() + .filter(|record| is_public_ref(&record.name)) + .collect(); + refs.sort_by(|a, b| a.name.as_str().cmp(b.name.as_str())); + let total = refs.len(); + let window: Vec = refs + .iter() + .skip(offset.get()) + .take(limit.get()) + .map(|record| RefWire { + name: record.name.as_str().to_string(), + sha: record.target, + }) + .collect(); + let cursor = next_cursor(offset, limit, Total::new(total)); + let default_branch = repo.head().map(|head| DefaultBranchWire { + name: head.name.as_str().to_string(), + head: Some(head.target.to_hex()), + }); + json( + ListRefsOut { + refs: window, + cursor, + default_branch, + }, + response_limit, + ) + }) + .await +} + +#[derive(Deserialize)] +pub(crate) struct ListReposParams { + #[serde(default)] + limit: Limit, + #[serde(default)] + cursor: Offset, + #[serde(default)] + order: Order, +} + +#[derive(Serialize)] +struct RepoWire { + repo: RepoDid, + status: &'static str, + #[serde(rename = "defaultBranch", skip_serializing_if = "Option::is_none")] + default_branch: Option, +} + +#[derive(Serialize)] +struct ListReposOut { + repos: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + cursor: Option, +} + +pub(crate) async fn sync_list_repos( + State(state): State>>, + ValidatedQuery(params): ValidatedQuery, +) -> Result { + if matches!(state.index.coverage().registry, Coverage::Warming) { + return Err(warming()); + } + let offset = params.cursor; + let limit = params.limit; + let mut repos = state.index.hosted_repos(); + if params.order.descending() { + repos.reverse(); + } + let total = repos.len(); + let page: Vec = repos + .into_iter() + .skip(offset.get()) + .take(limit.get()) + .collect(); + let cursor = next_cursor(offset, limit, Total::new(total)); + let layout = state.layout.clone(); + let response_limit = state.byte_limits.response.get(); + run_blocking(move || { + let repos: Vec = + page.iter() + .map(|did| RepoWire { + repo: did.clone(), + status: "active", + default_branch: open(&layout, did).ok().and_then(|repo| repo.head()).map( + |head| DefaultBranchWire { + name: head.name.as_str().to_string(), + head: Some(head.target.to_hex()), + }, + ), + }) + .collect(); + json(ListReposOut { repos, cursor }, response_limit) + }) + .await +} + +#[cfg(test)] +mod tests { + use super::{content_disposition, rfc5987_encode}; + + #[test] + fn content_disposition_quotes_dashes_quotes_and_adds_an_encoded_form_for_non_ascii() { + let cases: &[(&str, &str)] = &[ + ( + "squid-main.tar.gz", + "attachment; filename=\"squid-main.tar.gz\"", + ), + ( + "squid-a\"b.tar.gz", + "attachment; filename=\"squid-a-b.tar.gz\"", + ), + ( + "squid-café.zip", + "attachment; filename=\"squid-caf-.zip\"; filename*=UTF-8''squid-caf%C3%A9.zip", + ), + ]; + cases.iter().for_each(|(name, expected)| { + assert_eq!(content_disposition(name), *expected); + }); + } + + #[test] + fn rfc5987_percent_encodes_outside_the_attr_char_set() { + assert_eq!(rfc5987_encode("a b:c"), "a%20b%3Ac"); + assert_eq!(rfc5987_encode("plain-._~"), "plain-._~"); + } +} diff --git a/knot2/crates/knot-xrpc/src/receive.rs b/knot2/crates/knot-xrpc/src/receive.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/receive.rs @@ -0,0 +1,291 @@ +use std::future::Future; +use std::io::Read; +use std::pin::Pin; +use std::sync::Arc; + +use axum::Router; +use axum::body::Body; +use axum::extract::{DefaultBodyLimit, Path, State}; +use axum::http::{HeaderMap, HeaderValue, StatusCode, header}; +use axum::response::{IntoResponse, Response}; +use axum::routing::post; +use futures::TryStreamExt; +use knot_messages::{ErrorKey, HttpMessages}; +use knot_pack::{PackError, PackReceiver, ReceivedPack, SocketPeer}; +use knot_receive::{Push, land}; +use knot_runtime::{Clock, HttpTransport}; +use knot_types::{AccountDid, ActorId, ObjectFormat, RepoDid}; +use tokio_util::io::{StreamReader, SyncIoBridge}; + +use crate::{XrpcError, XrpcState, authenticate_and_authorize_push, run_blocking}; + +const READ_CHUNK: usize = 64 * 1024; +const ADVERTISEMENT: &str = "application/x-git-receive-pack-advertisement"; +const RESULT: &str = "application/x-git-receive-pack-result"; + +pub(crate) fn routes() -> Router>> { + Router::new() + .route( + "/{did}/{name}/git-receive-pack", + post(receive_named::), + ) + .route("/{did}/git-receive-pack", post(receive_did::)) + .layer(DefaultBodyLimit::disable()) +} + +pub fn advertiser( + state: Arc>, +) -> Arc { + Arc::new(ReceiveGate { state }) +} + +struct ReceiveGate { + state: Arc>, +} + +impl knot_pack::ReceiveAdvertiser for ReceiveGate { + fn advertise( + &self, + repo: RepoDid, + peer: SocketPeer, + headers: HeaderMap, + ) -> Pin + Send + '_>> { + Box::pin(async move { serve_advertisement(&self.state, repo, peer, &headers).await }) + } +} + +async fn serve_advertisement( + state: &Arc>, + repo: RepoDid, + peer: SocketPeer, + headers: &HeaderMap, +) -> Response { + if let Err(response) = authorized_pusher(state, peer, headers, &repo).await { + return response; + } + let layout = state.layout.clone(); + let target = repo.clone(); + let missing = state.catalog.http.repo_not_found.text(); + let advert = run_blocking(move || { + let repo = layout + .open(&target) + .map_err(|_| XrpcError::not_found(missing))?; + knot_pack::advertise_receive(&repo).map_err(map_pack) + }) + .await; + match advert { + Ok(body) => git_response(ADVERTISEMENT, body), + Err(error) => error.into_response(), + } +} + +async fn receive_named( + State(state): State>>, + Path(crate::RepoPathParams { did, name }): Path, + peer: SocketPeer, + headers: HeaderMap, + body: Body, +) -> Response { + let repo = match crate::resolve_repo_named(&state, &did, &name).await { + Ok(repo) => repo, + Err(error) => return error.into_response(), + }; + serve_receive(&state, repo, peer, &headers, body).await +} + +async fn receive_did( + State(state): State>>, + Path(did): Path, + peer: SocketPeer, + headers: HeaderMap, + body: Body, +) -> Response { + let repo = match crate::resolve_repo_did(&state, &did) { + Ok(repo) => repo, + Err(error) => return error.into_response(), + }; + serve_receive(&state, repo, peer, &headers, body).await +} + +async fn serve_receive( + state: &Arc>, + repo_did: RepoDid, + peer: SocketPeer, + headers: &HeaderMap, + body: Body, +) -> Response { + let pusher = match authorized_pusher(state, peer, headers, &repo_did).await { + Ok(pusher) => pusher, + Err(response) => return response, + }; + + let knot_actor = match state.secrets.public_key(&state.knot_did) { + Ok(public) => ActorId::from_secp256k1(public.as_bytes()), + Err(error) => return XrpcError::from(error).into_response(), + }; + + let format = { + let layout = state.layout.clone(); + let target = repo_did.clone(); + let missing = state.catalog.http.repo_not_found.text(); + match run_blocking(move || { + layout + .open(&target) + .map(|repo| repo.object_format()) + .map_err(|_| XrpcError::not_found(missing)) + }) + .await + { + Ok(format) => format, + Err(error) => return error.into_response(), + } + }; + + let _receive_permit = state.slots.receive.acquire().await; + + let received = { + let scratch = state.layout.scratch_dir().to_path_buf(); + let limits = state.pack_limits; + let limit = state.byte_limits.pack; + let catalog = Arc::clone(&state.catalog); + let reader = StreamReader::new(body.into_data_stream().map_err(std::io::Error::other)); + run_blocking(move || { + drain_pack( + SyncIoBridge::new(reader), + &scratch, + limit, + limits, + format, + &catalog.http, + ) + }) + .await + }; + let received = match received { + Ok(received) => received, + Err(error) => return error.into_response(), + }; + if received.is_empty() { + return git_response(RESULT, Vec::new()); + } + + let landed = land(Push { + layout: &state.layout, + repo_did: &repo_did, + received, + limits: state.pack_limits, + knot_actor, + committer: pusher, + events: Arc::clone(&state.events), + index: &state.index, + atproto: &state.atproto, + resolve_slots: &state.slots.resolve, + appview: &state.appview, + maintenance: &state.maintenance, + hostname: &state.knot_hostname, + languages_push_budget: state.budgets.languages_push, + catalog: Arc::clone(&state.catalog), + ci_logs: state.ci_logs.clone(), + }) + .await; + match landed { + Ok(framed) => git_response(RESULT, framed), + Err(error) => { + tracing::warn!(repo = repo_did.as_str(), %error, "http receive-pack failed"); + error.into_response() + } + } +} + +fn drain_pack( + mut reader: R, + scratch: &std::path::Path, + limit: knot_pack::MaxWireBytes, + limits: knot_pack::PackLimits, + format: ObjectFormat, + messages: &HttpMessages, +) -> Result { + let mut receiver = PackReceiver::new(scratch, limit, limits, format.kind()) + .map_err(|error| XrpcError::internal(format!("receive staging failed: {error}")))?; + let mut buffer = [0u8; READ_CHUNK]; + loop { + let read = reader + .read(&mut buffer) + .map_err(|error| XrpcError::invalid_request(format!("receive read error: {error}")))?; + if read == 0 { + break; + } + if receiver + .write(&buffer[..read]) + .map_err(|error| map_receive_read(error, messages))? + { + break; + } + } + receiver + .finish() + .map_err(|error| map_receive_read(error, messages)) +} + +async fn authorized_pusher( + state: &Arc>, + peer: SocketPeer, + headers: &HeaderMap, + repo: &RepoDid, +) -> Result { + let denied = state.catalog.http.push_denied.text(); + authenticate_and_authorize_push(state, peer, headers, repo, &denied) + .await + .map_err(challenge) +} + +fn challenge(error: XrpcError) -> Response { + if error.status() == StatusCode::UNAUTHORIZED { + ( + StatusCode::UNAUTHORIZED, + [(header::WWW_AUTHENTICATE, crate::BASIC_CHALLENGE)], + error.to_string(), + ) + .into_response() + } else { + error.into_response() + } +} + +fn git_response(content_type: &'static str, body: Vec) -> Response { + ( + StatusCode::OK, + [ + (header::CONTENT_TYPE, HeaderValue::from_static(content_type)), + ( + header::CACHE_CONTROL, + HeaderValue::from_static("no-cache, max-age=0, must-revalidate"), + ), + ], + body, + ) + .into_response() +} + +fn map_pack(error: PackError) -> XrpcError { + XrpcError::named(error.http_status(), "PackError", error.to_string()) +} + +fn map_receive_read(error: knot_pack::ReceiveReadError, messages: &HttpMessages) -> XrpcError { + match error { + knot_pack::ReceiveReadError::TooLarge => { + XrpcError::request_too_large(messages.push_too_large.text()) + } + knot_pack::ReceiveReadError::Truncated => { + XrpcError::invalid_request(messages.receive_ended_early.text()) + } + knot_pack::ReceiveReadError::Pack(error) => XrpcError::invalid_request( + messages + .malformed_pack + .line(|ErrorKey::Error| error.to_string()), + ), + knot_pack::ReceiveReadError::Io(error) => { + XrpcError::internal(format!("receive io error: {error}")) + } + } +} diff --git a/knot2/crates/knot-xrpc/src/repos.rs b/knot2/crates/knot-xrpc/src/repos.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/repos.rs @@ -0,0 +1,668 @@ +use std::sync::Arc; + +use axum::Json; +use axum::body::Bytes; +use axum::extract::State; +use axum::response::{IntoResponse, Response}; +use http::HeaderMap; +use serde::{Deserialize, Serialize}; + +use knot_acl::{KnotAcl, can_admin_knot, can_create_repo, can_delete_repo}; +use knot_atproto::{PreparedRepoDid, RecordPresence}; +use knot_cob::{CobHome, CobStore}; +use knot_cobs::{ + Registration, RegistryChange, Rename, RepoRef, RepoRegistryCob, deregister_repo, register_repo, +}; +use knot_git::{GitError, Layout, Repo}; +use knot_index::Resolved; +use knot_runtime::{Clock, HttpTransport, Signer}; +use knot_types::{ + AccountDid, ActorId, BranchName, OwnerDid, RefName, RepoDid, RepoName, RepoRkey, UnixSeconds, +}; + +use crate::body::SourceUrl; +use crate::error::XrpcError; +use crate::reservations::ReserveDecision; +use crate::{XrpcState, decode, ok_empty, run_blocking}; + +pub(crate) const CREATE_ROUTE: &str = "/xrpc/sh.tangled.repo.create"; +pub(crate) const DELETE_ROUTE: &str = "/xrpc/sh.tangled.repo.delete"; +pub(crate) const RENAME_ROUTE: &str = "/xrpc/sh.tangled.repo.rename"; +pub(crate) const RESERVE_ROUTE: &str = "/xrpc/sh.tangled.repo.reserveKey"; + +#[derive(Deserialize)] +struct CreateInput { + rkey: RepoRkey, + name: RepoName, + #[serde(rename = "defaultBranch")] + default_branch: Option, + #[serde(default, deserialize_with = "crate::body::optional_source_url")] + source: Option, + #[serde(rename = "repoDid")] + repo_did: Option, +} + +#[derive(Serialize)] +struct CreateOutput { + #[serde(rename = "repoDid")] + repo_did: RepoDid, + key: ActorId, + #[serde(rename = "lfsMissing", skip_serializing_if = "Vec::is_empty")] + lfs_missing: Vec, +} + +#[derive(Deserialize)] +struct DeleteInput { + did: OwnerDid, + rkey: RepoRkey, + #[serde(rename = "name")] + _name: RepoName, + #[serde(default)] + force: bool, +} + +#[derive(Deserialize)] +struct RenameInput { + repo: RepoDid, + rkey: RepoRkey, + name: RepoName, +} + +#[derive(Deserialize)] +struct ReserveInput { + #[serde(rename = "repoDid")] + repo_did: RepoDid, +} + +#[derive(Serialize)] +struct ReserveOutput { + #[serde(rename = "repoDid")] + repo_did: RepoDid, + key: ActorId, +} + +pub(crate) async fn reserve_key( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + body: Bytes, +) -> Result { + let actor = state.authenticate(&headers, &method).await?; + let acl = KnotAcl::new(&state.admins, state.admission, &state.index); + if !can_create_repo(&acl, &actor).is_allowed() { + return Err(XrpcError::forbidden( + "only knot admin or member may reserve a repository key", + )); + } + + let ReserveInput { repo_did } = decode(&body)?; + if !repo_did.as_str().starts_with("did:web:") { + return Err(XrpcError::invalid_request( + "only did:web repo identity needs a reserved key. Omit repoDid on create to mint a did:plc.", + )); + } + if repo_did.as_str() == state.knot_did.as_str() { + return Err(XrpcError::invalid_request( + "repoDid mustn't be knot's own identity", + )); + } + match state.index.owner_of(&repo_did) { + Resolved::Ready(Some(_)) => { + return Err(XrpcError::conflict( + "that repo DID is already hosted on this knot", + )); + } + Resolved::Warming => { + return Err(XrpcError::warming("registry projection is still warming")); + } + Resolved::Ready(None) => {} + } + + let now = state.now(); + state.reservations.prune(now); + + match state.reservations.try_reserve(&repo_did, &actor, now) { + ReserveDecision::HeldByOther => { + return Err(XrpcError::conflict( + "that repo DID is reserved by another account", + )); + } + ReserveDecision::PerActorFull => { + return Err(XrpcError::rate_limited( + "you are holding maximum number of reserved repository keys awaiting creation", + )); + } + ReserveDecision::GlobalFull => { + return Err(XrpcError::rate_limited( + "knot is holding maximum number of reserved repository keys awaiting creation", + )); + } + ReserveDecision::Fresh | ReserveDecision::Renewed => {} + } + + let public = match state.secrets.public_key(&state.knot_did) { + Ok(public) => public, + Err(error) => { + state.reservations.release(&repo_did); + return Err(error.into()); + } + }; + let key = ActorId::from_secp256k1(public.as_bytes()); + + Ok((http::StatusCode::OK, Json(ReserveOutput { repo_did, key })).into_response()) +} + +pub(crate) async fn create_repo( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + body: Bytes, +) -> Result { + let actor = state.authenticate(&headers, &method).await?; + let acl = KnotAcl::new(&state.admins, state.admission, &state.index); + if !can_create_repo(&acl, &actor).is_allowed() { + return Err(XrpcError::forbidden( + "only knot admin or member may create repositories", + )); + } + + let input: CreateInput = decode(&body)?; + let source = input + .source + .as_ref() + .map(|source| { + crate::forks::resolve_upstream(&state, source).map(|upstream| (source, upstream)) + }) + .transpose()?; + let head = input.default_branch.map(|branch| branch.head_ref()); + + let owner = OwnerDid::new(actor.as_str()).expect("account DID is always a valid owner DID"); + match state.index.resolve_repo(&owner, &input.rkey) { + Resolved::Ready(Some(existing)) => match state.index.rkey_of(&existing) { + Resolved::Ready(Some(canonical)) if canonical == input.rkey => { + return Err(XrpcError::conflict( + "repository with that record key already exists for this owner", + )); + } + Resolved::Warming => { + return Err(XrpcError::warming("registry projection is still warming")); + } + _ => {} + }, + Resolved::Warming => { + return Err(XrpcError::warming("registry projection is still warming")); + } + Resolved::Ready(None) => {} + } + + let (repo_did, provisioning) = + provision_repo_did(&state, &actor, &input.rkey, input.repo_did).await?; + let now = state.now(); + let knot_signer = state.secrets.signer(&state.knot_did)?; + let key = ActorId::from_secp256k1(knot_signer.public_key().as_bytes()); + + let registration = Registration { + owner, + rkey: input.rkey, + name: input.name, + repo: repo_did.clone(), + created_at: now, + }; + + let lfs_store = state.lfs.as_ref().map(|web| Arc::clone(&web.handle.store)); + let layout = state.layout.clone(); + let placed = repo_did.clone(); + let lfs = lfs_store.clone(); + let provisioning = run_blocking(move || { + let repo = layout.create(&placed).map_err(|error| match error { + GitError::AlreadyExists(_) => XrpcError::conflict("repository already exists on disk"), + GitError::ReservedDid(_) => { + XrpcError::invalid_request("repoDid mustn't be knot's own identity") + } + other => XrpcError::internal(other.to_string()), + })?; + let outcome = stage_repo(&repo, head.as_ref()); + if outcome.is_err() { + rollback_local(&layout, lfs.as_deref(), &placed); + } + outcome.map(|()| provisioning) + }) + .await?; + + let lfs_missing = match &source { + Some((source_url, upstream)) => { + match populate(&state, &repo_did, source_url, upstream).await { + Ok(missing) => missing, + Err(error) => { + let layout = state.layout.clone(); + let placed = repo_did.clone(); + let lfs = lfs_store.clone(); + let _ = run_blocking(move || { + rollback_local(&layout, lfs.as_deref(), &placed); + Ok(()) + }) + .await; + return Err(error); + } + } + } + None => Vec::new(), + }; + + let submission = match &provisioning { + Provisioned::Minted(prepared) => state + .atproto + .submit_plc_operation(prepared) + .await + .map(|_| ()), + Provisioned::Reserved => Ok(()), + }; + if let Err(error) = submission { + let layout = state.layout.clone(); + let placed = repo_did.clone(); + let lfs = lfs_store.clone(); + let _ = run_blocking(move || { + rollback_local(&layout, lfs.as_deref(), &placed); + Ok(()) + }) + .await; + return Err(error.into()); + } + + let reserved = matches!(provisioning, Provisioned::Reserved); + let layout = state.layout.clone(); + let cob_locks = Arc::clone(&state.cob_locks); + let meta_path = state.meta_path.clone(); + let placed = repo_did.clone(); + let lfs = lfs_store.clone(); + let home = CobHome::from(&state.knot_did); + run_blocking(move || { + let outcome = { + let _guard = cob_locks.meta(); + Repo::open(&meta_path) + .map_err(XrpcError::from) + .and_then(|meta| { + register( + &CobStore::new(&meta), + &home, + registration, + &knot_signer, + now, + ) + }) + }; + if outcome.is_err() { + rollback_local(&layout, lfs.as_deref(), &placed); + if !reserved { + tracing::error!( + repo = %placed, + "registration failed after did:plc submitted to PLC directory" + ); + } + } + outcome + }) + .await?; + + if reserved { + state.reservations.release(&repo_did); + } + + let index = Arc::clone(&state.index); + let refreshed = repo_did.clone(); + run_blocking(move || { + index.refresh_registry().map_err(|error| { + XrpcError::internal(format!( + "repo {refreshed} was created and registered but registry projection refresh failed: {error}" + )) + }) + }) + .await?; + + Ok(( + http::StatusCode::OK, + Json(CreateOutput { + repo_did, + key, + lfs_missing, + }), + ) + .into_response()) +} + +async fn populate( + state: &Arc>, + repo_did: &RepoDid, + source: &SourceUrl, + upstream: &crate::forks::Upstream, +) -> Result, XrpcError> { + let prefixes = vec![ + "HEAD".to_string(), + "refs/heads/".to_string(), + "refs/tags/".to_string(), + ]; + let refs = crate::forks::upstream_refs(state, upstream, prefixes).await?; + let tips = refs.tips(); + let pack = crate::forks::upstream_pack( + state, + upstream, + knot_pack::WantOids::new(refs.tips()), + knot_pack::HaveOids::default(), + ) + .await?; + let layout = state.layout.clone(); + let placed = repo_did.clone(); + let origin = source.clone(); + run_blocking(move || { + let repo = layout.open(&placed)?; + crate::forks::populate_fork(&repo, &refs, &pack, &origin) + }) + .await?; + crate::lfs::mirror_fork_objects( + Arc::clone(state), + upstream.clone(), + repo_did.clone(), + knot_pack::WantOids::new(tips), + knot_pack::HaveOids::default(), + ) + .await +} + +fn rollback_local(layout: &Layout, lfs: Option<&knot_lfs::DiskStore>, repo_did: &RepoDid) { + if let Some(store) = lfs + && let Err(error) = store.remove_repo(repo_did) + { + tracing::error!(repo = %repo_did, %error, "couldn't roll back lfs prefix"); + } + if let Err(error) = layout.remove(repo_did) { + tracing::error!(repo = %repo_did, %error, "couldn't roll back on-disk repo :3"); + } +} + +enum Provisioned { + Minted(PreparedRepoDid), + Reserved, +} + +async fn provision_repo_did( + state: &XrpcState, + actor: &knot_types::AccountDid, + rkey: &RepoRkey, + provided: Option, +) -> Result<(RepoDid, Provisioned), XrpcError> { + match provided { + Some(did) if did.as_str().starts_with("did:web:") => { + match state.index.owner_of(&did) { + Resolved::Ready(Some(_)) => { + return Err(XrpcError::conflict( + "that repo DID is already hosted on this knot", + )); + } + Resolved::Warming => { + return Err(XrpcError::warming("registry projection is still warming")); + } + Resolved::Ready(None) => {} + } + if !state.reservations.holder_is(&did, actor, state.now()) { + return Err(XrpcError::invalid_request( + "reserve this did:web for your own account via sh.tangled.repo.reserveKey before creating it", + )); + } + let knot_public = state.secrets.public_key(&state.knot_did)?; + state + .atproto + .verify_did_web_publishes_key(&did, &knot_public) + .await + .map_err(XrpcError::from)?; + Ok((did, Provisioned::Reserved)) + } + Some(_) => Err(XrpcError::invalid_request( + "repoDid must be did:web hosted on your own domain. Omit it to mint a did:plc.", + )), + None => { + let knot_signer = state.secrets.signer(&state.knot_did)?; + let owner = + OwnerDid::new(actor.as_str()).expect("account DID is always a valid owner DID"); + let nonce = knot_atproto::MintNonce::mint(&*state.entropy, &owner, rkey); + let prepared = + knot_atproto::prepare_repo_did(&knot_signer, &state.knot_service_url, &nonce) + .map_err(XrpcError::from)?; + Ok((prepared.did.clone(), Provisioned::Minted(prepared))) + } + } +} + +fn stage_repo(repo: &Repo, head: Option<&RefName>) -> Result<(), XrpcError> { + if let Some(refname) = head { + repo.set_head(refname)?; + } + Ok(()) +} + +fn register( + store: &CobStore, + home: &CobHome, + registration: Registration, + signer: &dyn Signer, + now: UnixSeconds, +) -> Result<(), XrpcError> { + match store + .list::() + .map_err(XrpcError::from)? + .as_slice() + { + [] => store + .create(home, &RegistryChange::Register(registration), signer, now) + .map(|_| ()) + .map_err(XrpcError::from), + [object] => register_repo(store, home, *object, registration, signer, now) + .map(|_| ()) + .map_err(XrpcError::from), + many => Err(XrpcError::internal(format!( + "{} repo registry objects share meta-repo", + many.len() + ))), + } +} + +pub(crate) async fn delete_repo( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + body: Bytes, +) -> Result { + let actor = state.authenticate(&headers, &method).await?; + let DeleteInput { + did, rkey, force, .. + } = decode(&body)?; + + let repo_did = match state.index.resolve_repo(&did, &rkey) { + Resolved::Ready(Some(repo_did)) => repo_did, + Resolved::Ready(None) => return Ok(ok_empty()), + Resolved::Warming => { + return Err(XrpcError::warming("registry projection is still warming")); + } + }; + + let acl = KnotAcl::new(&state.admins, state.admission, &state.index); + if !can_delete_repo(&acl, &actor, &repo_did).is_allowed() { + return Err(XrpcError::forbidden( + "only repository owner or a knot admin may delete it", + )); + } + + if force { + if !can_admin_knot(&acl, &actor).is_allowed() { + return Err(XrpcError::forbidden( + "only knot admin may force a delete past the PDS record check", + )); + } + } else { + let owner = AccountDid::from(did.clone()); + match state.atproto.repo_record_present(&owner, &rkey).await { + Ok(RecordPresence::Present) => { + return Err(XrpcError::conflict( + "sh.tangled.repo record still exists on the owner's PDS. Remove it there first or force the delete.", + )); + } + Ok(RecordPresence::Absent) => {} + Err(error) => { + tracing::warn!( + repo = %repo_did, + %error, + "proceeding w/ best-effort delete despite unconfirmed owner PDS record :p" + ); + } + } + } + + let now = state.now(); + let knot_signer = state.secrets.signer(&state.knot_did)?; + let layout = state.layout.clone(); + let index = Arc::clone(&state.index); + let cob_locks = Arc::clone(&state.cob_locks); + let meta_path = state.meta_path.clone(); + let target = RepoRef { owner: did, rkey }; + let deleted = repo_did.clone(); + let home = CobHome::from(&state.knot_did); + let lfs_store = state.lfs.as_ref().map(|lfs| Arc::clone(&lfs.handle.store)); + run_blocking(move || { + let _repo_guard = cob_locks.repo(&deleted); + let _meta_guard = cob_locks.meta(); + let meta = Repo::open(&meta_path)?; + let store = CobStore::new(&meta); + deregister(&store, &home, target, &deleted, &knot_signer, now)?; + let removal = layout.remove(&deleted); + index.refresh_registry().map_err(|error| { + XrpcError::internal(format!( + "repo {deleted} was deregistered but registry projection refresh failed: {error}" + )) + })?; + if let Some(store) = &lfs_store + && let Err(error) = store.remove_repo(&deleted) + { + tracing::warn!( + repo = %deleted, + %error, + "lfs prefix removal on delete failed, orphan sweep will reclaim" + ); + } + removal.map_err(|error| { + XrpcError::internal(format!( + "repo {deleted} was deregistered but its on-disk directory couldn't be removed: {error}" + )) + }) + }) + .await?; + + Ok(ok_empty()) +} + +fn deregister( + store: &CobStore, + home: &CobHome, + target: RepoRef, + expected: &RepoDid, + signer: &dyn Signer, + now: UnixSeconds, +) -> Result<(), XrpcError> { + match store + .list::() + .map_err(XrpcError::from)? + .as_slice() + { + [] => Ok(()), + [object] => deregister_repo(store, home, *object, target, expected.clone(), signer, now) + .map(|_| ()) + .map_err(XrpcError::from), + many => Err(XrpcError::internal(format!( + "{} repo registry objects share meta-repo", + many.len() + ))), + } +} + +pub(crate) async fn rename_repo( + State(state): State>>, + headers: HeaderMap, + method: crate::Method, + body: Bytes, +) -> Result { + let actor = state.authenticate(&headers, &method).await?; + let RenameInput { repo, rkey, name } = decode(&body)?; + + let owner = match state.index.owner_of(&repo) { + Resolved::Ready(Some(owner)) => owner, + Resolved::Ready(None) => { + return Err(XrpcError::not_found("no such repository on this knot")); + } + Resolved::Warming => { + return Err(XrpcError::warming("registry projection is still warming")); + } + }; + + crate::authorize_push( + &state, + &actor, + &repo, + "only repository owner or a collaborator may rename it", + ) + .await?; + + let now = state.now(); + let knot_signer = state.secrets.signer(&state.knot_did)?; + let index = Arc::clone(&state.index); + let cob_locks = Arc::clone(&state.cob_locks); + let meta_path = state.meta_path.clone(); + let renamed = repo.clone(); + let home = CobHome::from(&state.knot_did); + run_blocking(move || { + { + let _guard = cob_locks.meta(); + let meta = Repo::open(&meta_path)?; + let store = CobStore::new(&meta); + match store + .list::() + .map_err(XrpcError::from)? + .as_slice() + { + [] => { + return Err(XrpcError::not_found( + "no repositories are registered on this knot", + )); + } + [object] => { + knot_cobs::rename_repo( + &store, + &home, + *object, + Rename { + owner, + rkey, + name, + repo: renamed.clone(), + }, + &knot_signer, + now, + ) + .map(|_| ()) + .map_err(XrpcError::from)?; + } + many => { + return Err(XrpcError::internal(format!( + "{} repo registry objects share meta-repo", + many.len() + ))); + } + } + } + index.refresh_registry().map_err(|error| { + XrpcError::internal(format!( + "repo {renamed} was renamed but registry projection refresh failed: {error}" + )) + }) + }) + .await?; + + Ok(ok_empty()) +} diff --git a/knot2/crates/knot-xrpc/src/reservations.rs b/knot2/crates/knot-xrpc/src/reservations.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/reservations.rs @@ -0,0 +1,199 @@ +use knot_cache::{Admitted, Expiring, GroupQuota, Quotas, Rejected, TotalQuota}; +use knot_runtime::UnixMicros; +use knot_types::{AccountDid, RepoDid, UnixSeconds}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ReserveDecision { + Fresh, + Renewed, + HeldByOther, + PerActorFull, + GlobalFull, +} + +knot_types::scalar_newtype! { + pub struct ReservationTtl(i64); + pub struct PerActorQuota(usize); + pub struct GlobalQuota(usize); +} + +pub struct Reservations { + held: Expiring, + ttl: ReservationTtl, +} + +fn micros(seconds: UnixSeconds) -> UnixMicros { + UnixMicros::new((seconds.get().max(0) as u64).saturating_mul(1_000_000)) +} + +impl Reservations { + pub fn new(ttl: ReservationTtl, per_actor: PerActorQuota, global: GlobalQuota) -> Self { + Self { + held: Expiring::new(Quotas { + per_group: GroupQuota::new(per_actor.get()), + total: TotalQuota::new(global.get()), + }), + ttl, + } + } + + pub(crate) fn prune(&self, now: UnixSeconds) -> Vec { + self.held.prune(micros(now)) + } + + pub(crate) fn try_reserve( + &self, + repo: &RepoDid, + actor: &AccountDid, + now: UnixSeconds, + ) -> ReserveDecision { + let expires_at = micros(now.saturating_add_secs(self.ttl.get())); + match self.held.admit_or_renew( + repo.clone(), + actor.clone(), + actor.clone(), + expires_at, + micros(now), + ) { + Ok(Admitted::Inserted) => ReserveDecision::Fresh, + Ok(Admitted::Occupied(holder)) if &holder == actor => ReserveDecision::Renewed, + Ok(Admitted::Occupied(_)) => ReserveDecision::HeldByOther, + Err(Rejected::Group) => ReserveDecision::PerActorFull, + Err(Rejected::Total) => ReserveDecision::GlobalFull, + } + } + + pub(crate) fn holder_is(&self, repo: &RepoDid, actor: &AccountDid, now: UnixSeconds) -> bool { + self.held + .get(repo, micros(now)) + .is_some_and(|holder| &holder == actor) + } + + pub(crate) fn release(&self, repo: &RepoDid) { + self.held.remove(repo); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn actor(suffix: &str) -> AccountDid { + AccountDid::new(format!("did:web:{suffix}")).unwrap() + } + + fn repo(suffix: &str) -> RepoDid { + RepoDid::new(format!("did:web:{suffix}.olaren.dev")).unwrap() + } + + fn at(seconds: i64) -> UnixSeconds { + UnixSeconds::new(seconds) + } + + #[test] + fn a_reservation_binds_to_its_actor_and_blocks_a_stranger() { + let reservations = Reservations::new( + ReservationTtl::new(3_600), + PerActorQuota::new(8), + GlobalQuota::new(64), + ); + assert_eq!( + reservations.try_reserve(&repo("squid"), &actor("nel.pet"), at(0)), + ReserveDecision::Fresh + ); + assert_eq!( + reservations.try_reserve(&repo("squid"), &actor("olaren.dev"), at(1)), + ReserveDecision::HeldByOther, + "different account cannot take a live reservation" + ); + assert!(reservations.holder_is(&repo("squid"), &actor("nel.pet"), at(1))); + assert!(!reservations.holder_is(&repo("squid"), &actor("olaren.dev"), at(1))); + } + + #[test] + fn re_reserving_by_the_same_actor_renews_the_lease() { + let reservations = Reservations::new( + ReservationTtl::new(100), + PerActorQuota::new(8), + GlobalQuota::new(64), + ); + assert_eq!( + reservations.try_reserve(&repo("squid"), &actor("nel.pet"), at(0)), + ReserveDecision::Fresh + ); + assert_eq!( + reservations.try_reserve(&repo("squid"), &actor("nel.pet"), at(50)), + ReserveDecision::Renewed + ); + assert!( + reservations.holder_is(&repo("squid"), &actor("nel.pet"), at(140)), + "renewal pushed the expiry out from the second call instead of the first" + ); + } + + #[test] + fn the_per_actor_limit_bounds_one_account_without_touching_another() { + let reservations = Reservations::new( + ReservationTtl::new(3_600), + PerActorQuota::new(2), + GlobalQuota::new(64), + ); + assert_eq!( + reservations.try_reserve(&repo("a"), &actor("nel.pet"), at(0)), + ReserveDecision::Fresh + ); + assert_eq!( + reservations.try_reserve(&repo("b"), &actor("nel.pet"), at(0)), + ReserveDecision::Fresh + ); + assert_eq!( + reservations.try_reserve(&repo("c"), &actor("nel.pet"), at(0)), + ReserveDecision::PerActorFull, + "one account is held to its per-actor budget" + ); + assert_eq!( + reservations.try_reserve(&repo("c"), &actor("olaren.dev"), at(0)), + ReserveDecision::Fresh, + "different account keeps its own budget" + ); + } + + #[test] + fn the_global_limit_bounds_the_total_across_accounts() { + let reservations = Reservations::new( + ReservationTtl::new(3_600), + PerActorQuota::new(64), + GlobalQuota::new(2), + ); + assert_eq!( + reservations.try_reserve(&repo("a"), &actor("nel.pet"), at(0)), + ReserveDecision::Fresh + ); + assert_eq!( + reservations.try_reserve(&repo("b"), &actor("olaren.dev"), at(0)), + ReserveDecision::Fresh + ); + assert_eq!( + reservations.try_reserve(&repo("c"), &actor("teq.dev"), at(0)), + ReserveDecision::GlobalFull + ); + } + + #[test] + fn an_expired_reservation_is_pruned_and_frees_its_slot() { + let reservations = Reservations::new( + ReservationTtl::new(100), + PerActorQuota::new(8), + GlobalQuota::new(64), + ); + reservations.try_reserve(&repo("squid"), &actor("nel.pet"), at(0)); + assert!(reservations.prune(at(50)).is_empty(), "not yet expired"); + let pruned = reservations.prune(at(150)); + assert_eq!(pruned, vec![repo("squid")], "expired lease is reaped"); + assert_eq!( + reservations.try_reserve(&repo("squid"), &actor("olaren.dev"), at(160)), + ReserveDecision::Fresh, + "once the lease lapses a different account may claim DID" + ); + } +} diff --git a/knot2/crates/knot-xrpc/src/service.rs b/knot2/crates/knot-xrpc/src/service.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/service.rs @@ -0,0 +1,68 @@ +use std::sync::Arc; + +use axum::Json; +use axum::extract::State; +use axum::response::{IntoResponse, Response}; +use serde::Serialize; + +use knot_runtime::{Clock, HttpTransport}; +use knot_types::AccountDid; + +use crate::XrpcState; + +pub(crate) const VERSION_ROUTE: &str = "/xrpc/sh.tangled.knot.version"; +pub(crate) const OWNER_ROUTE: &str = "/xrpc/sh.tangled.owner"; +pub(crate) const HEALTH_ROUTE: &str = "/xrpc/_health"; + +const WIRE_VERSION: &str = "v1.15.0"; + +#[derive(Serialize)] +struct VersionWire { + version: &'static str, + capabilities: [&'static str; 1], +} + +#[derive(Serialize)] +struct HealthWire { + version: String, +} + +pub(crate) async fn health( + State(state): State>>, +) -> Response { + if let Some(lfs) = state.lfs.as_ref() + && !lfs.ready().await + { + return ( + axum::http::StatusCode::SERVICE_UNAVAILABLE, + "lfs store is unreachable or not writable", + ) + .into_response(); + } + Json(HealthWire { + version: format!("knot {}", env!("CARGO_PKG_VERSION")), + }) + .into_response() +} + +#[derive(Serialize)] +struct OwnerWire { + owner: AccountDid, +} + +pub(crate) async fn version() -> Response { + Json(VersionWire { + version: WIRE_VERSION, + capabilities: ["knot-acl"], + }) + .into_response() +} + +pub(crate) async fn owner( + State(state): State>>, +) -> Response { + Json(OwnerWire { + owner: state.service_owner.clone(), + }) + .into_response() +} diff --git a/knot2/crates/knot-xrpc/src/sniff.rs b/knot2/crates/knot-xrpc/src/sniff.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/sniff.rs @@ -0,0 +1,264 @@ +const SNIFF_LIMIT: usize = 512; + +fn is_ws(byte: u8) -> bool { + matches!(byte, b'\t' | b'\n' | 0x0c | b'\r' | b' ') +} + +fn is_tt(byte: u8) -> bool { + matches!(byte, b' ' | b'>') +} + +enum Sig { + Exact(&'static [u8], &'static str), + Masked { + mask: &'static [u8], + pat: &'static [u8], + skip_ws: bool, + ct: &'static str, + }, + Html(&'static [u8]), + Mp4, + Text, +} + +impl Sig { + fn detect(&self, data: &[u8], first_non_ws: usize) -> Option<&'static str> { + match self { + Sig::Exact(sig, ct) => data.starts_with(sig).then_some(*ct), + Sig::Masked { + mask, + pat, + skip_ws, + ct, + } => { + let data = if *skip_ws { + &data[first_non_ws..] + } else { + data + }; + (mask.len() == pat.len() + && data.len() >= pat.len() + && pat + .iter() + .zip(mask.iter()) + .enumerate() + .all(|(index, (byte, mask))| data[index] & mask == *byte)) + .then_some(*ct) + } + Sig::Html(tag) => { + let data = &data[first_non_ws..]; + (data.len() > tag.len() + && tag.iter().enumerate().all(|(index, byte)| { + let candidate = data[index]; + let candidate = match byte.is_ascii_uppercase() { + true => candidate & 0xDF, + false => candidate, + }; + *byte == candidate + }) + && is_tt(data[tag.len()])) + .then_some("text/html; charset=utf-8") + } + Sig::Mp4 => mp4(data), + Sig::Text => text(data, first_non_ws), + } + } +} + +fn mp4(data: &[u8]) -> Option<&'static str> { + if data.len() < 12 { + return None; + } + let box_size = u32::from_be_bytes([data[0], data[1], data[2], data[3]]) as usize; + if data.len() < box_size || !box_size.is_multiple_of(4) || &data[4..8] != b"ftyp" { + return None; + } + (8..box_size) + .step_by(4) + .filter(|start| *start != 12) + .any(|start| &data[start..start + 3] == b"mp4") + .then_some("video/mp4") +} + +fn text(data: &[u8], first_non_ws: usize) -> Option<&'static str> { + data[first_non_ws..] + .iter() + .all(|byte| !matches!(byte, 0x00..=0x08 | 0x0b | 0x0e..=0x1a | 0x1c..=0x1f)) + .then_some("text/plain; charset=utf-8") +} + +const SIGNATURES: &[Sig] = &[ + Sig::Html(b" &'static str { + let data = &content[..content.len().min(SNIFF_LIMIT)]; + let first_non_ws = data + .iter() + .position(|byte| !is_ws(*byte)) + .unwrap_or(data.len()); + SIGNATURES + .iter() + .find_map(|sig| sig.detect(data, first_non_ws)) + .unwrap_or("application/octet-stream") +} + +pub(crate) fn override_by_extension(path: &str, detected: &'static str) -> &'static str { + let extension = path.rsplit_once('.').map(|(_, ext)| ext).unwrap_or(""); + match extension.to_ascii_lowercase().as_str() { + "svg" => "image/svg+xml", + "avif" => "image/avif", + "jxl" => "image/jxl", + "heic" | "heif" => "image/heif", + _ => detected, + } +} + +pub(crate) fn is_textual_mime(mime: &str) -> bool { + mime.starts_with("text/") + || matches!( + mime, + "application/json" + | "application/xml" + | "application/yaml" + | "application/x-yaml" + | "application/toml" + | "application/javascript" + | "application/ecmascript" + ) +} + +#[cfg(test)] +mod tests { + use super::detect_content_type; + + #[test] + fn detects_common_content_signatures() { + let cases: &[(&[u8], &str)] = &[ + (b" \n", "text/html; charset=utf-8"), + (b"", "text/html; charset=utf-8"), + (b"", "text/html; charset=utf-8"), + (b"\n\t", "text/xml; charset=utf-8"), + (b"\xfe\xff\x00h", "text/plain; charset=utf-16be"), + (b"\xef\xbb\xbfhello", "text/plain; charset=utf-8"), + (b"\x89PNG\x0d\x0a\x1a\x0a", "image/png"), + (b"GIF89a", "image/gif"), + (b"fn main() {}\n", "text/plain; charset=utf-8"), + (b"\x00\x01\x02\x03", "application/octet-stream"), + (b"", "text/plain; charset=utf-8"), + ]; + cases.iter().for_each(|(input, expected)| { + assert_eq!(detect_content_type(input), *expected); + }); + } +} diff --git a/knot2/crates/knot-xrpc/src/tests.rs b/knot2/crates/knot-xrpc/src/tests.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/tests.rs @@ -0,0 +1,3044 @@ +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; + +use axum::body::Bytes; +use axum::extract::State; +use axum::response::{IntoResponse, Response}; +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use futures::StreamExt; +use http::{HeaderMap, HeaderValue, StatusCode, header::AUTHORIZATION}; +use serde_json::json; +use tempfile::TempDir; + +use knot_atproto::Atproto; +use knot_git::Layout; +use knot_index::{Index, Resolved}; +use knot_runtime::{ + FakeHttp, HttpRequest, HttpResponse, HttpTransport, K256Signer, ManualClock, NetworkError, + OsEntropy, SeededEntropy, Signer, UnixMicros, +}; +use knot_secrets::{MasterKey, SealedStore}; +use knot_types::{ + AccountDid, AdmissionPolicy, AuthorName, Email, KnotHostname, KnotId, OwnerDid, RepoDid, + RepoRkey, +}; + +use crate::XrpcState; + +const KNOT_HOST: &str = "knot.nel.pet"; +const ADMIN_HOST: &str = "admin.nel.pet"; +const MEMBER_HOST: &str = "member.nel.pet"; +const STRANGER_HOST: &str = "stranger.nel.pet"; + +const ADD_MEMBER: &str = "sh.tangled.knot.addMember"; +const REMOVE_MEMBER: &str = "sh.tangled.knot.removeMember"; +const BAN: &str = "sh.tangled.knot.ban"; +const UNBAN: &str = "sh.tangled.knot.unban"; +const CREATE: &str = "sh.tangled.repo.create"; +const RESERVE: &str = "sh.tangled.repo.reserveKey"; +const DELETE: &str = "sh.tangled.repo.delete"; +const RENAME: &str = "sh.tangled.repo.rename"; +const ADD_COLLAB: &str = "sh.tangled.repo.addCollaborator"; +const REMOVE_COLLAB: &str = "sh.tangled.repo.removeCollaborator"; +const SET_DEFAULT: &str = "sh.tangled.repo.setDefaultBranch"; +const DELETE_BRANCH: &str = "sh.tangled.repo.deleteBranch"; + +type Responder = Box Result + Send + Sync>; +type SharedState = Arc, ManualClock>>; + +static JTI: AtomicU64 = AtomicU64::new(0); + +fn knot_did() -> KnotId { + KnotId::new(format!("did:web:{KNOT_HOST}")).unwrap() +} + +fn account(host: &str) -> AccountDid { + AccountDid::new(format!("did:web:{host}")).unwrap() +} + +fn signer(seed: u64) -> K256Signer { + K256Signer::generate(&SeededEntropy::new(seed)) +} + +fn did_web_doc(did: &str, sec1: &[u8], pds: &str) -> Bytes { + let multikey = knot_types::crypto::multikey(0xe7, sec1); + Bytes::from( + serde_json::to_vec(&json!({ + "id": did, + "alsoKnownAs": [], + "verificationMethod": [{ + "id": format!("{did}#atproto"), + "type": "Multikey", + "controller": did, + "publicKeyMultibase": multikey + }], + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": pds + }] + })) + .unwrap(), + ) +} + +fn repo_did_doc(did: &str, multikey: &str) -> Bytes { + Bytes::from( + serde_json::to_vec(&json!({ + "id": did, + "verificationMethod": [{ + "id": format!("{did}#repo"), + "type": "Multikey", + "controller": did, + "publicKeyMultibase": multikey + }] + })) + .unwrap(), + ) +} + +fn mint(signer: &K256Signer, issuer: &AccountDid, method: &str) -> String { + let jti = JTI.fetch_add(1, Ordering::Relaxed); + let header = URL_SAFE_NO_PAD.encode(br#"{"alg":"ES256K","typ":"JWT"}"#); + let payload = URL_SAFE_NO_PAD.encode( + serde_json::to_vec(&json!({ + "iss": issuer.as_str(), + "aud": format!("did:web:{KNOT_HOST}"), + "exp": 1_100, + "iat": 999, + "jti": format!("nonce-{jti}"), + "lxm": method, + })) + .unwrap(), + ); + let signing_input = format!("{header}.{payload}"); + let signature = signer.sign(signing_input.as_bytes()); + format!( + "{signing_input}.{}", + URL_SAFE_NO_PAD.encode(signature.as_bytes()) + ) +} + +fn bearer(token: &str) -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert( + AUTHORIZATION, + HeaderValue::from_str(&format!("Bearer {token}")).unwrap(), + ); + headers +} + +fn body(value: serde_json::Value) -> Bytes { + Bytes::from(serde_json::to_vec(&value).unwrap()) +} + +fn into_response(result: Result) -> Response { + match result { + Ok(response) => response, + Err(error) => error.into_response(), + } +} + +async fn json_of(response: Response) -> serde_json::Value { + let bytes = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + serde_json::from_slice(&bytes).unwrap() +} + +async fn call( + world: &World, + handler: F, + signer: &K256Signer, + host: &str, + nsid: &str, + value: serde_json::Value, +) -> Response +where + F: FnOnce(State, HeaderMap, crate::Method, Bytes) -> Fut, + Fut: std::future::Future>, +{ + let token = mint(signer, &account(host), nsid); + into_response( + handler( + world.state(), + bearer(&token), + crate::Method::from_nsid(nsid), + body(value), + ) + .await, + ) +} + +async fn as_member( + world: &World, + handler: F, + nsid: &str, + value: serde_json::Value, +) -> Response +where + F: FnOnce(State, HeaderMap, crate::Method, Bytes) -> Fut, + Fut: std::future::Future>, +{ + call(world, handler, &world.member, MEMBER_HOST, nsid, value).await +} + +async fn as_admin( + world: &World, + handler: F, + nsid: &str, + value: serde_json::Value, +) -> Response +where + F: FnOnce(State, HeaderMap, crate::Method, Bytes) -> Fut, + Fut: std::future::Future>, +{ + call(world, handler, &world.admin, ADMIN_HOST, nsid, value).await +} + +async fn as_stranger( + world: &World, + handler: F, + nsid: &str, + value: serde_json::Value, +) -> Response +where + F: FnOnce(State, HeaderMap, crate::Method, Bytes) -> Fut, + Fut: std::future::Future>, +{ + call(world, handler, &world.stranger, STRANGER_HOST, nsid, value).await +} + +fn member_owner() -> OwnerDid { + OwnerDid::new(format!("did:web:{MEMBER_HOST}")).unwrap() +} + +fn resolve(world: &World, rkey: &str) -> Resolved> { + world + .state + .index + .resolve_repo(&member_owner(), &RepoRkey::new(rkey).unwrap()) +} + +fn replay(world: &World) -> Vec> { + world + .state + .events + .replay( + knot_events::EventCursor::START, + knot_events::ReplayBounds::new( + knot_events::ReplayEvents::new(64).unwrap(), + knot_events::ReplayBytes::new(16 << 20).unwrap(), + ), + ) + .events +} + +fn event_count(world: &World) -> usize { + replay(world).len() +} + +fn last_event(world: &World, nsid: &str) -> std::sync::Arc { + replay(world) + .into_iter() + .rev() + .find(|event| event.nsid == nsid) + .unwrap_or_else(|| panic!("{nsid} event is emitted")) +} + +fn git_events(world: &World) -> Vec> { + replay(world) + .into_iter() + .filter(|event| { + !matches!( + event.nsid, + "sh.tangled.knot.memberUpdate" | "sh.tangled.repo.collaboratorUpdate" + ) + }) + .collect() +} + +fn only_git_event(world: &World) -> std::sync::Arc { + let mut events = git_events(world); + assert_eq!(events.len(), 1, "expected exactly one non-acl event"); + events.remove(0) +} + +fn bootstrap(dir: &TempDir, rebuild: bool) -> (Layout, Arc, PathBuf) { + let scan_path = dir.path().join("repos"); + std::fs::create_dir_all(&scan_path).unwrap(); + let knot = knot_did(); + let layout = Layout::new(&scan_path).reserving_meta(&knot).unwrap(); + layout.bootstrap_meta(&knot).unwrap(); + let meta_path = layout.meta_path(&knot).unwrap(); + let index = Arc::new(Index::new(meta_path.clone(), layout.clone())); + if rebuild { + index.rebuild().unwrap(); + } + (layout, index, meta_path) +} + +fn state_from( + dir: &TempDir, + boot: (Layout, Arc, PathBuf), + responder: Responder, + admission: AdmissionPolicy, + reservations: Arc, + git_http: Arc, +) -> SharedState { + let (layout, index, meta_path) = boot; + let knot = knot_did(); + let knot_url = knot_types::KnotServiceUrl::new(format!("https://{KNOT_HOST}")).unwrap(); + let atproto = Arc::new(Atproto::new( + FakeHttp::new(responder), + ManualClock::new(UnixMicros::new(1_000_000_000)), + knot.clone(), + knot_atproto::PlcDirectory::new(url::Url::parse("https://plc.directory/").unwrap()) + .unwrap(), + )); + let secrets = Arc::new( + SealedStore::open( + dir.path().join("keys.sealed"), + &MasterKey::new([7u8; 32]).unwrap(), + Box::new(OsEntropy), + ) + .unwrap(), + ); + secrets.ensure(&knot).unwrap(); + Arc::new(XrpcState { + layout, + index, + atproto, + secrets, + entropy: Arc::new(OsEntropy), + ci_logs: None, + admins: BTreeSet::from([account(ADMIN_HOST)]), + admission, + knot_did: knot, + knot_hostname: KnotHostname::new(KNOT_HOST).unwrap(), + meta_path, + knot_service_url: knot_url, + limiter: Arc::new(crate::PreAuthLimiter::default()), + cob_locks: Arc::new(crate::CobLocks::default()), + reservations, + trusted_proxy_header: None, + committer: crate::Committer { + name: AuthorName::new("Tangled"), + email: Email::new("noreply@tangled.sh"), + }, + byte_limits: crate::ByteLimits::default(), + budgets: crate::Budgets::default(), + git_http, + pack_limits: knot_pack::PackLimits::default(), + service_owner: account(ADMIN_HOST), + events: Arc::new(knot_events::EventLog::new( + ManualClock::new(UnixMicros::new(1_000_000_000)), + knot_events::ReplayBounds::new( + knot_events::ReplayEvents::new(1024).unwrap(), + knot_events::ReplayBytes::new(16 << 20).unwrap(), + ), + )), + subscriber_gate: Arc::new(knot_events::SubscriberGate::new( + knot_events::GlobalSubscriberLimit::new(16), + knot_events::PerPeerSubscriberLimit::new(8), + )), + maintenance: knot_maintenance::MaintenanceHandle::disabled(), + appview: knot_types::AppviewEndpoint::new("https://tangled.test").unwrap(), + slots: knot_resource::Slots::testing(8), + lfs: None, + catalog: Arc::new(knot_messages::Catalog::defaults()), + }) +} + +fn world_responder( + pubkeys: HashMap>, + repo_docs: Arc>>, + pds_records: Arc>>, +) -> Responder { + let doc_url = format!("https://{KNOT_HOST}"); + Box::new(move |request: &HttpRequest| { + if request.method == http::Method::POST { + return Ok(HttpResponse { + status: StatusCode::OK, + headers: http::HeaderMap::new(), + body: Bytes::new(), + }); + } + if request.url.path().ends_with("com.atproto.repo.getRecord") { + let rkey = request + .url + .query_pairs() + .find(|(key, _)| key == "rkey") + .map(|(_, value)| value.into_owned()) + .unwrap_or_default(); + let present = pds_records.lock().unwrap().contains(&rkey); + return Ok(HttpResponse { + status: if present { + StatusCode::OK + } else { + StatusCode::BAD_REQUEST + }, + headers: http::HeaderMap::new(), + body: if present { + Bytes::new() + } else { + Bytes::from_static(b"{\"error\":\"RecordNotFound\"}") + }, + }); + } + let host = request.url.host_str().unwrap_or_default(); + if let Some(multikey) = repo_docs.lock().unwrap().get(host).cloned() { + return Ok(HttpResponse { + status: StatusCode::OK, + headers: http::HeaderMap::new(), + body: repo_did_doc(&format!("did:web:{host}"), &multikey), + }); + } + match pubkeys.get(host) { + Some(sec1) => Ok(HttpResponse { + status: StatusCode::OK, + headers: http::HeaderMap::new(), + body: did_web_doc(&format!("did:web:{host}"), sec1, &doc_url), + }), + None => Ok(HttpResponse { + status: StatusCode::NOT_FOUND, + headers: http::HeaderMap::new(), + body: Bytes::new(), + }), + } + }) +} + +struct World { + _dir: TempDir, + layout: Layout, + state: SharedState, + admin: K256Signer, + member: K256Signer, + stranger: K256Signer, + repo_docs: Arc>>, + pds_records: Arc>>, +} + +impl World { + fn new() -> Self { + Self::build(256, 256, AdmissionPolicy::Closed, None) + } + + fn open() -> Self { + Self::build(256, 256, AdmissionPolicy::Open, None) + } + + fn with_pending_limit(limit: usize) -> Self { + Self::build(limit, limit, AdmissionPolicy::Closed, None) + } + + fn with_limits(global: usize, per_actor: usize) -> Self { + Self::build(global, per_actor, AdmissionPolicy::Closed, None) + } + + fn with_git_http(git_http: Arc) -> Self { + Self::build(256, 256, AdmissionPolicy::Closed, Some(git_http)) + } + + fn build( + global: usize, + per_actor: usize, + admission: AdmissionPolicy, + git_http: Option>, + ) -> Self { + let dir = tempfile::tempdir().unwrap(); + let (layout, index, meta_path) = bootstrap(&dir, true); + let admin = signer(1); + let member = signer(2); + let stranger = signer(3); + let repo_docs: Arc>> = Arc::new(Mutex::new(HashMap::new())); + let pds_records: Arc>> = Arc::new(Mutex::new(HashSet::new())); + let pubkeys = HashMap::from([ + ( + ADMIN_HOST.to_string(), + admin.public_key().as_bytes().to_vec(), + ), + ( + MEMBER_HOST.to_string(), + member.public_key().as_bytes().to_vec(), + ), + ( + STRANGER_HOST.to_string(), + stranger.public_key().as_bytes().to_vec(), + ), + ]); + let responder = world_responder(pubkeys, Arc::clone(&repo_docs), Arc::clone(&pds_records)); + let state = state_from( + &dir, + (layout.clone(), index, meta_path), + responder, + admission, + Arc::new(crate::Reservations::new( + crate::ReservationTtl::new(1_000_000), + crate::PerActorQuota::new(per_actor), + crate::GlobalQuota::new(global), + )), + git_http.unwrap_or_else(no_git_upstream), + ); + Self { + _dir: dir, + layout, + state, + admin, + member, + stranger, + repo_docs, + pds_records, + } + } + + fn state(&self) -> State { + State(Arc::clone(&self.state)) + } + + fn publish_repo_doc(&self, host: &str, multikey: &str) { + self.repo_docs + .lock() + .unwrap() + .insert(host.to_string(), multikey.to_string()); + } + + fn publish_pds_record(&self, rkey: &str) { + self.pds_records.lock().unwrap().insert(rkey.to_string()); + } +} + +fn build_state(responder: Responder, rebuild: bool) -> (TempDir, SharedState) { + let dir = tempfile::tempdir().unwrap(); + let (layout, index, meta_path) = bootstrap(&dir, rebuild); + let state = state_from( + &dir, + (layout, index, meta_path), + responder, + AdmissionPolicy::Closed, + Arc::new(crate::Reservations::new( + crate::ReservationTtl::new(1_000_000), + crate::PerActorQuota::new(256), + crate::GlobalQuota::new(256), + )), + no_git_upstream(), + ); + (dir, state) +} + +fn no_git_upstream() -> Arc { + Arc::new(FakeHttp::new(|_request: &HttpRequest| { + Err(NetworkError::Connect( + "no git upstream is served in this test".to_string(), + )) + })) +} + +fn doc_responder( + sec1: Vec, + post_status: impl Fn() -> StatusCode + Send + Sync + 'static, +) -> Responder { + let pds = format!("https://{KNOT_HOST}"); + Box::new(move |request: &HttpRequest| { + let post = request.method == http::Method::POST; + let host = request.url.host_str().unwrap_or_default(); + Ok(HttpResponse { + status: if post { post_status() } else { StatusCode::OK }, + headers: http::HeaderMap::new(), + body: if post { + Bytes::new() + } else { + did_web_doc(&format!("did:web:{host}"), &sec1, &pds) + }, + }) + }) +} + +async fn add_member_helper(world: &World) { + assert_eq!( + as_admin( + world, + crate::members::add_member, + ADD_MEMBER, + json!({ "subject": format!("did:web:{MEMBER_HOST}") }) + ) + .await + .status(), + StatusCode::OK + ); +} + +async fn create_repo_helper(world: &World, name: &str) -> RepoDid { + assert_eq!( + as_member( + world, + crate::repos::create_repo, + CREATE, + json!({ "rkey": name, "name": name }) + ) + .await + .status(), + StatusCode::OK + ); + match resolve(world, name) { + Resolved::Ready(Some(did)) => did, + other => panic!("repo {name} wasn't registered: {other:?}"), + } +} + +async fn create_status( + world: &World, + signer: &K256Signer, + host: &str, + value: serde_json::Value, +) -> StatusCode { + call( + world, + crate::repos::create_repo, + signer, + host, + CREATE, + value, + ) + .await + .status() +} + +async fn reserve_status(world: &World, signer: &K256Signer, host: &str, did: &str) -> StatusCode { + call( + world, + crate::repos::reserve_key, + signer, + host, + RESERVE, + json!({ "repoDid": did }), + ) + .await + .status() +} + +async fn reserve_repo_key(world: &World, did_web: &str) -> String { + let response = as_member( + world, + crate::repos::reserve_key, + RESERVE, + json!({ "repoDid": did_web }), + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + let key = json_of(response).await["key"].as_str().unwrap().to_string(); + let host = did_web.strip_prefix("did:web:").unwrap(); + world.publish_repo_doc(host, &key); + key +} + +async fn rename_repo_as( + world: &World, + signer: &K256Signer, + host: &str, + repo: &RepoDid, + rkey: &str, +) -> StatusCode { + call( + world, + crate::repos::rename_repo, + signer, + host, + RENAME, + json!({ "repo": repo.as_str(), "rkey": rkey, "name": rkey }), + ) + .await + .status() +} + +#[tokio::test] +async fn admission_gates_repo_creation() { + let closed = World::new(); + let make = || json!({ "rkey": "anemone", "name": "anemone" }); + assert_eq!( + create_status(&closed, &closed.stranger, STRANGER_HOST, make()).await, + StatusCode::FORBIDDEN, + "a closed knot denies a stranger" + ); + + let open = World::open(); + assert_eq!( + create_status(&open, &open.stranger, STRANGER_HOST, make()).await, + StatusCode::OK + ); + assert!( + matches!( + open.state.index.resolve_repo( + &OwnerDid::new(format!("did:web:{STRANGER_HOST}")).unwrap(), + &RepoRkey::new("anemone").unwrap() + ), + Resolved::Ready(Some(_)) + ), + "an open knot registers the stranger's repo without membership" + ); +} + +#[tokio::test] +async fn blocklist_lifecycle() { + let world = World::open(); + let subject = format!("did:web:{STRANGER_HOST}"); + let make = || json!({ "rkey": "anemone", "name": "anemone" }); + + assert_eq!( + as_admin( + &world, + crate::blocklist::ban, + BAN, + json!({ "subject": subject }) + ) + .await + .status(), + StatusCode::OK + ); + assert!(matches!( + world.state.index.is_blocked(&account(STRANGER_HOST)), + Resolved::Ready(true) + )); + assert_eq!( + create_status(&world, &world.stranger, STRANGER_HOST, make()).await, + StatusCode::FORBIDDEN, + "a banned account cannot create" + ); + + assert_eq!( + as_admin( + &world, + crate::blocklist::unban, + UNBAN, + json!({ "subject": subject }) + ) + .await + .status(), + StatusCode::OK + ); + assert!(matches!( + world.state.index.is_blocked(&account(STRANGER_HOST)), + Resolved::Ready(false) + )); + assert_eq!( + create_status(&world, &world.stranger, STRANGER_HOST, make()).await, + StatusCode::OK, + "an unban restores creation" + ); + + assert_eq!( + as_admin( + &world, + crate::blocklist::ban, + BAN, + json!({ "subject": format!("did:web:{ADMIN_HOST}") }) + ) + .await + .status(), + StatusCode::FORBIDDEN, + "an admin cannot be banned" + ); + assert_eq!( + as_stranger( + &world, + crate::blocklist::ban, + BAN, + json!({ "subject": format!("did:web:{MEMBER_HOST}") }) + ) + .await + .status(), + StatusCode::FORBIDDEN, + "a non-admin cannot ban" + ); +} + +#[tokio::test] +async fn member_lifecycle() { + let world = World::new(); + let subject = json!({ "subject": format!("did:web:{MEMBER_HOST}") }); + + assert_eq!( + as_admin( + &world, + crate::members::add_member, + ADD_MEMBER, + subject.clone() + ) + .await + .status(), + StatusCode::OK + ); + assert_eq!( + world.state.index.is_member(&account(MEMBER_HOST)), + Resolved::Ready(true), + "member is effective on the very next read, with no firehose" + ); + let events = replay(&world); + let [added] = events.as_slice() else { + panic!("expected exactly one event, got {}", events.len()); + }; + assert_eq!(added.nsid, "sh.tangled.knot.memberUpdate"); + assert_eq!(added.payload["op"], "add"); + assert_eq!(added.payload["subject"], account(MEMBER_HOST).to_string()); + + let baseline = event_count(&world); + assert_eq!( + as_admin( + &world, + crate::members::add_member, + ADD_MEMBER, + subject.clone() + ) + .await + .status(), + StatusCode::OK + ); + assert_eq!( + event_count(&world), + baseline, + "a redundant add is a no-op and emits no event" + ); + + assert_eq!( + as_admin( + &world, + crate::members::remove_member, + REMOVE_MEMBER, + subject.clone() + ) + .await + .status(), + StatusCode::OK + ); + assert_eq!( + world.state.index.is_member(&account(MEMBER_HOST)), + Resolved::Ready(false), + "removed member is gone on the very next read" + ); + let removed = last_event(&world, "sh.tangled.knot.memberUpdate"); + assert_eq!(removed.payload["op"], "remove"); + assert_eq!(removed.payload["subject"], account(MEMBER_HOST).to_string()); + + let baseline = event_count(&world); + assert_eq!( + as_admin( + &world, + crate::members::remove_member, + REMOVE_MEMBER, + subject + ) + .await + .status(), + StatusCode::OK + ); + assert_eq!( + event_count(&world), + baseline, + "removing a non-member is a no-op and emits no event" + ); +} + +#[tokio::test] +async fn add_member_auth_outcomes() { + struct Case { + headers: HeaderMap, + body: serde_json::Value, + status: StatusCode, + why: &'static str, + } + + let world = World::new(); + let lowercase = { + let token = mint(&world.admin, &account(ADMIN_HOST), ADD_MEMBER); + let mut headers = HeaderMap::new(); + headers.insert( + AUTHORIZATION, + HeaderValue::from_str(&format!("bearer {token}")).unwrap(), + ); + headers + }; + let cases = vec![ + Case { + headers: bearer(&mint(&world.member, &account(MEMBER_HOST), ADD_MEMBER)), + body: json!({ "subject": "did:web:olaren.dev" }), + status: StatusCode::FORBIDDEN, + why: "a non-admin cannot add a member", + }, + Case { + headers: HeaderMap::new(), + body: json!({ "subject": format!("did:web:{MEMBER_HOST}") }), + status: StatusCode::UNAUTHORIZED, + why: "a request without a token is unauthorized", + }, + Case { + headers: bearer(&mint(&world.admin, &account(ADMIN_HOST), ADD_MEMBER)), + body: json!({ "subject": "not-a-did" }), + status: StatusCode::BAD_REQUEST, + why: "an invalid DID is rejected at decode by the newtype Deserialize, never reaching a handler", + }, + Case { + headers: lowercase, + body: json!({ "subject": "did:web:olaren.dev" }), + status: StatusCode::OK, + why: "the bearer scheme is matched case-insensitively per RFC 7235", + }, + ]; + + let world_ref = &world; + futures::stream::iter(cases) + .for_each(|case| async move { + let status = into_response( + crate::members::add_member( + world_ref.state(), + case.headers, + crate::Method::from_nsid(ADD_MEMBER), + body(case.body), + ) + .await, + ) + .status(); + assert_eq!(status, case.status, "{}", case.why); + }) + .await; +} + +#[tokio::test] +async fn a_transient_upstream_identity_failure_is_a_503_named_upstream_unavailable() { + let responder: Responder = Box::new(|_request: &HttpRequest| { + Ok(HttpResponse { + status: StatusCode::INTERNAL_SERVER_ERROR, + headers: http::HeaderMap::new(), + body: Bytes::new(), + }) + }); + let (_dir, state) = build_state(responder, true); + let admin = signer(1); + let token = mint(&admin, &account(ADMIN_HOST), ADD_MEMBER); + let response = into_response( + crate::members::add_member( + State(state), + bearer(&token), + crate::Method::from_nsid(ADD_MEMBER), + body(json!({ "subject": format!("did:web:{MEMBER_HOST}") })), + ) + .await, + ); + assert_eq!( + response.status(), + StatusCode::SERVICE_UNAVAILABLE, + "transient issuer-doc failure is 503, distinct from a bad token's 401" + ); + assert_eq!( + json_of(response).await["error"], + "UpstreamUnavailable", + "transient upstream identity failure is named distinctly from a warming projection" + ); +} + +#[tokio::test] +async fn concurrent_first_member_adds_converge_to_a_single_cob_object() { + use knot_cob::CobStore; + use knot_cobs::MembersCob; + use knot_git::Repo; + + let world = World::new(); + let (left, right) = tokio::join!( + as_admin( + &world, + crate::members::add_member, + ADD_MEMBER, + json!({ "subject": "did:web:witchcraft.systems" }) + ), + as_admin( + &world, + crate::members::add_member, + ADD_MEMBER, + json!({ "subject": "did:web:isabelroses.com" }) + ), + ); + assert_eq!(left.status(), StatusCode::OK); + assert_eq!(right.status(), StatusCode::OK); + + let meta = Repo::open(world.state.meta_path.clone()).unwrap(); + assert_eq!( + CobStore::new(&meta).list::().unwrap().len(), + 1, + "concurrent first adds serialize onto one singleton members COB, never splitting it" + ); + assert_eq!( + world.state.index.is_member(&account("witchcraft.systems")), + Resolved::Ready(true) + ); + assert_eq!( + world.state.index.is_member(&account("isabelroses.com")), + Resolved::Ready(true) + ); +} + +#[tokio::test] +async fn re_adding_a_member_under_warming_appends_no_redundant_change() { + use knot_cob::CobStore; + use knot_cobs::{Grant, MembersChange, MembersCob}; + use knot_git::Repo; + + let admin = signer(1); + let responder = doc_responder(admin.public_key().as_bytes().to_vec(), || StatusCode::OK); + let (_dir, state) = build_state(responder, false); + + let now = state.now(); + let knot_signer = state.secrets.signer(&state.knot_did).unwrap(); + let subject = account(MEMBER_HOST); + let meta = Repo::open(state.meta_path.clone()).unwrap(); + let created = CobStore::new(&meta) + .create( + &knot_cob::CobHome::from(&state.knot_did), + &MembersChange::Add(Grant { + subject: subject.clone(), + added_by: account(ADMIN_HOST), + created_at: now, + }), + &knot_signer, + now, + ) + .unwrap(); + + let token = mint(&admin, &account(ADMIN_HOST), ADD_MEMBER); + assert_eq!( + into_response( + crate::members::add_member( + State(Arc::clone(&state)), + bearer(&token), + crate::Method::from_nsid(ADD_MEMBER), + body(json!({ "subject": subject.as_str() })), + ) + .await + ) + .status(), + StatusCode::OK + ); + + let meta = Repo::open(state.meta_path.clone()).unwrap(); + let delta = CobStore::new(&meta) + .changes_since::(created.object, Some(created.tip)) + .unwrap(); + assert!( + delta.changes.is_empty(), + "re-adding an existing member appends no change, even while projection is warming" + ); +} + +#[tokio::test] +async fn create_mints_a_did_plc_repo_and_refuses_a_duplicate_name() { + let world = World::new(); + add_member_helper(&world).await; + + let repo_did = create_repo_helper(&world, "anemone").await; + assert!( + repo_did.as_str().starts_with("did:plc:"), + "knot minted a did:plc identity for the repo" + ); + assert!( + world.layout.open(&repo_did).is_ok(), + "bare repo exists on disk under its minted DID" + ); + assert_eq!( + world.state.secrets.len(), + 1, + "only the shared knot key is sealed" + ); + + assert_eq!( + create_status( + &world, + &world.member, + MEMBER_HOST, + json!({ "rkey": "anemone", "name": "anemone" }) + ) + .await, + StatusCode::CONFLICT, + "a second repo of the same name is refused, never silently overwritten" + ); + assert_eq!( + resolve(&world, "anemone"), + Resolved::Ready(Some(repo_did.clone())), + "the original repo still owns the name" + ); + assert!( + world.layout.open(&repo_did).is_ok(), + "the original repo is untouched on disk" + ); +} + +#[tokio::test] +async fn create_and_reserve_reject_bad_identities() { + let world = World::new(); + add_member_helper(&world).await; + + assert_eq!( + create_status( + &world, + &world.member, + MEMBER_HOST, + json!({ "rkey": "a", "name": "a", "repoDid": "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa" }) + ) + .await, + StatusCode::BAD_REQUEST, + "a did:plc cannot be brought; the knot mints those itself" + ); + + assert_eq!( + create_status( + &world, + &world.member, + MEMBER_HOST, + json!({ "rkey": "evil", "name": "evil", "repoDid": format!("did:web:{KNOT_HOST}") }) + ) + .await, + StatusCode::BAD_REQUEST, + "the knot's own DID as a repoDid is a client error instead of a 500" + ); + assert_eq!( + world.state.index.is_member(&account(MEMBER_HOST)), + Resolved::Ready(true), + "the meta-repo is intact" + ); + + assert_eq!( + create_status( + &world, + &world.member, + MEMBER_HOST, + json!({ "rkey": "uni", "name": "uni", "repoDid": "did:web:uni.olaren.dev" }) + ) + .await, + StatusCode::BAD_REQUEST, + "a did:web create without a prior reserveKey is refused" + ); + assert_eq!( + resolve(&world, "uni"), + Resolved::Ready(None), + "nothing was registered for the unproven did:web" + ); + + assert_eq!( + reserve_status( + &world, + &world.member, + MEMBER_HOST, + &format!("did:web:{KNOT_HOST}") + ) + .await, + StatusCode::BAD_REQUEST, + "the knot's own DID cannot have a repo key reserved against it" + ); + + let unpublished = "did:web:conch.olaren.dev"; + assert_eq!( + reserve_status(&world, &world.member, MEMBER_HOST, unpublished).await, + StatusCode::OK + ); + let impostor = knot_types::crypto::multikey(0xe7, signer(99).public_key().as_bytes()); + world.publish_repo_doc("conch.olaren.dev", &impostor); + assert_eq!( + create_status( + &world, + &world.member, + MEMBER_HOST, + json!({ "rkey": "conch", "name": "conch", "repoDid": unpublished }) + ) + .await, + StatusCode::BAD_REQUEST, + "a did:web whose document publishes a different key fails the control proof" + ); + assert!( + world + .layout + .open(&RepoDid::new(unpublished).unwrap()) + .is_err(), + "the repo was never created on disk" + ); + + let victim = "did:web:victim.olaren.dev"; + let reserved = reserve_repo_key(&world, victim).await; + assert_eq!( + reserve_repo_key(&world, victim).await, + reserved, + "re-reserving returns the same key so an already-published document stays valid" + ); + assert_eq!( + create_status( + &world, + &world.admin, + ADMIN_HOST, + json!({ "rkey": "victim", "name": "victim", "repoDid": victim }) + ) + .await, + StatusCode::BAD_REQUEST, + "only the account that reserved the did:web may create it" + ); + assert!( + world.layout.open(&RepoDid::new(victim).unwrap()).is_err(), + "the hijack attempt created no repo on disk" + ); + + let before = world.state.secrets.len(); + assert_eq!( + create_status( + &world, + &world.member, + MEMBER_HOST, + json!({ "rkey": "doomed", "name": "doomed", "defaultBranch": "bad..name" }) + ) + .await, + StatusCode::BAD_REQUEST + ); + assert_eq!( + world.state.secrets.len(), + before, + "an invalid branch is rejected before any key is minted, sealed, or DID submitted" + ); +} + +#[tokio::test] +async fn a_byo_did_web_repo_is_accepted_and_its_key_is_returned() { + let world = World::new(); + add_member_helper(&world).await; + let did = "did:web:nautilus.olaren.dev"; + let reserved_key = reserve_repo_key(&world, did).await; + + let response = as_member( + &world, + crate::repos::create_repo, + CREATE, + json!({ "rkey": "nautilus", "name": "nautilus", "repoDid": did }), + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + let output = json_of(response).await; + assert_eq!(output["repoDid"].as_str(), Some(did)); + + let repo_did = RepoDid::new(did).unwrap(); + assert!( + world.layout.open(&repo_did).is_ok(), + "bring-your-own did:web repo is on disk" + ); + let knot_public = world + .state + .secrets + .public_key(&world.state.knot_did) + .unwrap(); + let expected_key = knot_types::crypto::multikey(0xe7, knot_public.as_bytes()); + assert_eq!( + expected_key, reserved_key, + "reserve returns the knot key the owner publishes in their did:web document" + ); + assert_eq!( + output["key"].as_str(), + Some(expected_key.as_str()), + "create returns the knot-held key the owner published in their own did:web document" + ); + + assert_eq!( + as_member( + &world, + crate::collaborators::add_collaborator, + ADD_COLLAB, + json!({ "repo": did, "subject": "did:web:witchcraft.systems" }) + ) + .await + .status(), + StatusCode::OK + ); + assert_eq!( + world + .state + .index + .is_collaborator(&repo_did, &account("witchcraft.systems")), + Resolved::Ready(true), + "the collaborator COB signed by the knot-held repo key lands and is visible" + ); + + let squid = "did:web:squid.olaren.dev"; + let victim = RepoDid::new(squid).unwrap(); + world.layout.create(&victim).unwrap(); + reserve_repo_key(&world, squid).await; + assert_eq!( + create_status( + &world, + &world.member, + MEMBER_HOST, + json!({ "rkey": "anemone", "name": "anemone", "repoDid": squid }) + ) + .await, + StatusCode::CONFLICT + ); + assert!( + world.layout.open(&victim).is_ok(), + "a colliding create mustn't delete the repository already on disk" + ); +} + +#[tokio::test] +async fn a_rejected_plc_submission_is_a_bad_gateway() { + let admin = signer(1); + let key = admin.public_key().as_bytes().to_vec(); + let responder = doc_responder(key, || StatusCode::BAD_REQUEST); + let (_dir, state) = build_state(responder, true); + let token = mint(&admin, &account(ADMIN_HOST), CREATE); + let status = into_response( + crate::repos::create_repo( + State(Arc::clone(&state)), + bearer(&token), + crate::Method::from_nsid(CREATE), + body(json!({ "rkey": "conch", "name": "conch" })), + ) + .await, + ) + .status(); + assert_eq!( + status, + StatusCode::BAD_GATEWAY, + "non-transient PLC rejection surfaces as 502, distinct from an internal 500" + ); + assert_eq!( + state.secrets.len(), + 1, + "rejected PLC submission rolls back the minted repo key, leaving only the knot's own. Unpublished did:plc orphans nothing" + ); + assert_eq!( + state.index.resolve_repo( + &OwnerDid::new(format!("did:web:{ADMIN_HOST}")).unwrap(), + &RepoRkey::new("conch").unwrap() + ), + Resolved::Ready(None), + "repo isn't registered after a rejected PLC submission" + ); +} + +#[tokio::test] +async fn a_rejected_plc_submission_never_touches_the_registry() { + let admin = signer(1); + let key = admin.public_key().as_bytes().to_vec(); + let reject_posts = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let reject = Arc::clone(&reject_posts); + let responder = doc_responder(key, move || { + if reject.load(Ordering::Relaxed) { + StatusCode::BAD_REQUEST + } else { + StatusCode::OK + } + }); + let (_dir, state) = build_state(responder, true); + let owner = OwnerDid::new(format!("did:web:{ADMIN_HOST}")).unwrap(); + + let mint_create = || mint(&admin, &account(ADMIN_HOST), CREATE); + let anemone = || body(json!({ "rkey": "anemone", "name": "anemone" })); + assert_eq!( + into_response( + crate::repos::create_repo( + State(Arc::clone(&state)), + bearer(&mint_create()), + crate::Method::from_nsid(CREATE), + anemone(), + ) + .await + ) + .status(), + StatusCode::OK + ); + let victim = match state + .index + .resolve_repo(&owner, &RepoRkey::new("anemone").unwrap()) + { + Resolved::Ready(Some(did)) => did, + other => panic!("victim repo wasn't registered: {other:?}"), + }; + + let token = mint(&admin, &account(ADMIN_HOST), RENAME); + assert_eq!( + into_response( + crate::repos::rename_repo( + State(Arc::clone(&state)), + bearer(&token), + crate::Method::from_nsid(RENAME), + body(json!({ "repo": victim.as_str(), "rkey": "barnacle", "name": "barnacle" })), + ) + .await + ) + .status(), + StatusCode::OK + ); + + reject_posts.store(true, Ordering::Relaxed); + assert_eq!( + into_response( + crate::repos::create_repo( + State(Arc::clone(&state)), + bearer(&mint_create()), + crate::Method::from_nsid(CREATE), + anemone(), + ) + .await + ) + .status(), + StatusCode::BAD_GATEWAY + ); + + assert_eq!( + state + .index + .resolve_repo(&owner, &RepoRkey::new("anemone").unwrap()), + Resolved::Ready(Some(victim.clone())), + "stale alias still resolves to its prior holder because the failed create never registered" + ); + + state.index.refresh_registry().unwrap(); + assert_eq!( + state + .index + .resolve_repo(&owner, &RepoRkey::new("anemone").unwrap()), + Resolved::Ready(Some(victim.clone())), + "durable registry COB has no trace of the failed create" + ); + assert_eq!( + state.index.rkey_of(&victim), + Resolved::Ready(Some(RepoRkey::new("barnacle").unwrap())), + "victim's canonical rkey is unmoved" + ); +} + +#[tokio::test] +async fn resolve_by_name_matches_the_rkey_case_sensitively() { + let admin = signer(1); + let key = admin.public_key().as_bytes().to_vec(); + let responder = doc_responder(key, || StatusCode::OK); + let (_dir, state) = build_state(responder, true); + let owner = OwnerDid::new(format!("did:web:{ADMIN_HOST}")).unwrap(); + + assert_eq!( + into_response( + crate::repos::create_repo( + State(Arc::clone(&state)), + bearer(&mint(&admin, &account(ADMIN_HOST), CREATE)), + crate::Method::from_nsid(CREATE), + body(json!({ "rkey": "anemone", "name": "anemone" })), + ) + .await + ) + .status(), + StatusCode::OK + ); + + assert!( + crate::merge::resolve_by_name(&*state, &owner, "anemone").is_ok(), + "the exact rkey resolves" + ); + assert!( + crate::merge::resolve_by_name(&*state, &owner, "Anemone").is_err(), + "a differently-cased name must not resolve to a distinct rkey, atproto record keys are case-sensitive" + ); +} + +#[tokio::test] +async fn reserve_key_refuses_once_the_pending_limit_is_reached() { + let world = World::with_pending_limit(2); + add_member_helper(&world).await; + + assert_eq!( + reserve_status(&world, &world.member, MEMBER_HOST, "did:web:p0.olaren.dev").await, + StatusCode::OK, + "first reservation is within the limit" + ); + assert_eq!( + reserve_status(&world, &world.member, MEMBER_HOST, "did:web:p1.olaren.dev").await, + StatusCode::OK, + "second reservation reaches the limit" + ); + assert_eq!( + reserve_status(&world, &world.member, MEMBER_HOST, "did:web:p2.olaren.dev").await, + StatusCode::TOO_MANY_REQUESTS, + "member cannot grow the sealed store without bound past the pending-reservation limit" + ); +} + +#[tokio::test] +async fn one_account_cannot_exhaust_the_global_reservation_budget() { + let world = World::with_limits(256, 2); + add_member_helper(&world).await; + + let world_ref = &world; + futures::stream::iter(["did:web:m0.olaren.dev", "did:web:m1.olaren.dev"]) + .for_each(|did| async move { + assert_eq!( + reserve_status(world_ref, &world_ref.member, MEMBER_HOST, did).await, + StatusCode::OK + ); + }) + .await; + assert_eq!( + reserve_status(&world, &world.member, MEMBER_HOST, "did:web:m2.olaren.dev").await, + StatusCode::TOO_MANY_REQUESTS, + "member is held to its per-actor reservation budget" + ); + assert_eq!( + reserve_status( + &world, + &world.admin, + ADMIN_HOST, + "did:web:admin0.olaren.dev" + ) + .await, + StatusCode::OK, + "a different account keeps its own budget while global capacity remains" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn concurrent_reserve_key_calls_all_succeed() { + let world = World::new(); + add_member_helper(&world).await; + + let handles: Vec<_> = (0..24) + .map(|i| { + let state = world.state(); + let token = mint(&world.member, &account(MEMBER_HOST), RESERVE); + let payload = body(json!({ "repoDid": format!("did:web:r{i}.olaren.dev") })); + tokio::spawn(async move { + into_response( + crate::repos::reserve_key( + state, + bearer(&token), + crate::Method::from_nsid(RESERVE), + payload, + ) + .await, + ) + .status() + }) + }) + .collect(); + + futures::future::join_all(handles) + .await + .into_iter() + .for_each(|result| { + assert_eq!( + result.unwrap(), + StatusCode::OK, + "concurrent reserveKey mustn't race the in-memory reservation map" + ) + }); +} + +#[tokio::test] +async fn collaborator_lifecycle() { + let world = World::new(); + add_member_helper(&world).await; + let repo_did = create_repo_helper(&world, "scallop").await; + let subject = || json!({ "repo": repo_did.as_str(), "subject": "did:web:witchcraft.systems" }); + + assert_eq!( + as_member( + &world, + crate::collaborators::add_collaborator, + ADD_COLLAB, + subject() + ) + .await + .status(), + StatusCode::OK + ); + assert_eq!( + world + .state + .index + .is_collaborator(&repo_did, &account("witchcraft.systems")), + Resolved::Ready(true) + ); + let added = last_event(&world, "sh.tangled.repo.collaboratorUpdate"); + assert_eq!(added.payload["op"], "add"); + assert_eq!( + added.payload["subject"], + account("witchcraft.systems").to_string() + ); + assert_eq!(added.payload["repo"], repo_did.to_string()); + + assert_eq!( + as_member( + &world, + crate::collaborators::remove_collaborator, + REMOVE_COLLAB, + subject() + ) + .await + .status(), + StatusCode::OK + ); + assert_eq!( + world + .state + .index + .is_collaborator(&repo_did, &account("witchcraft.systems")), + Resolved::Ready(false), + "removed collaborator is gone on the very next read" + ); + let removed = last_event(&world, "sh.tangled.repo.collaboratorUpdate"); + assert_eq!(removed.payload["op"], "remove"); + assert_eq!( + removed.payload["subject"], + account("witchcraft.systems").to_string() + ); + assert_eq!(removed.payload["repo"], repo_did.to_string()); + + let baseline = event_count(&world); + assert_eq!( + as_member( + &world, + crate::collaborators::remove_collaborator, + REMOVE_COLLAB, + subject() + ) + .await + .status(), + StatusCode::OK + ); + assert_eq!( + event_count(&world), + baseline, + "removing a non-collaborator is a no-op and emits no event" + ); +} + +#[tokio::test] +async fn repo_management_is_owner_or_collaborator_gated() { + let world = World::new(); + add_member_helper(&world).await; + let repo_did = create_repo_helper(&world, "squid").await; + let at = format!("at://did:web:{MEMBER_HOST}/sh.tangled.repo/squid"); + let did = format!("did:web:{MEMBER_HOST}"); + + assert_eq!( + as_admin( + &world, + crate::collaborators::add_collaborator, + ADD_COLLAB, + json!({ "repo": repo_did.as_str(), "subject": "did:web:isabelroses.com" }) + ) + .await + .status(), + StatusCode::FORBIDDEN, + "collaborator management is the repo owner's right instead of a knot admin's" + ); + assert_eq!( + as_stranger( + &world, + crate::branches::set_default_branch, + SET_DEFAULT, + json!({ "repo": at, "defaultBranch": "trunk" }) + ) + .await + .status(), + StatusCode::FORBIDDEN, + "a stranger cannot set the default branch" + ); + assert_eq!( + as_stranger( + &world, + crate::branches::delete_branch, + DELETE_BRANCH, + json!({ "repo": at, "branch": "trunk" }) + ) + .await + .status(), + StatusCode::FORBIDDEN, + "a stranger cannot delete a branch" + ); + assert_eq!( + as_stranger( + &world, + crate::repos::delete_repo, + DELETE, + json!({ "did": did, "name": "squid", "rkey": "squid" }) + ) + .await + .status(), + StatusCode::FORBIDDEN, + "neither owner nor a knot admin, so delete is refused" + ); + + assert_eq!( + rename_repo_as(&world, &world.stranger, STRANGER_HOST, &repo_did, "stolen").await, + StatusCode::FORBIDDEN + ); + assert_eq!( + world.state.index.rkey_of(&repo_did), + Resolved::Ready(Some(RepoRkey::new("squid").unwrap())), + "the canonical rkey is untouched by the rejected rename" + ); + + assert_eq!( + as_member( + &world, + crate::collaborators::add_collaborator, + ADD_COLLAB, + json!({ "repo": repo_did.as_str(), "subject": format!("did:web:{STRANGER_HOST}") }) + ) + .await + .status(), + StatusCode::OK + ); + assert_eq!( + rename_repo_as( + &world, + &world.stranger, + STRANGER_HOST, + &repo_did, + "periwinkle" + ) + .await, + StatusCode::OK, + "rename is gated by can_push, so a collaborator may rename" + ); +} + +#[tokio::test] +async fn the_owner_sets_the_default_branch_through_an_at_uri() { + let world = World::new(); + add_member_helper(&world).await; + let repo_did = create_repo_helper(&world, "mussel").await; + + assert_eq!( + as_member(&world, crate::branches::set_default_branch, SET_DEFAULT, json!({ "repo": format!("at://did:web:{MEMBER_HOST}/sh.tangled.repo/mussel"), "defaultBranch": "trunk" })).await.status(), + StatusCode::OK + ); + + let repo = world.layout.open(&repo_did).unwrap(); + assert_eq!( + repo.default_branch().map(|name| name.as_str().to_string()), + Some("refs/heads/trunk".to_string()), + "HEAD now points at the requested default branch" + ); + + let event = only_git_event(&world); + assert_eq!(event.nsid, "sh.tangled.git.refUpdate"); + assert_eq!(event.payload["repo"], repo_did.to_string()); + assert_eq!(event.payload["ownerDid"], account(MEMBER_HOST).to_string()); + assert_eq!( + event.payload["committerDid"], + account(MEMBER_HOST).to_string(), + "the actor who set the default branch is the committer on the wire" + ); +} + +#[tokio::test] +async fn set_default_branch_rejections() { + use knot_cob::CobStore; + use knot_cobs::{CollaboratorsChange, Grant}; + use knot_git::RefUpdate; + use knot_types::RefName; + + let world = World::new(); + add_member_helper(&world).await; + let repo_did = create_repo_helper(&world, "mussel").await; + + let git = world.layout.open(&repo_did).unwrap(); + let now = world.state.now(); + let signer = world.state.secrets.signer(&world.state.knot_did).unwrap(); + let created = CobStore::new(&git) + .create( + &knot_cob::CobHome::from(&repo_did), + &CollaboratorsChange::Add(Grant { + subject: account("olaren.dev"), + added_by: account("olaren.dev"), + created_at: now, + }), + &signer, + now, + ) + .unwrap(); + git.update_ref(&RefUpdate::Create { + name: RefName::new("refs/heads/main").unwrap(), + new: created.tip.oid(), + }) + .unwrap(); + + assert_eq!( + as_member(&world, crate::branches::set_default_branch, SET_DEFAULT, json!({ "repo": format!("at://did:web:{MEMBER_HOST}/sh.tangled.notrepo/mussel"), "defaultBranch": "trunk" })).await.status(), + StatusCode::BAD_REQUEST, + "an at-uri addressing a collection other than sh.tangled.repo is rejected" + ); + assert_eq!( + as_member(&world, crate::branches::set_default_branch, SET_DEFAULT, json!({ "repo": format!("at://did:web:{MEMBER_HOST}/sh.tangled.repo/mussel"), "defaultBranch": "ghost" })).await.status(), + StatusCode::NOT_FOUND, + "a populated repo rejects a default pointing at a branch that doesn't exist" + ); +} + +#[tokio::test] +async fn delete_branch_removes_a_branch_and_refuses_the_default() { + use knot_cob::CobStore; + use knot_cobs::{CollaboratorsChange, Grant}; + use knot_git::RefUpdate; + use knot_types::RefName; + + let world = World::new(); + add_member_helper(&world).await; + let repo_did = create_repo_helper(&world, "periwinkle").await; + let git = world.layout.open(&repo_did).unwrap(); + let now = world.state.now(); + let signer = world.state.secrets.signer(&world.state.knot_did).unwrap(); + let created = CobStore::new(&git) + .create( + &knot_cob::CobHome::from(&repo_did), + &CollaboratorsChange::Add(Grant { + subject: account("olaren.dev"), + added_by: account("olaren.dev"), + created_at: now, + }), + &signer, + now, + ) + .unwrap(); + let oid = created.tip.oid(); + ["refs/heads/main", "refs/heads/trunk"] + .into_iter() + .for_each(|name| { + git.update_ref(&RefUpdate::Create { + name: RefName::new(name).unwrap(), + new: oid, + }) + .unwrap(); + }); + git.set_head(&RefName::new("refs/heads/main").unwrap()) + .unwrap(); + + let at = format!("at://did:web:{MEMBER_HOST}/sh.tangled.repo/periwinkle"); + assert_eq!( + as_member( + &world, + crate::branches::delete_branch, + DELETE_BRANCH, + json!({ "repo": at, "branch": "trunk" }) + ) + .await + .status(), + StatusCode::OK, + "a non-default branch is deleted" + ); + assert!( + git.find_ref(&RefName::new("refs/heads/trunk").unwrap()) + .unwrap() + .is_none(), + "trunk is gone" + ); + assert_eq!( + as_member( + &world, + crate::branches::delete_branch, + DELETE_BRANCH, + json!({ "repo": at, "branch": "main" }) + ) + .await + .status(), + StatusCode::BAD_REQUEST, + "the current default branch cannot be deleted" + ); + + let event = only_git_event(&world); + assert_eq!(event.nsid, "sh.tangled.git.refUpdate"); + assert_eq!(event.payload["repo"], repo_did.to_string()); + assert_eq!(event.payload["ref"], "refs/heads/trunk"); + assert_eq!( + event.payload["oldSha"], + oid.to_string(), + "the deletion event includes the branch's old tip" + ); + assert_eq!( + event.payload["newSha"], + git.object_format().null_oid().to_string(), + "deletion reports the null oid as the new sha" + ); + assert_eq!( + event.payload["committerDid"], + account(MEMBER_HOST).to_string() + ); +} + +#[tokio::test] +async fn delete_repo_lifecycle_and_guards() { + let world = World::new(); + add_member_helper(&world).await; + let did = format!("did:web:{MEMBER_HOST}"); + + let plain = create_repo_helper(&world, "whelk").await; + assert_eq!( + as_member( + &world, + crate::repos::delete_repo, + DELETE, + json!({ "did": did, "name": "whelk", "rkey": "whelk" }) + ) + .await + .status(), + StatusCode::OK + ); + assert!( + world.layout.open(&plain).is_err(), + "bare repo is removed from disk" + ); + assert_eq!( + resolve(&world, "whelk"), + Resolved::Ready(None), + "repo is deregistered" + ); + + let guarded = create_repo_helper(&world, "conch").await; + world.publish_pds_record("conch"); + let delete_conch = || json!({ "did": did, "name": "conch", "rkey": "conch" }); + assert_eq!( + as_member(&world, crate::repos::delete_repo, DELETE, delete_conch()) + .await + .status(), + StatusCode::CONFLICT, + "the guard refuses while the sh.tangled.repo record is still on the owner's PDS" + ); + assert!( + world.layout.open(&guarded).is_ok(), + "a refused delete left the repo intact on disk" + ); + + let force_conch = || json!({ "did": did, "name": "conch", "rkey": "conch", "force": true }); + assert_eq!( + as_member(&world, crate::repos::delete_repo, DELETE, force_conch()) + .await + .status(), + StatusCode::FORBIDDEN, + "force is an admin-only escape hatch instead of the owner's" + ); + assert_eq!( + as_admin(&world, crate::repos::delete_repo, DELETE, force_conch()) + .await + .status(), + StatusCode::OK, + "a knot admin forces the delete past the lingering PDS record" + ); + assert!( + world.layout.open(&guarded).is_err(), + "forced delete removed the repo from disk" + ); +} + +#[tokio::test] +async fn rename_alias_lifecycle() { + let world = World::new(); + add_member_helper(&world).await; + + let repo_a = create_repo_helper(&world, "alpha").await; + assert_eq!( + rename_repo_as(&world, &world.member, MEMBER_HOST, &repo_a, "alphanew").await, + StatusCode::OK + ); + assert_eq!( + resolve(&world, "alphanew"), + Resolved::Ready(Some(repo_a.clone())), + "the new rkey resolves on the very next read" + ); + assert_eq!( + resolve(&world, "alpha"), + Resolved::Ready(Some(repo_a.clone())), + "the prior rkey keeps resolving as an alias" + ); + assert_eq!( + world.state.index.rkey_of(&repo_a), + Resolved::Ready(Some(RepoRkey::new("alphanew").unwrap())), + "the new rkey is canonical" + ); + + assert_eq!( + as_member(&world, crate::branches::set_default_branch, SET_DEFAULT, json!({ "repo": format!("at://did:web:{MEMBER_HOST}/sh.tangled.repo/alpha"), "defaultBranch": "trunk" })).await.status(), + StatusCode::OK, + "an at-uri with the pre-rename rkey still reaches the repo" + ); + + let repo_a2 = create_repo_helper(&world, "alpha").await; + assert_ne!(repo_a2, repo_a, "a brand-new repo DID was minted"); + assert_eq!( + resolve(&world, "alpha"), + Resolved::Ready(Some(repo_a2.clone())), + "the reused rkey now resolves to the new repo" + ); + assert_eq!( + resolve(&world, "alphanew"), + Resolved::Ready(Some(repo_a.clone())), + "the renamed repo keeps its canonical rkey" + ); + + let repo_b = create_repo_helper(&world, "beta").await; + assert_eq!( + rename_repo_as(&world, &world.member, MEMBER_HOST, &repo_b, "alpha").await, + StatusCode::CONFLICT, + "a rename cannot take the canonical rkey of another live repo" + ); + assert_eq!( + resolve(&world, "alpha"), + Resolved::Ready(Some(repo_a2)), + "the contested rkey still belongs to its original repo" + ); + + let repo_g = create_repo_helper(&world, "gamma").await; + assert_eq!( + rename_repo_as(&world, &world.member, MEMBER_HOST, &repo_g, "gammanew").await, + StatusCode::OK + ); + assert_eq!( + as_member(&world, crate::repos::delete_repo, DELETE, json!({ "did": format!("did:web:{MEMBER_HOST}"), "name": "gammanew", "rkey": "gammanew" })).await.status(), + StatusCode::OK + ); + assert!( + world.layout.open(&repo_g).is_err(), + "a renamed repo is deleted through its new rkey" + ); + assert_eq!( + resolve(&world, "gamma"), + Resolved::Ready(None), + "deletion drops the retained alias along with the repo" + ); + + let ghost = RepoDid::new("did:web:ghost.nel.pet").unwrap(); + assert_eq!( + rename_repo_as(&world, &world.member, MEMBER_HOST, &ghost, "kelp").await, + StatusCode::NOT_FOUND, + "renaming a repo this knot doesn't host is 404 instead of 403" + ); +} + +#[tokio::test] +async fn a_rename_against_a_warming_registry_is_unavailable_not_forbidden() { + let admin = signer(1); + let responder = doc_responder(admin.public_key().as_bytes().to_vec(), || StatusCode::OK); + let (_dir, state) = build_state(responder, false); + + let token = mint(&admin, &account(ADMIN_HOST), RENAME); + assert_eq!( + into_response( + crate::repos::rename_repo( + State(Arc::clone(&state)), + bearer(&token), + crate::Method::from_nsid(RENAME), + body(json!({ "repo": "did:web:squid.nel.pet", "rkey": "kelp", "name": "kelp" })), + ) + .await + ) + .status(), + StatusCode::SERVICE_UNAVAILABLE, + "a warming registry is a retryable 503, never a permanent 403" + ); +} + +#[tokio::test] +async fn the_router_sheds_a_pre_auth_flood_from_one_peer() { + use axum::body::Body; + use axum::extract::ConnectInfo; + use std::net::SocketAddr; + use tower::ServiceExt; + + let world = World::new(); + let app = crate::router(Arc::clone(&world.state)); + let peer = SocketAddr::from(([203, 0, 113, 7], 5555)); + + let statuses: Vec = futures::stream::iter(0..22) + .then(|_| { + let app = app.clone(); + async move { + let mut request = http::Request::builder() + .method("POST") + .uri(crate::members::ADD_ROUTE) + .body(Body::empty()) + .unwrap(); + request.extensions_mut().insert(ConnectInfo(peer)); + app.oneshot(request).await.unwrap().status() + } + }) + .collect() + .await; + + assert!( + statuses[..20] + .iter() + .all(|status| *status == StatusCode::UNAUTHORIZED), + "per-peer burst is admitted and then fails auth on the missing token, got {statuses:?}" + ); + assert!( + statuses[20..] + .iter() + .all(|status| *status == StatusCode::TOO_MANY_REQUESTS), + "past the burst the router sheds the flood before it can reach the resolver, got {statuses:?}" + ); +} + +#[tokio::test] +async fn the_router_binds_each_route_to_its_matched_method_scope() { + use axum::body::Body; + use axum::extract::ConnectInfo; + use std::net::SocketAddr; + use tower::ServiceExt; + + let world = World::new(); + let app = crate::router(Arc::clone(&world.state)); + let peer = SocketAddr::from(([203, 0, 113, 23], 5555)); + + let post = |token: String| { + let mut request = http::Request::builder() + .method("POST") + .uri(crate::members::ADD_ROUTE) + .header(http::header::AUTHORIZATION, format!("Bearer {token}")) + .body(Body::from( + serde_json::to_vec(&json!({ "subject": format!("did:web:{MEMBER_HOST}") })) + .unwrap(), + )) + .unwrap(); + request.extensions_mut().insert(ConnectInfo(peer)); + request + }; + + let matched = mint(&world.admin, &account(ADMIN_HOST), ADD_MEMBER); + assert_eq!( + app.clone().oneshot(post(matched)).await.unwrap().status(), + StatusCode::OK, + "a token whose lxm is the route's own method authenticates" + ); + + let sibling = mint(&world.admin, &account(ADMIN_HOST), REMOVE_MEMBER); + assert_eq!( + app.oneshot(post(sibling)).await.unwrap().status(), + StatusCode::UNAUTHORIZED, + "a token minted for a sibling method is rejected at the addMember route" + ); +} + +#[tokio::test] +async fn the_http_push_surface_sheds_a_bogus_credential_flood_from_one_peer() { + use axum::body::Body; + use axum::extract::ConnectInfo; + use base64::Engine as _; + use std::net::SocketAddr; + use tower::ServiceExt; + + let world = World::new(); + add_member_helper(&world).await; + let repo = create_repo_helper(&world, "kelp").await; + let app = crate::router(Arc::clone(&world.state)); + let peer = SocketAddr::from(([203, 0, 113, 11], 5555)); + let credential = format!( + "Basic {}", + base64::engine::general_purpose::STANDARD.encode("git:not-a-service-jwt") + ); + + let statuses: Vec = futures::stream::iter(0..22) + .then(|_| { + let app = app.clone(); + let uri = format!("/{}/git-receive-pack", repo.as_str()); + let credential = credential.clone(); + async move { + let mut request = http::Request::builder() + .method("POST") + .uri(uri) + .header(http::header::AUTHORIZATION, credential) + .body(Body::empty()) + .unwrap(); + request.extensions_mut().insert(ConnectInfo(peer)); + app.oneshot(request).await.unwrap().status() + } + }) + .collect() + .await; + + assert!( + statuses[..20] + .iter() + .all(|status| *status == StatusCode::UNAUTHORIZED), + "bogus credentials inside the burst fail authentication, got {statuses:?}" + ); + assert!( + statuses[20..] + .iter() + .all(|status| *status == StatusCode::TOO_MANY_REQUESTS), + "past the burst the push surface sheds the flood before it can reach the resolver, got {statuses:?}" + ); +} + +#[tokio::test] +async fn health_is_unauthenticated_and_exempt_from_shedding() { + use axum::body::Body; + use axum::extract::ConnectInfo; + use std::net::SocketAddr; + use tower::ServiceExt; + + let world = World::new(); + let app = crate::router(Arc::clone(&world.state)); + let peer = SocketAddr::from(([203, 0, 113, 9], 5555)); + let health = || { + let mut request = http::Request::builder() + .method("GET") + .uri(crate::service::HEALTH_ROUTE) + .body(Body::empty()) + .unwrap(); + request.extensions_mut().insert(ConnectInfo(peer)); + request + }; + + let statuses = futures::future::join_all((0..30).map(|_| { + let app = app.clone(); + async move { app.oneshot(health()).await.unwrap() } + })) + .await; + assert!( + statuses + .iter() + .all(|response| response.status() == StatusCode::OK), + "health stays 200 even past the pre-auth burst, got {:?}", + statuses.iter().map(|r| r.status()).collect::>() + ); + + let wire = json_of(app.oneshot(health()).await.unwrap()).await; + assert!( + wire["version"] + .as_str() + .is_some_and(|v| v.starts_with("knot ")), + "health reports a knot version, got {wire}" + ); +} + +mod merge_endpoints { + use super::*; + use std::path::Path; + + use knot_git::{EntryKind, Identity, NewCommit, RefUpdate, StagedAction, StagedChange}; + use knot_types::{Oid, RefName, UnixSeconds}; + + const EMPTY_TREE: &str = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"; + const MERGE: &str = "sh.tangled.repo.merge"; + + const UNIFIED_PATCH: &str = concat!( + "diff --git a/reef.txt b/reef.txt\n", + "index 1111111..2222222 100644\n", + "--- a/reef.txt\n", + "+++ b/reef.txt\n", + "@@ -1 +1 @@\n", + "-old line\n", + "+new line\n", + ); + + const CONFLICTING_PATCH: &str = concat!( + "diff --git a/reef.txt b/reef.txt\n", + "index 1111111..2222222 100644\n", + "--- a/reef.txt\n", + "+++ b/reef.txt\n", + "@@ -1 +1 @@\n", + "-something else entirely\n", + "+new line\n", + ); + + fn seed_main(world: &World, repo_did: &RepoDid, files: &[(&str, &str)]) -> Oid { + let repo = world.layout.open(repo_did).unwrap(); + let staged: Vec = files + .iter() + .map(|(path, content)| StagedChange { + path: knot_types::RepoPath::new(*path).unwrap(), + action: StagedAction::Put { + content: content.as_bytes().to_vec(), + kind: EntryKind::Blob, + }, + }) + .collect(); + let tree = repo + .write_staged_tree(Oid::from_hex(EMPTY_TREE).unwrap(), &staged) + .unwrap(); + let nel = Identity { + name: AuthorName::new("nel"), + email: Email::new("nel@oyster.cafe"), + time: UnixSeconds::new(1_000), + offset_seconds: 0, + }; + let commit = repo + .write_commit(&NewCommit { + tree, + parents: Vec::new(), + author: nel.clone(), + committer: nel, + message: "base".to_string(), + extra_headers: Vec::new(), + }) + .unwrap(); + repo.update_ref(&RefUpdate::Create { + name: RefName::new("refs/heads/main").unwrap(), + new: commit, + }) + .unwrap(); + commit + } + + fn main_tip(world: &World, repo_did: &RepoDid) -> Oid { + world + .layout + .open(repo_did) + .unwrap() + .find_ref(&RefName::new("refs/heads/main").unwrap()) + .unwrap() + .unwrap() + } + + fn file_count(dir: &Path) -> usize { + std::fs::read_dir(dir) + .map(|entries| { + entries + .flatten() + .map(|entry| match entry.file_type() { + Ok(kind) if kind.is_dir() => file_count(&entry.path()), + _ => 1, + }) + .sum() + }) + .unwrap_or(0) + } + + fn blob_at(repo: &knot_git::Repo, commit: Oid, path: &str) -> Vec { + let entry = repo + .entry_at(commit, &knot_types::RepoPath::new(path).unwrap()) + .unwrap() + .unwrap(); + repo.read_blob(entry.oid).unwrap() + } + + #[tokio::test] + async fn the_owner_merges_a_unified_patch_natively_and_cleans_up() { + let world = World::new(); + add_member_helper(&world).await; + let repo_did = create_repo_helper(&world, "kelp").await; + let base = seed_main(&world, &repo_did, &[("reef.txt", "old line\n")]); + + assert_eq!( + as_member( + &world, + crate::merge::merge, + MERGE, + json!({ + "did": format!("did:web:{MEMBER_HOST}"), + "name": "kelp", + "branch": "main", + "patch": UNIFIED_PATCH, + "commitMessage": "Merge tide", + "commitBody": "body text", + "authorName": "bailey", + "authorEmail": "bailey@nel.pet", + }) + ) + .await + .status(), + StatusCode::OK + ); + + let repo = world.layout.open(&repo_did).unwrap(); + let tip = main_tip(&world, &repo_did); + assert_ne!(tip, base); + let commit = repo.find_commit(tip).unwrap(); + assert_eq!(commit.parents, vec![base]); + assert_eq!(commit.author.name.as_str(), "bailey"); + assert_eq!(commit.author.email.as_str(), "bailey@nel.pet"); + assert_eq!(commit.committer.name.as_str(), "Tangled"); + assert_eq!(commit.committer.email.as_str(), "noreply@tangled.sh"); + assert_eq!(commit.message, "Merge tide\n\nbody text\n"); + assert_eq!(blob_at(&repo, tip, "reef.txt"), b"new line\n"); + + let event = only_git_event(&world); + assert_eq!(event.nsid, "sh.tangled.git.refUpdate"); + assert_eq!(event.payload["repo"], repo_did.to_string()); + assert_eq!(event.payload["ref"], "refs/heads/main"); + assert_eq!(event.payload["oldSha"], base.to_string()); + assert_eq!(event.payload["newSha"], tip.to_string()); + assert_eq!( + event.payload["committerDid"], + account(MEMBER_HOST).to_string() + ); + + let staging = std::fs::read_dir(repo.path()) + .unwrap() + .flatten() + .filter(|entry| { + entry + .file_name() + .to_str() + .is_some_and(|name| name.starts_with(knot_git::INCOMING_PREFIX)) + }) + .count(); + assert_eq!( + staging, 0, + "a completed merge cleans up its staging directory" + ); + } + + #[tokio::test] + async fn a_native_merge_advances_the_branch_without_a_pipeline_event() { + let world = World::new(); + add_member_helper(&world).await; + let repo_did = create_repo_helper(&world, "kelp").await; + seed_main( + &world, + &repo_did, + &[ + ("reef.txt", "old line\n"), + ( + ".tangled/workflows/ci.yml", + "engine: nixery.dev/x\nwhen:\n - event: push\n branch: ['**']\n", + ), + ], + ); + + assert_eq!( + as_member( + &world, + crate::merge::merge, + MERGE, + json!({ + "did": format!("did:web:{MEMBER_HOST}"), + "name": "kelp", + "branch": "main", + "patch": UNIFIED_PATCH, + "commitMessage": "Merge tide", + "authorName": "bailey", + "authorEmail": "bailey@nel.pet", + }) + ) + .await + .status(), + StatusCode::OK + ); + + let update = last_event(&world, "sh.tangled.git.refUpdate"); + assert_eq!(update.payload["ref"], "refs/heads/main"); + assert!( + replay(&world) + .iter() + .all(|event| event.nsid != "sh.tangled.pipeline"), + "the knot emits no pipeline record" + ); + } + + #[tokio::test] + async fn a_format_patch_merge_creates_one_commit_per_patch_with_change_id() { + let world = World::new(); + add_member_helper(&world).await; + let repo_did = create_repo_helper(&world, "limpet").await; + let base = seed_main(&world, &repo_did, &[("reef.txt", "one\n")]); + + let mbox = concat!( + "From 1111111111111111111111111111111111111111 Mon Sep 17 00:00:00 2001\n", + "From: olaren \n", + "Date: Tue, 5 Sep 2023 12:00:00 +0530\n", + "Subject: [PATCH 1/2] first step\n", + "\n", + "step one body\n", + "---\n", + " reef.txt | 2 +-\n", + "\n", + "diff --git a/reef.txt b/reef.txt\n", + "index 1111111..2222222 100644\n", + "--- a/reef.txt\n", + "+++ b/reef.txt\n", + "@@ -1 +1 @@\n", + "-one\n", + "+two\n", + "-- \n2.43.0\n\n", + "From 2222222222222222222222222222222222222222 Mon Sep 17 00:00:00 2001\n", + "From: olaren \n", + "Date: Tue, 5 Sep 2023 13:00:00 +0530\n", + "Subject: [PATCH 2/2] second step\n", + "Change-Id: Ifeedfacecafe\n", + "\n", + "---\n", + "diff --git a/reef.txt b/reef.txt\n", + "index 2222222..3333333 100644\n", + "--- a/reef.txt\n", + "+++ b/reef.txt\n", + "@@ -1 +1 @@\n", + "-two\n", + "+three\n", + ); + + assert_eq!( + as_member( + &world, + crate::merge::merge, + MERGE, + json!({ + "did": format!("did:web:{MEMBER_HOST}"), + "name": "limpet", + "branch": "main", + "patch": mbox, + }) + ) + .await + .status(), + StatusCode::OK + ); + + let repo = world.layout.open(&repo_did).unwrap(); + let tip = main_tip(&world, &repo_did); + let second = repo.find_commit(tip).unwrap(); + assert_eq!(second.message, "second step\n"); + assert_eq!( + second.change_id(), + Some(knot_git::CommitChangeId::new("Ifeedfacecafe").unwrap()) + ); + assert_eq!(second.author.name.as_str(), "olaren"); + assert_eq!(second.author.email.as_str(), "olaren@olaren.dev"); + assert_eq!(second.author.time.get(), 1_693_899_000); + assert_eq!(second.author.offset_seconds, 19_800); + assert_eq!(second.committer.name.as_str(), "Tangled"); + assert_eq!(second.committer.email.as_str(), "noreply@tangled.sh"); + + let first = repo.find_commit(second.parents[0]).unwrap(); + assert_eq!(first.message, "first step\n\nstep one body\n"); + assert_eq!(first.author.time.get(), 1_693_895_400); + assert_eq!(first.parents, vec![base]); + assert_eq!(blob_at(&repo, tip, "reef.txt"), b"three\n"); + } + + #[tokio::test] + async fn merge_rejections_move_nothing() { + let world = World::new(); + add_member_helper(&world).await; + let repo_did = create_repo_helper(&world, "scallop").await; + let base = seed_main(&world, &repo_did, &[("reef.txt", "old line\n")]); + let did = format!("did:web:{MEMBER_HOST}"); + + assert_eq!( + as_stranger( + &world, + crate::merge::merge, + MERGE, + json!({ "did": did, "name": "scallop", "branch": "main", "patch": UNIFIED_PATCH }) + ) + .await + .status(), + StatusCode::FORBIDDEN, + "a stranger cannot merge" + ); + + assert_eq!( + as_member( + &world, + crate::merge::merge, + MERGE, + json!({ "did": did, "name": "scallop", "branch": "main", "patch": UNIFIED_PATCH }) + ) + .await + .status(), + StatusCode::BAD_REQUEST, + "a merge without a commit message is rejected" + ); + assert_eq!( + main_tip(&world, &repo_did), + base, + "rejected merge mustn't move the branch" + ); + + assert_eq!( + as_member(&world, crate::merge::merge, MERGE, json!({ "did": did, "name": "scallop", "branch": "driftwood", "patch": UNIFIED_PATCH, "commitMessage": "tide" })).await.status(), + StatusCode::BAD_REQUEST, + "merging into a branch the repo lacks is an invalid request" + ); + + let response = as_member(&world, crate::merge::merge, MERGE, json!({ "did": did, "name": "scallop", "branch": "main", "patch": CONFLICTING_PATCH, "commitMessage": "tide" })).await; + assert_eq!(response.status(), StatusCode::CONFLICT); + let json = json_of(response).await; + assert_eq!(json["error"], "MergeConflict"); + assert!( + json["message"] + .as_str() + .unwrap() + .starts_with("Merge failed due to conflicts"), + ); + assert_eq!( + main_tip(&world, &repo_did), + base, + "a conflicted merge mustn't move the branch" + ); + } + + #[tokio::test] + async fn merge_check_is_open_and_reports_clean_conflicted_and_broken() { + let world = World::new(); + add_member_helper(&world).await; + let repo_did = create_repo_helper(&world, "scallop").await; + let base = seed_main(&world, &repo_did, &[("reef.txt", "old line\n")]); + + let repo = world.layout.open(&repo_did).unwrap(); + let objects_before = file_count(&repo.objects_dir()); + + let input = |patch: &str| { + body(json!({ + "did": format!("did:web:{MEMBER_HOST}"), + "name": "scallop", + "branch": "main", + "patch": patch, + })) + }; + + let clean = json_of( + crate::merge::merge_check(world.state(), input(UNIFIED_PATCH)) + .await + .unwrap(), + ) + .await; + assert_eq!(clean["is_conflicted"], false); + assert!(clean.get("conflicts").is_none()); + + let conflicted = json_of( + crate::merge::merge_check(world.state(), input(CONFLICTING_PATCH)) + .await + .unwrap(), + ) + .await; + assert_eq!(conflicted["is_conflicted"], true); + assert_eq!(conflicted["conflicts"][0]["filename"], "reef.txt"); + assert_eq!(conflicted["conflicts"][0]["reason"], "patch doesn't apply"); + assert_eq!(conflicted["message"], "patch cannot be applied cleanly"); + + let broken = json_of( + crate::merge::merge_check(world.state(), input("hello world\n")) + .await + .unwrap(), + ) + .await; + assert_eq!(broken["is_conflicted"], true); + assert!(broken["error"].as_str().is_some()); + + assert_eq!( + file_count(&repo.objects_dir()), + objects_before, + "merge check must write nothing into the object database" + ); + assert_eq!(main_tip(&world, &repo_did), base); + } +} + +mod fork_endpoints { + use super::*; + + use knot_git::{EntryKind, Identity, NewCommit, RefUpdate, Repo, StagedAction, StagedChange}; + use knot_runtime::HttpTransport; + use knot_types::{Oid, RefName, UnixSeconds}; + + const EMPTY_TREE: &str = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"; + + fn ident(time: i64) -> Identity { + Identity { + name: AuthorName::new("nel"), + email: Email::new("nel@oyster.cafe"), + time: UnixSeconds::new(time), + offset_seconds: 0, + } + } + + fn put_commit(repo: &Repo, parent: Option, path: &str, content: &str, time: i64) -> Oid { + let base_tree = match parent { + Some(parent) => repo.find_commit(parent).unwrap().tree, + None => Oid::from_hex(EMPTY_TREE).unwrap(), + }; + let staged = vec![StagedChange { + path: knot_types::RepoPath::new(path).unwrap(), + action: StagedAction::Put { + content: content.as_bytes().to_vec(), + kind: EntryKind::Blob, + }, + }]; + let tree = repo.write_staged_tree(base_tree, &staged).unwrap(); + repo.write_commit(&NewCommit { + tree, + parents: parent.into_iter().collect(), + author: ident(time), + committer: ident(time), + message: format!("put {path}"), + extra_headers: Vec::new(), + }) + .unwrap() + } + + fn advance(repo: &Repo, branch: &RefName, path: &str, content: &str, time: i64) -> Oid { + let old = repo.find_ref(branch).unwrap(); + let new = put_commit(repo, old, path, content, time); + let update = match old { + Some(old) => RefUpdate::Update { + name: branch.clone(), + old, + new, + }, + None => RefUpdate::Create { + name: branch.clone(), + new, + }, + }; + repo.update_ref(&update).unwrap(); + new + } + + fn main_ref() -> RefName { + RefName::new("refs/heads/main").unwrap() + } + + fn member_did() -> OwnerDid { + OwnerDid::new(format!("did:web:{MEMBER_HOST}")).unwrap() + } + + fn source_url(rkey: &str) -> String { + format!("https://{KNOT_HOST}/did:web:{MEMBER_HOST}/{rkey}") + } + + async fn fork_repo(world: &World, source: &str, rkey: &str) -> RepoDid { + assert_eq!( + as_member( + world, + crate::repos::create_repo, + CREATE, + json!({ "rkey": rkey, "name": rkey, "source": source }) + ) + .await + .status(), + StatusCode::OK + ); + match world + .state + .index + .resolve_repo(&member_did(), &RepoRkey::new(rkey).unwrap()) + { + Resolved::Ready(Some(did)) => did, + other => panic!("fork {rkey} wasn't registered: {other:?}"), + } + } + + struct ForkWorld { + world: World, + source_did: RepoDid, + fork_did: RepoDid, + tip: Oid, + } + + async fn forked_world() -> ForkWorld { + let world = World::new(); + add_member_helper(&world).await; + let source_did = create_repo_helper(&world, "kelp").await; + let source = world.layout.open(&source_did).unwrap(); + advance(&source, &main_ref(), "reef.txt", "kelp forest\n", 1_000); + let tip = advance(&source, &main_ref(), "tide.txt", "rock pool\n", 1_001); + [ + "refs/tags/v1", + "refs/cobs/sh.tangled.repo.collaborator/limpet", + "refs/hidden/feature/main", + ] + .into_iter() + .for_each(|name| { + source + .update_ref(&RefUpdate::Create { + name: RefName::new(name).unwrap(), + new: tip, + }) + .unwrap(); + }); + let fork_did = fork_repo(&world, &source_url("kelp"), "uni").await; + ForkWorld { + world, + source_did, + fork_did, + tip, + } + } + + async fn sync_fork(world: &World, signer: &K256Signer, host: &str, branch: &str) -> StatusCode { + call( + world, + crate::forks::fork_sync, + signer, + host, + "sh.tangled.repo.forkSync", + json!({ + "did": format!("did:web:{MEMBER_HOST}"), + "name": "uni", + "source": format!("at://did:web:{MEMBER_HOST}/sh.tangled.repo/kelp"), + "branch": branch, + }), + ) + .await + .status() + } + + async fn track_hidden(world: &World, fork_ref: &str, remote_ref: &str) -> StatusCode { + as_member( + world, + crate::forks::hidden_ref, + "sh.tangled.repo.hiddenRef", + json!({ + "repo": format!("at://did:web:{MEMBER_HOST}/sh.tangled.repo/uni"), + "forkRef": fork_ref, + "remoteRef": remote_ref, + }), + ) + .await + .status() + } + + async fn fork_status( + world: &World, + branch: &str, + hidden_ref: &str, + ) -> (StatusCode, Option) { + let response = as_member( + world, + crate::forks::fork_status, + "sh.tangled.repo.forkStatus", + json!({ + "did": format!("did:web:{MEMBER_HOST}"), + "name": "uni", + "source": source_url("kelp"), + "branch": branch, + "hiddenRef": hidden_ref, + }), + ) + .await; + let status = response.status(); + let value = json_of(response).await; + (status, value["status"].as_u64()) + } + + #[tokio::test] + async fn a_member_forks_a_repo_hosted_on_this_knot() { + let setup = forked_world().await; + assert_ne!(setup.fork_did, setup.source_did); + + let fork = setup.world.layout.open(&setup.fork_did).unwrap(); + assert_eq!(fork.find_ref(&main_ref()).unwrap(), Some(setup.tip)); + assert_eq!( + fork.find_ref(&RefName::new("refs/tags/v1").unwrap()) + .unwrap(), + Some(setup.tip) + ); + assert_eq!(fork.default_branch().unwrap().as_str(), "refs/heads/main"); + assert_eq!( + fork.origin_url().as_deref(), + Some(source_url("kelp").as_str()) + ); + assert!( + fork.references().unwrap().iter().all(|record| { + !record.name.as_str().starts_with("refs/cobs/") + && !record.name.as_str().starts_with("refs/hidden/") + }), + "a fork must copy only heads and tags, never cob or hidden refs" + ); + + let entry = fork + .entry_at(setup.tip, &knot_types::RepoPath::new("tide.txt").unwrap()) + .unwrap() + .unwrap(); + assert_eq!(fork.read_blob(entry.oid).unwrap(), b"rock pool\n"); + } + + #[tokio::test] + async fn forking_a_source_this_knot_does_not_host_is_not_found() { + let world = World::new(); + add_member_helper(&world).await; + assert_eq!( + as_member(&world, crate::repos::create_repo, CREATE, json!({ "rkey": "uni", "name": "uni", "source": format!("https://{KNOT_HOST}/did:plc:whelk/ghost") })).await.status(), + StatusCode::NOT_FOUND + ); + assert!(matches!( + world + .state + .index + .resolve_repo(&member_did(), &RepoRkey::new("uni").unwrap()), + Resolved::Ready(None) + )); + } + + #[tokio::test] + async fn fork_sync_lifecycle() { + let setup = forked_world().await; + let source = setup.world.layout.open(&setup.source_did).unwrap(); + let new_tip = advance(&source, &main_ref(), "spray.txt", "salt\n", 1_002); + + assert_eq!( + sync_fork(&setup.world, &setup.world.member, MEMBER_HOST, "main").await, + StatusCode::OK + ); + let fork = setup.world.layout.open(&setup.fork_did).unwrap(); + assert_eq!(fork.find_ref(&main_ref()).unwrap(), Some(new_tip)); + + let event = only_git_event(&setup.world); + assert_eq!(event.nsid, "sh.tangled.git.refUpdate"); + assert_eq!(event.payload["repo"], setup.fork_did.to_string()); + assert_eq!(event.payload["ref"], "refs/heads/main"); + assert_eq!(event.payload["oldSha"], setup.tip.to_string()); + assert_eq!(event.payload["newSha"], new_tip.to_string()); + assert_eq!( + event.payload["committerDid"], + account(MEMBER_HOST).to_string() + ); + + assert_eq!( + sync_fork(&setup.world, &setup.world.member, MEMBER_HOST, "main").await, + StatusCode::OK, + "an up-to-date sync is a no-op" + ); + assert_eq!( + git_events(&setup.world).len(), + 1, + "an up-to-date sync emits no further event" + ); + + assert_eq!( + sync_fork(&setup.world, &setup.world.stranger, STRANGER_HOST, "main").await, + StatusCode::FORBIDDEN, + "a stranger cannot sync a fork" + ); + assert_eq!( + sync_fork(&setup.world, &setup.world.member, MEMBER_HOST, "driftwood").await, + StatusCode::NOT_FOUND, + "syncing a branch the upstream lacks isn't found" + ); + } + + #[tokio::test] + async fn hidden_ref_tracks_the_upstream_branch_and_stays_hidden() { + let setup = forked_world().await; + let source = setup.world.layout.open(&setup.source_did).unwrap(); + let new_tip = advance(&source, &main_ref(), "spray.txt", "salt\n", 1_002); + + assert_eq!( + track_hidden(&setup.world, "feature", "main").await, + StatusCode::OK + ); + + let fork = setup.world.layout.open(&setup.fork_did).unwrap(); + let hidden = RefName::new("refs/hidden/feature/main").unwrap(); + assert_eq!(fork.find_ref(&hidden).unwrap(), Some(new_tip)); + assert!( + fork.advertised_refs() + .unwrap() + .iter() + .all(|record| !record.name.as_str().starts_with("refs/hidden/")), + "a hidden ref must stay out of the public advertisement" + ); + assert_eq!( + track_hidden(&setup.world, "feature", "main").await, + StatusCode::OK, + "tracking an already-tracked ref is idempotent" + ); + } + + #[tokio::test] + async fn fork_status_reports_up_to_date_fast_forwardable_and_conflict() { + let setup = forked_world().await; + assert_eq!( + track_hidden(&setup.world, "feature", "main").await, + StatusCode::OK + ); + assert_eq!( + fork_status(&setup.world, "main", "refs/hidden/feature/main").await, + (StatusCode::OK, Some(0)) + ); + + let source = setup.world.layout.open(&setup.source_did).unwrap(); + advance(&source, &main_ref(), "spray.txt", "salt\n", 1_002); + assert_eq!( + track_hidden(&setup.world, "feature", "main").await, + StatusCode::OK + ); + assert_eq!( + fork_status(&setup.world, "main", "refs/hidden/feature/main").await, + (StatusCode::OK, Some(1)) + ); + + let fork = setup.world.layout.open(&setup.fork_did).unwrap(); + advance(&fork, &main_ref(), "wreck.txt", "barnacle\n", 1_003); + assert_eq!( + fork_status(&setup.world, "main", "refs/hidden/feature/main").await, + (StatusCode::OK, Some(2)) + ); + + assert_eq!( + fork_status(&setup.world, "main", "refs/hidden/ghost/main").await, + (StatusCode::BAD_REQUEST, None), + "an unresolvable revision is an invalid request" + ); + } + + #[tokio::test] + async fn fork_status_reports_up_to_date_when_the_fork_is_ahead() { + let setup = forked_world().await; + assert_eq!( + track_hidden(&setup.world, "feature", "main").await, + StatusCode::OK + ); + let fork = setup.world.layout.open(&setup.fork_did).unwrap(); + advance(&fork, &main_ref(), "wreck.txt", "barnacle\n", 1_003); + assert_eq!( + fork_status(&setup.world, "main", "refs/hidden/feature/main").await, + (StatusCode::OK, Some(0)) + ); + } + + #[tokio::test] + async fn a_fork_from_a_remote_knot_fetches_over_http() { + let upstream_dir = tempfile::tempdir().unwrap(); + let upstream_path = upstream_dir.path().join("uni.git"); + let upstream = Repo::create(&upstream_path).unwrap(); + upstream + .set_head(&RefName::new("refs/heads/main").unwrap()) + .unwrap(); + advance(&upstream, &main_ref(), "reef.txt", "kelp forest\n", 1_000); + let tip = advance(&upstream, &main_ref(), "tide.txt", "rock pool\n", 1_001); + + let served = upstream_path.clone(); + let git_http: Arc = + Arc::new(FakeHttp::new(move |request: &HttpRequest| { + let _keep = &upstream_dir; + let repo = Repo::open(&served).unwrap(); + let body = if request.url.path().ends_with("/info/refs") { + knot_pack::advertise_upload(&repo).unwrap() + } else { + knot_pack::upload_pack(&repo, request.body.as_deref().unwrap_or_default()) + .unwrap() + }; + Ok(HttpResponse { + status: StatusCode::OK, + headers: http::HeaderMap::new(), + body: body.into(), + }) + })); + + let world = World::with_git_http(git_http); + add_member_helper(&world).await; + let remote = "https://barnacle.nel.pet/did:plc:squid/uni"; + let fork_did = fork_repo(&world, remote, "uni").await; + let fork = world.layout.open(&fork_did).unwrap(); + assert_eq!(fork.find_ref(&main_ref()).unwrap(), Some(tip)); + assert_eq!(fork.origin_url().as_deref(), Some(remote)); + + let new_tip = advance( + &Repo::open(&upstream_path).unwrap(), + &main_ref(), + "spray.txt", + "salt\n", + 1_002, + ); + assert_eq!( + sync_fork(&world, &world.member, MEMBER_HOST, "main").await, + StatusCode::OK + ); + let fork = world.layout.open(&fork_did).unwrap(); + assert_eq!(fork.find_ref(&main_ref()).unwrap(), Some(new_tip)); + } +} diff --git a/knot2/crates/knot-xrpc/src/wire.rs b/knot2/crates/knot-xrpc/src/wire.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-xrpc/src/wire.rs @@ -0,0 +1,617 @@ +use base64::Engine; +use base64::engine::general_purpose::STANDARD; +use serde::ser::SerializeSeq; +use serde::{Serialize, Serializer}; + +use knot_git::{ + BranchInfo, BranchTip, Commit, CommitChangeId, EntryKind, FilePatch, Hunk, Identity, LineOp, + PatchStatus, TagInfo, +}; +use knot_types::{AuthorName, Email, Oid, TagName}; + +pub(crate) const ZERO_TIME: &str = "0001-01-01T00:00:00Z"; + +fn display_opt( + value: &Option, + serializer: S, +) -> Result { + match value { + Some(id) => serializer.serialize_str(id.as_str()), + None => serializer.serialize_none(), + } +} + +fn zoned(seconds: i64, offset_seconds: i32) -> chrono::DateTime { + let offset = chrono::FixedOffset::east_opt(offset_seconds) + .unwrap_or_else(|| chrono::FixedOffset::east_opt(0).expect("zero offset is valid")); + chrono::DateTime::from_timestamp(seconds, 0) + .unwrap_or_default() + .with_timezone(&offset) +} + +pub(crate) fn rfc3339(seconds: i64, offset_seconds: i32) -> String { + zoned(seconds, offset_seconds).to_rfc3339_opts(chrono::SecondsFormat::Secs, true) +} + +pub(crate) fn rfc2822(seconds: i64, offset_seconds: i32) -> String { + zoned(seconds, offset_seconds).to_rfc2822() +} + +pub(crate) struct HashBytes(pub Oid); + +impl Serialize for HashBytes { + fn serialize(&self, serializer: S) -> Result { + let bytes = self.0.object_id(); + let mut seq = serializer.serialize_seq(Some(bytes.as_bytes().len()))?; + bytes + .as_bytes() + .iter() + .try_for_each(|byte| seq.serialize_element(byte))?; + seq.end() + } +} + +pub(crate) struct Base64Bytes(Vec); + +impl Serialize for Base64Bytes { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&STANDARD.encode(&self.0)) + } +} + +#[derive(Serialize)] +pub(crate) struct SignatureWire { + #[serde(rename = "Name")] + pub name: AuthorName, + #[serde(rename = "Email")] + pub email: Email, + #[serde(rename = "When")] + pub when: String, +} + +impl SignatureWire { + pub fn of(identity: &Identity) -> Self { + Self { + name: identity.name.clone(), + email: identity.email.clone(), + when: rfc3339(identity.time.get(), identity.offset_seconds), + } + } + + pub fn utc(identity: &Identity) -> Self { + Self { + name: identity.name.clone(), + email: identity.email.clone(), + when: rfc3339(identity.time.get(), 0), + } + } + + pub fn zero() -> Self { + Self { + name: AuthorName::new(""), + email: Email::new(""), + when: ZERO_TIME.to_string(), + } + } +} + +#[derive(Serialize)] +pub(crate) struct CommitWire { + pub hash: HashBytes, + pub author: SignatureWire, + pub committer: SignatureWire, + pub message: String, + pub tree: Oid, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub parent_hashes: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub pgp_signature: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub merge_tag: Option, + #[serde( + skip_serializing_if = "Option::is_none", + serialize_with = "display_opt" + )] + pub change_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub extra_headers: Option>, + pub this: Oid, + #[serde(skip_serializing_if = "Option::is_none")] + pub parent: Option, +} + +impl CommitWire { + pub fn of(commit: &Commit) -> Self { + let extra_headers: std::collections::BTreeMap = commit + .extra_headers + .iter() + .map(|(name, value)| (name.clone(), Base64Bytes(value.clone()))) + .collect(); + Self { + hash: HashBytes(commit.id), + author: SignatureWire::of(&commit.author), + committer: SignatureWire::of(&commit.committer), + message: commit.message.clone(), + tree: commit.tree, + parent_hashes: commit.parents.iter().map(|oid| HashBytes(*oid)).collect(), + pgp_signature: commit.pgp_signature.clone(), + merge_tag: commit.merge_tag.clone(), + change_id: commit.change_id(), + extra_headers: (!extra_headers.is_empty()).then_some(extra_headers), + this: commit.id, + parent: commit.parents.first().copied(), + } + } +} + +#[derive(Serialize)] +pub(crate) struct BranchCommitWire { + #[serde(rename = "Hash")] + pub hash: HashBytes, + #[serde(rename = "Author")] + pub author: SignatureWire, + #[serde(rename = "Committer")] + pub committer: SignatureWire, + #[serde(rename = "MergeTag")] + pub merge_tag: String, + #[serde(rename = "PGPSignature")] + pub pgp_signature: String, + #[serde(rename = "Message")] + pub message: String, + #[serde(rename = "TreeHash")] + pub tree_hash: HashBytes, + #[serde(rename = "ParentHashes")] + pub parent_hashes: Vec, + #[serde(rename = "Encoding")] + pub encoding: String, + #[serde(rename = "ExtraHeaders")] + pub extra_headers: Option<()>, +} + +#[derive(Serialize)] +pub(crate) struct Reference { + pub name: String, + pub hash: Oid, +} + +#[derive(Serialize)] +pub(crate) struct BranchWire { + pub reference: Reference, + pub commit: BranchCommitWire, + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub is_default: bool, +} + +impl BranchWire { + pub fn of(branch: &BranchInfo, is_default: bool, absent: Oid) -> Self { + Self { + reference: Reference { + name: branch.name.to_string(), + hash: branch.tip.id(), + }, + commit: match &branch.tip { + BranchTip::Commit(commit) => BranchCommitWire { + hash: HashBytes(commit.id), + author: SignatureWire::utc(&commit.author), + committer: SignatureWire::utc(&commit.committer), + merge_tag: String::new(), + pgp_signature: String::new(), + message: commit.message.trim_end().to_string(), + tree_hash: HashBytes(commit.tree), + parent_hashes: commit.parents.iter().map(|oid| HashBytes(*oid)).collect(), + encoding: String::new(), + extra_headers: None, + }, + BranchTip::Opaque { id, message, .. } => BranchCommitWire { + hash: HashBytes(*id), + author: SignatureWire::zero(), + committer: SignatureWire::zero(), + merge_tag: String::new(), + pgp_signature: String::new(), + message: message.trim_end().to_string(), + tree_hash: HashBytes(absent), + parent_hashes: Vec::new(), + encoding: String::new(), + extra_headers: None, + }, + }, + is_default, + } + } +} + +#[derive(Serialize)] +pub(crate) struct TagObjectWire { + #[serde(rename = "Hash")] + pub hash: HashBytes, + #[serde(rename = "Name")] + pub name: TagName, + #[serde(rename = "Tagger")] + pub tagger: SignatureWire, + #[serde(rename = "Message")] + pub message: String, + #[serde(rename = "PGPSignature")] + pub pgp_signature: String, + #[serde(rename = "TargetType")] + pub target_type: i8, + #[serde(rename = "Target")] + pub target: HashBytes, +} + +#[derive(Serialize)] +pub(crate) struct TagWire { + pub name: TagName, + pub hash: Oid, + #[serde(skip_serializing_if = "Option::is_none")] + pub tag: Option, + #[serde(skip_serializing_if = "String::is_empty")] + pub message: String, +} + +const TARGET_TYPE_TAG: i8 = 4; + +impl TagWire { + pub fn of(info: &TagInfo) -> Self { + let message = recombine_message(&info.message); + let tag = + info.annotated.as_ref().map(|annotated| TagObjectWire { + hash: HashBytes(info.id), + name: info.name.clone(), + tagger: annotated.tagger.as_ref().map(SignatureWire::utc).unwrap_or( + SignatureWire { + name: AuthorName::new(""), + email: Email::new(""), + when: rfc3339(0, 0), + }, + ), + message: message.clone(), + pgp_signature: annotated.pgp_signature.clone().unwrap_or_default(), + target_type: TARGET_TYPE_TAG, + target: HashBytes(annotated.target), + }); + Self { + name: info.name.clone(), + hash: info.id, + tag, + message, + } + } +} + +pub(crate) fn fold_subject(message: &str) -> String { + message + .split("\n\n") + .next() + .unwrap_or_default() + .lines() + .map(str::trim_end) + .filter(|line| !line.is_empty()) + .collect::>() + .join(" ") +} + +pub(crate) fn message_body(message: &str) -> String { + message + .split_once("\n\n") + .map(|(_, body)| body.trim_matches('\n').to_string()) + .unwrap_or_default() +} + +fn recombine_message(message: &str) -> String { + let subject = fold_subject(message); + let body = message_body(message); + match (subject.is_empty(), body.is_empty()) { + (_, true) => subject, + (true, false) => body, + (false, false) => format!("{subject}\n\n{body}"), + } +} + +#[derive(Serialize)] +pub(crate) struct LineWire { + #[serde(rename = "Op")] + pub op: u8, + #[serde(rename = "Line")] + pub line: String, +} + +#[derive(Serialize)] +pub(crate) struct TextFragmentWire { + #[serde(rename = "Comment")] + pub comment: String, + #[serde(rename = "OldPosition")] + pub old_position: i64, + #[serde(rename = "OldLines")] + pub old_lines: i64, + #[serde(rename = "NewPosition")] + pub new_position: i64, + #[serde(rename = "NewLines")] + pub new_lines: i64, + #[serde(rename = "LinesAdded")] + pub lines_added: i64, + #[serde(rename = "LinesDeleted")] + pub lines_deleted: i64, + #[serde(rename = "LeadingContext")] + pub leading_context: i64, + #[serde(rename = "TrailingContext")] + pub trailing_context: i64, + #[serde(rename = "Lines")] + pub lines: Vec, +} + +impl TextFragmentWire { + pub fn of(hunk: &Hunk) -> Self { + let lines: Vec = hunk + .lines + .iter() + .map(|line| LineWire { + op: match line.op { + LineOp::Context => 0, + LineOp::Delete => 1, + LineOp::Add => 2, + }, + line: String::from_utf8_lossy(&line.text).into_owned(), + }) + .collect(); + let leading = lines.iter().take_while(|line| line.op == 0).count(); + let trailing = if leading == lines.len() { + 0 + } else { + lines.iter().rev().take_while(|line| line.op == 0).count() + }; + Self { + comment: String::new(), + old_position: hunk.old_start.get() as i64, + old_lines: hunk.old_lines.get() as i64, + new_position: hunk.new_start.get() as i64, + new_lines: hunk.new_lines.get() as i64, + lines_added: hunk.added().get() as i64, + lines_deleted: hunk.deleted().get() as i64, + leading_context: leading as i64, + trailing_context: trailing as i64, + lines, + } + } +} + +#[derive(Serialize)] +pub(crate) struct DiffNameWire { + pub old: String, + pub new: String, +} + +#[derive(Serialize)] +pub(crate) struct DiffWire { + pub name: DiffNameWire, + pub text_fragments: Option>, + pub is_binary: bool, + pub is_new: bool, + pub is_delete: bool, + pub is_copy: bool, + pub is_rename: bool, +} + +impl DiffWire { + pub fn of(patch: &FilePatch) -> Self { + let fragments: Vec = + patch.hunks.iter().map(TextFragmentWire::of).collect(); + Self { + name: DiffNameWire { + old: match patch.status { + PatchStatus::Added => String::new(), + _ => patch.path.to_string(), + }, + new: match patch.status { + PatchStatus::Deleted => String::new(), + _ => patch.path.to_string(), + }, + }, + text_fragments: (!fragments.is_empty()).then_some(fragments), + is_binary: patch.is_binary, + is_new: patch.status == PatchStatus::Added, + is_delete: patch.status == PatchStatus::Deleted, + is_copy: false, + is_rename: false, + } + } +} + +#[derive(Serialize)] +pub(crate) struct DiffStatWire { + pub insertions: i64, + pub deletions: i64, + pub files_changed: i64, +} + +#[derive(Serialize)] +pub(crate) struct NiceDiffWire { + pub commit: CommitWire, + pub stat: DiffStatWire, + pub diff: Option>, +} + +pub(crate) fn nice_diff(commit: &Commit, patches: &[FilePatch]) -> NiceDiffWire { + let diffs: Vec = patches.iter().map(DiffWire::of).collect(); + let stat = DiffStatWire { + insertions: patches + .iter() + .flat_map(|patch| patch.hunks.iter()) + .map(|hunk| hunk.added().get() as i64) + .sum(), + deletions: patches + .iter() + .flat_map(|patch| patch.hunks.iter()) + .map(|hunk| hunk.deleted().get() as i64) + .sum(), + files_changed: patches.len() as i64, + }; + NiceDiffWire { + commit: CommitWire::of(commit), + stat, + diff: (!diffs.is_empty()).then_some(diffs), + } +} + +#[derive(Serialize)] +pub(crate) struct PatchIdentityWire { + #[serde(rename = "Name")] + pub name: AuthorName, + #[serde(rename = "Email")] + pub email: Email, +} + +#[derive(Serialize)] +pub(crate) struct FormatPatchWire { + #[serde(rename = "Files")] + pub files: Option>, + #[serde(rename = "SHA")] + pub sha: Oid, + #[serde(rename = "Author")] + pub author: Option, + #[serde(rename = "AuthorDate")] + pub author_date: String, + #[serde(rename = "Committer")] + pub committer: Option<()>, + #[serde(rename = "CommitterDate")] + pub committer_date: String, + #[serde(rename = "Title")] + pub title: String, + #[serde(rename = "Body")] + pub body: String, + #[serde(rename = "SubjectPrefix")] + pub subject_prefix: String, + #[serde(rename = "BodyAppendix")] + pub body_appendix: String, + #[serde(rename = "RawHeaders")] + pub raw_headers: Option>>, + #[serde(rename = "Raw")] + pub raw: String, +} + +pub(crate) fn normalize_message_section<'a>(lines: impl Iterator) -> String { + lines + .map(str::trim_end) + .fold((String::new(), 0usize), |(mut out, blanks), line| { + if line.is_empty() { + return (out, blanks + 1); + } + if !out.is_empty() { + out.push('\n'); + if blanks > 0 { + out.push('\n'); + } + } + out.push_str(line); + (out, 0) + }) + .0 +} + +fn entry_mode_decimal(kind: EntryKind) -> u32 { + match kind { + EntryKind::Tree => 0o040000, + EntryKind::Blob => 0o100644, + EntryKind::BlobExecutable => 0o100755, + EntryKind::Link => 0o120000, + EntryKind::Commit => 0o160000, + } +} + +pub(crate) fn entry_mode_octal(kind: EntryKind) -> String { + format!("{:06o}", entry_mode_decimal(kind)) +} + +#[derive(Serialize)] +pub(crate) struct FileWire { + #[serde(rename = "OldName")] + pub old_name: String, + #[serde(rename = "NewName")] + pub new_name: String, + #[serde(rename = "IsNew")] + pub is_new: bool, + #[serde(rename = "IsDelete")] + pub is_delete: bool, + #[serde(rename = "IsCopy")] + pub is_copy: bool, + #[serde(rename = "IsRename")] + pub is_rename: bool, + #[serde(rename = "OldMode")] + pub old_mode: u32, + #[serde(rename = "NewMode")] + pub new_mode: u32, + #[serde(rename = "OldOIDPrefix")] + pub old_oid_prefix: String, + #[serde(rename = "NewOIDPrefix")] + pub new_oid_prefix: String, + #[serde(rename = "Score")] + pub score: i64, + #[serde(rename = "TextFragments")] + pub text_fragments: Option>, + #[serde(rename = "IsBinary")] + pub is_binary: bool, + #[serde(rename = "BinaryFragment")] + pub binary_fragment: Option<()>, + #[serde(rename = "ReverseBinaryFragment")] + pub reverse_binary_fragment: Option<()>, +} + +impl FileWire { + pub fn of(patch: &FilePatch) -> Self { + let fragments: Vec = + patch.hunks.iter().map(TextFragmentWire::of).collect(); + let same_mode = patch.old_kind.is_some() && patch.old_kind == patch.new_kind; + Self { + old_name: match patch.status { + PatchStatus::Added => String::new(), + _ => patch.path.to_string(), + }, + new_name: match patch.status { + PatchStatus::Deleted => String::new(), + _ => patch.path.to_string(), + }, + is_new: patch.status == PatchStatus::Added, + is_delete: patch.status == PatchStatus::Deleted, + is_copy: false, + is_rename: false, + old_mode: match patch.status { + PatchStatus::Added => 0, + PatchStatus::Deleted | PatchStatus::Modified => { + patch.old_kind.map(entry_mode_decimal).unwrap_or(0) + } + }, + new_mode: match patch.status { + PatchStatus::Added => patch.new_kind.map(entry_mode_decimal).unwrap_or(0), + PatchStatus::Deleted => 0, + PatchStatus::Modified if same_mode => 0, + PatchStatus::Modified => patch.new_kind.map(entry_mode_decimal).unwrap_or(0), + }, + old_oid_prefix: patch.old_oid.to_hex(), + new_oid_prefix: patch.new_oid.to_hex(), + score: 0, + text_fragments: (!fragments.is_empty()).then_some(fragments), + is_binary: patch.is_binary, + binary_fragment: None, + reverse_binary_fragment: None, + } + } +} + +#[cfg(test)] +mod tests { + use super::{rfc2822, rfc3339}; + + const A_JUNE_INSTANT: i64 = 1_717_236_600; + + #[test] + fn rfc3339_renders_the_commits_own_offset_independent_of_the_host_zone() { + assert!(rfc3339(A_JUNE_INSTANT, 7200).ends_with("+02:00")); + assert!(rfc3339(A_JUNE_INSTANT, -18000).ends_with("-05:00")); + assert!(rfc3339(A_JUNE_INSTANT, 0).ends_with('Z')); + } + + #[test] + fn rfc2822_keeps_the_signed_offset() { + assert!(rfc2822(A_JUNE_INSTANT, 7200).ends_with("+0200")); + assert!(rfc2822(A_JUNE_INSTANT, -18000).ends_with("-0500")); + } +} diff --git a/knot2/crates/knot-xrpc/tests/lfs.rs b/knot2/crates/knot-xrpc/tests/lfs.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-xrpc/tests/lfs.rs @@ -0,0 +1,299 @@ +mod common; + +use axum::body::Body; +use http::{Request, StatusCode, header}; +use sha2::{Digest, Sha256}; +use tower::ServiceExt; + +use knot_lfs::{LfsOid, LfsStore}; +use knot_types::RepoDid; + +use common::{OWNER, World, empty_repo, get}; + +const MEDIA: &[u8] = b"\xff\x00heavy media bytes that live outside the odb"; + +fn seeded_object(world: &World, repo: &RepoDid) -> (LfsOid, usize) { + let oid = LfsOid::from_digest(Sha256::digest(MEDIA).into()); + world + .state + .lfs + .as_ref() + .unwrap() + .handle + .store + .put( + repo, + &oid, + knot_lfs::ClaimedSize::new(MEDIA.len() as u64), + &mut &MEDIA[..], + ) + .unwrap(); + (oid, MEDIA.len()) +} + +fn absent_oid() -> LfsOid { + LfsOid::from_digest(Sha256::digest(b"never uploaded anywhere").into()) +} + +async fn post_batch(world: &World, path: &str, body: String) -> (StatusCode, serde_json::Value) { + let request = Request::post(path) + .header(header::CONTENT_TYPE, "application/vnd.git-lfs+json") + .body(Body::from(body)) + .unwrap(); + let response = world.router.clone().oneshot(request).await.unwrap(); + let status = response.status(); + let bytes = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + let value = serde_json::from_slice(&bytes).unwrap_or(serde_json::Value::Null); + (status, value) +} + +fn batch_body(operation: &str, oids: &[(String, u64)]) -> String { + let objects: Vec = oids + .iter() + .map(|(oid, size)| serde_json::json!({"oid": oid, "size": size})) + .collect(); + serde_json::json!({ + "operation": operation, + "transfers": ["basic", "ssh"], + "objects": objects, + "hash_algo": "sha256", + }) + .to_string() +} + +#[tokio::test] +async fn the_batch_download_surface_answers_every_addressing_form() { + let world = World::new(); + let (did, _bare, _work) = empty_repo(&world, "barnacle"); + let (oid, size) = seeded_object(&world, &did); + let missing = absent_oid(); + + let body = batch_body( + "download", + &[ + (oid.as_str().to_string(), size as u64), + (missing.as_str().to_string(), 9), + ], + ); + let paths = [ + format!("/{did}/info/lfs/objects/batch"), + format!("/{did}.git/info/lfs/objects/batch"), + format!("/{OWNER}/barnacle/info/lfs/objects/batch"), + format!("/{OWNER}/barnacle.git/info/lfs/objects/batch"), + ]; + futures::future::join_all(paths.iter().map(|path| { + let world = &world; + let body = body.clone(); + let oid = oid.clone(); + async move { + let (status, json) = post_batch(world, path, body).await; + assert_eq!(status, StatusCode::OK, "batch at {path}"); + assert_eq!(json["transfer"], "basic"); + assert_eq!(json["hash_algo"], "sha256"); + let objects = json["objects"].as_array().unwrap(); + assert_eq!(objects.len(), 2); + assert_eq!(objects[0]["oid"], oid.as_str()); + assert_eq!(objects[0]["size"], size as u64); + assert_eq!(objects[0]["authenticated"], true); + let href = objects[0]["actions"]["download"]["href"].as_str().unwrap(); + assert!( + href.ends_with(&format!("/info/lfs/objects/{oid}")), + "href {href}" + ); + assert!(href.starts_with("https://"), "href {href}"); + assert!(objects[0].get("error").is_none()); + assert_eq!(objects[1]["error"]["code"], 404); + assert!(objects[1].get("actions").is_none()); + } + })) + .await; + + let upload = batch_body("upload", &[(oid.as_str().to_string(), size as u64)]); + let (status, _) = post_batch(&world, &format!("/{did}/info/lfs/objects/batch"), upload).await; + assert_eq!( + status, + StatusCode::UNAUTHORIZED, + "an unauthenticated HTTP upload batch is challenged for credentials, never answered" + ); +} + +#[tokio::test] +async fn the_object_route_streams_ranges_and_stays_anonymous() { + let world = World::new(); + let (did, _bare, _work) = empty_repo(&world, "limpet"); + let (oid, size) = seeded_object(&world, &did); + + let request = Request::get(format!("/{did}/info/lfs/objects/{oid}")) + .body(Body::empty()) + .unwrap(); + let response = world.router.clone().oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.headers()[header::CONTENT_TYPE], + "application/octet-stream" + ); + assert_eq!( + response.headers()[header::CACHE_CONTROL], + "public, max-age=31536000, immutable" + ); + let bytes = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + assert_eq!(&bytes[..], MEDIA); + + let request = Request::get(format!("/{OWNER}/limpet.git/info/lfs/objects/{oid}")) + .header(header::RANGE, "bytes=3-6") + .body(Body::empty()) + .unwrap(); + let response = world.router.clone().oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + let bytes = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + assert_eq!(&bytes[..], &MEDIA[3..=6]); + + let request = Request::get(format!("/{did}/info/lfs/objects/{oid}")) + .header(header::IF_NONE_MATCH, format!("\"{oid}\"")) + .body(Body::empty()) + .unwrap(); + let response = world.router.clone().oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::NOT_MODIFIED); + assert_eq!( + response.headers()[header::ETAG], + format!("\"{oid}\"").as_str() + ); + let _ = size; +} + +#[tokio::test] +async fn missing_and_hostile_objects_get_typed_404s() { + let world = World::new(); + let (did, _bare, _work) = empty_repo(&world, "scallop"); + seeded_object(&world, &did); + + let absent = absent_oid(); + let cases = [ + format!("/{did}/info/lfs/objects/{absent}"), + format!("/{did}/info/lfs/objects/deadbeef"), + format!("/{did}/info/lfs/objects/..%2f..%2fetc%2fpasswd"), + format!("/did:plc:nowhere/info/lfs/objects/{absent}"), + ]; + futures::future::join_all(cases.iter().map(|path| { + let world = &world; + async move { + let request = Request::get(path).body(Body::empty()).unwrap(); + let response = world.router.clone().oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND, "GET {path}"); + assert_eq!( + response.headers()[header::CONTENT_TYPE], + "application/vnd.git-lfs+json", + "GET {path}" + ); + } + })) + .await; + + let request = Request::get(format!("/{did}/info/lfs/objects/{absent}")) + .header(header::IF_NONE_MATCH, "*") + .body(Body::empty()) + .unwrap(); + let response = world.router.clone().oneshot(request).await.unwrap(); + assert_eq!( + response.status(), + StatusCode::NOT_FOUND, + "an absent object must 404 even when revalidated" + ); +} + +#[tokio::test] +async fn readiness_reflects_the_lfs_store() { + let world = World::new(); + let (status, _, _) = get(&world, "/xrpc/_health").await; + assert_eq!(status, StatusCode::OK, "a writable store reports ready"); + + let incoming = world.lfs_dir.join(".incoming"); + std::fs::remove_dir(&incoming).unwrap(); + let (status, _, body) = get(&world, "/xrpc/_health").await; + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "an unwritable store reports unready: {}", + String::from_utf8_lossy(&body) + ); + + std::fs::create_dir_all(&incoming).unwrap(); + let (status, _, _) = get(&world, "/xrpc/_health").await; + assert_eq!(status, StatusCode::OK, "a recovered store reports ready"); +} + +#[tokio::test] +async fn hostile_batches_get_clean_typed_rejections() { + let world = World::new(); + let (did, _bare, _work) = empty_repo(&world, "conch"); + let (oid, size) = seeded_object(&world, &did); + let base = format!("/{did}/info/lfs/objects/batch"); + + let (status, _) = post_batch(&world, &base, "not json at all".to_string()).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY); + + let traversal = serde_json::json!({ + "operation": "download", + "objects": [{"oid": "../../../../etc/passwd", "size": 1}], + }) + .to_string(); + let (status, _) = post_batch(&world, &base, traversal).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY); + + let oversized_list: Vec<(String, u64)> = (0..1001) + .map(|index| { + let digest = Sha256::digest(index.to_string().as_bytes()); + (LfsOid::from_digest(digest.into()).as_str().to_string(), 1) + }) + .collect(); + let (status, _) = post_batch(&world, &base, batch_body("download", &oversized_list)).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY); + + let foreign_algo = serde_json::json!({ + "operation": "download", + "objects": [{"oid": oid.as_str(), "size": size}], + "hash_algo": "sha1", + }) + .to_string(); + let (status, _) = post_batch(&world, &base, foreign_algo).await; + assert_eq!(status, StatusCode::CONFLICT); + + let no_common_transfer = serde_json::json!({ + "operation": "download", + "transfers": ["tus"], + "objects": [{"oid": oid.as_str(), "size": size}], + }) + .to_string(); + let (status, _) = post_batch(&world, &base, no_common_transfer).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY); + + let over_limit = format!( + r#"{{"operation":"download","objects":[],"pad":"{}"}}"#, + "a".repeat(1024 * 1024 + 1) + ); + let (status, _) = post_batch(&world, &base, over_limit).await; + assert_eq!( + status, + StatusCode::PAYLOAD_TOO_LARGE, + "a batch body over the limit is refused before buffering" + ); + + let absurd_size = serde_json::json!({ + "operation": "download", + "objects": [{"oid": oid.as_str(), "size": u64::MAX}], + }) + .to_string(); + let (status, json) = post_batch(&world, &base, absurd_size).await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + json["objects"][0]["size"], size as u64, + "a download answer reports the stored size, ignoring the client's absurd claim" + ); + assert!(json["objects"][0]["actions"]["download"]["href"].is_string()); +} diff --git a/knot2/crates/knot-xrpc/tests/lfs_soak.rs b/knot2/crates/knot-xrpc/tests/lfs_soak.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-xrpc/tests/lfs_soak.rs @@ -0,0 +1,159 @@ +mod common; + +use axum::body::Body; +use futures::StreamExt; +use http::{Request, StatusCode, header}; +use sha2::{Digest, Sha256}; +use tower::ServiceExt; + +use knot_lfs::{LfsOid, LfsStore}; +use knot_types::RepoDid; + +use common::{World, empty_repo}; + +const OBJECT_BYTES: usize = 8 * 1024 * 1024; +const OBJECTS: usize = 4; +const DOWNLOADERS: usize = 12; +const ROUNDS: usize = 4; +const PAGE_BYTES: u64 = 4096; +const PEAK_CEILING: u64 = 512 * 1024 * 1024; +const GROWTH_SLACK: u64 = 64 * 1024 * 1024; + +fn rss_bytes() -> u64 { + let statm = std::fs::read_to_string("/proc/self/statm").expect("/proc/self/statm is readable"); + statm + .split_whitespace() + .nth(1) + .and_then(|pages| pages.parse::().ok()) + .map(|pages| pages * PAGE_BYTES) + .expect("statm lists the resident page count") +} + +fn incompressible(len: usize, seed: u64) -> Vec { + let mut state = seed | 1; + (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + (state & 0xff) as u8 + }) + .collect() +} + +fn seed_objects(world: &World, repo: &RepoDid) -> Vec { + let store = &world.state.lfs.as_ref().unwrap().handle.store; + (0..OBJECTS) + .map(|index| { + let body = incompressible(OBJECT_BYTES, 0x5eed_0000 + index as u64); + let oid = LfsOid::from_digest(Sha256::digest(&body).into()); + store + .put( + repo, + &oid, + knot_lfs::ClaimedSize::new(body.len() as u64), + &mut &body[..], + ) + .unwrap(); + oid + }) + .collect() +} + +async fn download(world: &World, did: &RepoDid, oid: &LfsOid, tag: &str) { + let request = Request::get(format!("/{did}/info/lfs/objects/{oid}")) + .body(Body::empty()) + .unwrap(); + let response = world.router.clone().oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK, "{tag}"); + let (streamed, hasher) = response + .into_body() + .into_data_stream() + .fold( + (0u64, Sha256::new()), + |(streamed, mut hasher), chunk| async move { + let chunk = chunk.unwrap(); + hasher.update(&chunk); + (streamed + chunk.len() as u64, hasher) + }, + ) + .await; + assert_eq!(streamed, OBJECT_BYTES as u64, "{tag}"); + assert_eq!( + LfsOid::from_digest(hasher.finalize().into()), + oid.clone(), + "{tag}: downloaded bytes must hash to the requested oid" + ); +} + +async fn batch(world: &World, did: &RepoDid, oids: &[LfsOid], tag: &str) { + let objects: Vec = oids + .iter() + .map(|oid| serde_json::json!({"oid": oid.as_str(), "size": OBJECT_BYTES})) + .collect(); + let body = serde_json::json!({"operation": "download", "objects": objects}).to_string(); + let request = Request::post(format!("/{did}/info/lfs/objects/batch")) + .header(header::CONTENT_TYPE, "application/vnd.git-lfs+json") + .body(Body::from(body)) + .unwrap(); + let response = world.router.clone().oneshot(request).await.unwrap(); + let status = response.status(); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + assert_eq!( + status, + StatusCode::OK, + "{tag}: {}", + String::from_utf8_lossy(&body) + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn sustained_anonymous_downloads_stay_bounded_and_leak_nothing() { + let world = World::unshed(); + let (did, _bare, _work) = empty_repo(&world, "nautilus"); + let oids = seed_objects(&world, &did); + + let storm = |round: usize| { + let world = &world; + let did = &did; + let oids = &oids; + async move { + futures::future::join_all((0..DOWNLOADERS).map(|task| { + let oid = oids[task % OBJECTS].clone(); + async move { + let tag = format!("round {round} task {task}"); + batch(world, did, oids, &tag).await; + download(world, did, &oid, &tag).await; + } + })) + .await; + } + }; + + storm(0).await; + let settled = rss_bytes(); + + let peaks: Vec = futures::stream::iter(1..ROUNDS) + .then(|round| { + let storm = &storm; + async move { + storm(round).await; + rss_bytes() + } + }) + .collect() + .await; + + let peak = peaks.iter().copied().max().unwrap_or(settled); + assert!( + peak < PEAK_CEILING, + "concurrent downloads peaked at {peak} bytes, ceiling {PEAK_CEILING}" + ); + let last = *peaks.last().unwrap_or(&settled); + assert!( + last <= settled + GROWTH_SLACK, + "rss grew from {settled} to {last} across rounds, downloads are leaking" + ); +} diff --git a/knot2/crates/knot-xrpc/tests/reads.rs b/knot2/crates/knot-xrpc/tests/reads.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-xrpc/tests/reads.rs @@ -0,0 +1,1971 @@ +mod common; + +use std::future::Future; +use std::pin::Pin; +use std::time::{Duration, Instant}; + +use futures::StreamExt; +use futures::stream; +use http::{HeaderMap, StatusCode, header}; +use tokio_tungstenite::tungstenite; + +use knot_events::{EventCursor, GitRefUpdate}; +use knot_types::{AccountDid, ObjectFormat, Oid, OwnerDid, RepoDid}; +use knot_xrpc::{ArchiveLimit, ResponseLimit}; + +use common::{ + OWNER, World, archive_full, assert_immutable_round_trip, assert_post_rejected, assert_warming, + commit_file, empty_repo, get, get_error, get_json, get_with_headers, git_run, post_authed, + post_json, ref_names, repo_dids, seeded, seeded_feature_branch, seeded_with_format, sh_git, + sh_git_at, +}; + +#[tokio::test] +async fn the_seeded_read_surface_renders_each_wire_shape_once() { + let world = World::new(); + let (did, work) = seeded(&world, "coral"); + let head = sh_git(work.path(), &["rev-parse", "HEAD"]); + let parent = sh_git(work.path(), &["rev-parse", "HEAD~1"]); + let tag_object = sh_git(work.path(), &["rev-parse", "v1.0.0"]); + let tagged_commit = sh_git(work.path(), &["rev-parse", "v1.0.0^{commit}"]); + + let tree = get_json( + &world, + &format!("/xrpc/sh.tangled.repo.tree?repo={did}&ref=main"), + ) + .await; + assert_eq!(tree["ref"], "main"); + assert!(tree.get("parent").is_none()); + assert!(tree.get("dotdot").is_none()); + let files = tree["files"].as_array().unwrap(); + let names: Vec<&str> = files + .iter() + .map(|file| file["name"].as_str().unwrap()) + .collect(); + assert_eq!(names, vec!["README.md", "logo.png", "src"]); + let readme_entry = &files[0]; + assert_eq!(readme_entry["mode"], "0100644"); + assert_eq!( + readme_entry["size"].as_i64().unwrap(), + b"# coral\n\nhello reef\n".len() as i64 + ); + assert_eq!( + readme_entry["last_commit"]["hash"].as_str().unwrap(), + head, + "README was last touched by the head commit" + ); + assert_eq!(readme_entry["last_commit"]["message"], "update readme"); + assert_eq!(files[2]["mode"], "0040000"); + assert_eq!(tree["readme"]["filename"], "README.md"); + assert_eq!(tree["readme"]["contents"], "# coral\n\nhello reef\n"); + assert_eq!(tree["lastCommit"]["hash"], head.as_str()); + assert_eq!(tree["lastCommit"]["author"]["name"], "nel"); + assert_eq!(tree["lastCommit"]["author"]["when"], ""); + + let sub = get_json( + &world, + &format!("/xrpc/sh.tangled.repo.tree?repo={did}&ref=main&path=src"), + ) + .await; + assert_eq!(sub["parent"], "src"); + assert!(sub.get("dotdot").is_none()); + assert_eq!(sub["files"][0]["name"], "main.rs"); + assert_eq!( + get_error( + &world, + &format!("/xrpc/sh.tangled.repo.tree?repo={did}&ref=main&path=nope"), + ) + .await, + (StatusCode::NOT_FOUND, "PathNotFound".to_string()) + ); + + let log = get_json( + &world, + &format!("/xrpc/sh.tangled.repo.log?repo={did}&ref=main"), + ) + .await; + assert_eq!(log["total"].as_i64(), Some(4)); + assert_eq!(log["page"].as_i64(), Some(1)); + assert_eq!(log["per_page"].as_i64(), Some(50)); + assert_eq!(log["log"], true); + assert_eq!(log["ref"], "main"); + let commits = log["commits"].as_array().unwrap(); + assert_eq!(commits.len(), 4); + let first = &commits[0]; + let hash_bytes: Vec = first["hash"] + .as_array() + .unwrap() + .iter() + .map(|byte| byte.as_u64().unwrap() as u8) + .collect(); + assert_eq!( + hash_bytes, + (0..head.len()) + .step_by(2) + .map(|index| u8::from_str_radix(&head[index..index + 2], 16).unwrap()) + .collect::>(), + "commit hash rides as a byte array" + ); + assert_eq!(first["this"], head.as_str()); + assert_eq!(first["parent"], parent.as_str()); + assert_eq!(first["author"]["Name"], "nel"); + assert_eq!(first["author"]["Email"], "nel@oyster.cafe"); + assert_eq!(first["author"]["When"], "2026-06-01T12:33:00+02:00"); + assert_eq!(first["message"], "update readme\n"); + assert!(first["tree"].as_str().unwrap().len() == 40); + let paged = get_json( + &world, + &format!("/xrpc/sh.tangled.repo.log?repo={did}&ref=main&limit=2&cursor=2"), + ) + .await; + assert_eq!(paged["commits"].as_array().unwrap().len(), 2); + assert_eq!(paged["page"].as_i64(), Some(2)); + assert_eq!(paged["per_page"].as_i64(), Some(2)); + + let branches = get_json( + &world, + &format!("/xrpc/sh.tangled.repo.branches?repo={did}"), + ) + .await; + let listed = branches["branches"].as_array().unwrap(); + assert_eq!(listed.len(), 1); + assert_eq!(listed[0]["reference"]["name"], "main"); + assert_eq!(listed[0]["reference"]["hash"], head.as_str()); + assert_eq!(listed[0]["is_default"], true); + assert_eq!(listed[0]["commit"]["Author"]["Name"], "nel"); + assert!(listed[0]["commit"]["Hash"].is_array()); + assert_eq!(listed[0]["commit"]["ExtraHeaders"], serde_json::Value::Null); + assert_eq!(listed[0]["commit"]["Message"], "update readme"); + let branch = get_json( + &world, + &format!("/xrpc/sh.tangled.repo.branch?repo={did}&name=main"), + ) + .await; + assert_eq!(branch["name"], "main"); + assert_eq!(branch["hash"], head.as_str()); + assert_eq!(branch["shortHash"], head[..7].to_string().as_str()); + assert_eq!(branch["isDefault"], true); + assert_eq!(branch["author"]["name"], "nel"); + assert_eq!(branch["when"], "2026-06-01T12:33:00+02:00"); + assert_eq!( + get_error( + &world, + &format!("/xrpc/sh.tangled.repo.branch?repo={did}&name=mangrove"), + ) + .await, + (StatusCode::NOT_FOUND, "BranchNotFound".to_string()) + ); + + let tags = get_json(&world, &format!("/xrpc/sh.tangled.repo.tags?repo={did}")).await; + let tag_list = tags["tags"].as_array().unwrap(); + assert_eq!(tag_list.len(), 2); + let annotated = tag_list.iter().find(|tag| tag["name"] == "v1.0.0").unwrap(); + assert_eq!(annotated["hash"], tag_object.as_str()); + assert_eq!(annotated["message"], "release one"); + assert_eq!(annotated["tag"]["TargetType"].as_i64(), Some(4)); + assert_eq!(annotated["tag"]["Tagger"]["Name"], "nel"); + let target_bytes = annotated["tag"]["Target"].as_array().unwrap(); + assert_eq!(target_bytes.len(), 20); + assert_eq!( + target_bytes[0].as_u64().unwrap() as u8, + u8::from_str_radix(&tagged_commit[..2], 16).unwrap() + ); + let lightweight = tag_list + .iter() + .find(|tag| tag["name"] == "lightweight") + .unwrap(); + assert!(lightweight.get("tag").is_none()); + assert_eq!(lightweight["hash"], tagged_commit.as_str()); + assert_eq!(lightweight["message"], "add logo"); + let single = get_json( + &world, + &format!("/xrpc/sh.tangled.repo.tag?repo={did}&tag=v1.0.0"), + ) + .await; + assert_eq!(single["tag"]["name"], "v1.0.0"); + assert_eq!( + get_error( + &world, + &format!("/xrpc/sh.tangled.repo.tag?repo={did}&tag=v9.9.9"), + ) + .await, + (StatusCode::BAD_REQUEST, "TagNotFound".to_string()) + ); + + let text = get_json( + &world, + &format!("/xrpc/sh.tangled.repo.blob?repo={did}&ref=main&path=README.md"), + ) + .await; + assert_eq!(text["encoding"], "utf-8"); + assert_eq!(text["isBinary"], false); + assert_eq!(text["content"], "# coral\n\nhello reef\n"); + assert_eq!(text["mimeType"], "text/plain; charset=utf-8"); + assert_eq!(text["lastCommit"]["message"], "update readme"); + + let binary = get_json( + &world, + &format!("/xrpc/sh.tangled.repo.blob?repo={did}&ref=main&path=logo.png"), + ) + .await; + assert_eq!(binary["encoding"], "base64"); + assert_eq!(binary["isBinary"], true); + assert_eq!(binary["mimeType"], "image/png"); + + let (status, headers, body) = get( + &world, + &format!("/xrpc/sh.tangled.repo.blob?repo={did}&ref=main&path=logo.png&raw=true"), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(headers.get(header::CONTENT_TYPE).unwrap(), "image/png"); + assert_eq!( + headers.get(header::X_CONTENT_TYPE_OPTIONS).unwrap(), + "nosniff" + ); + assert_eq!( + headers.get(header::CONTENT_SECURITY_POLICY).unwrap(), + "default-src 'none'; style-src 'unsafe-inline'; sandbox" + ); + let etag = headers + .get(header::ETAG) + .unwrap() + .to_str() + .unwrap() + .to_string(); + assert!(body.starts_with(b"\x89PNG")); + + let mut cached = HeaderMap::new(); + cached.insert( + header::IF_NONE_MATCH, + http::HeaderValue::from_str(&etag).unwrap(), + ); + let (status, _, _) = get_with_headers( + &world, + &format!("/xrpc/sh.tangled.repo.blob?repo={did}&ref=main&path=logo.png&raw=true"), + cached, + ) + .await; + assert_eq!(status, StatusCode::NOT_MODIFIED); + + let mut weak = HeaderMap::new(); + weak.insert( + header::IF_NONE_MATCH, + http::HeaderValue::from_str(&format!("W/{etag}")).unwrap(), + ); + let (status, _, _) = get_with_headers( + &world, + &format!("/xrpc/sh.tangled.repo.blob?repo={did}&ref=main&path=logo.png&raw=true"), + weak, + ) + .await; + assert_eq!( + status, + StatusCode::NOT_MODIFIED, + "weak validator must revalidate too" + ); + assert_eq!( + get_error( + &world, + &format!("/xrpc/sh.tangled.repo.blob?repo={did}&ref=main&path=ghost.txt"), + ) + .await, + (StatusCode::NOT_FOUND, "FileNotFound".to_string()) + ); +} + +#[tokio::test] +async fn tree_directory_last_commit_is_the_newest_touching_commit() { + let world = World::new(); + let (did, bare, work) = empty_repo(&world, "periwinkle"); + let work = work.path(); + commit_file( + work, + "src/a.rs", + b"fn a() {}\n", + "add a", + "2026-06-01T12:30:00+02:00", + ); + let older = sh_git(work, &["rev-parse", "HEAD"]); + commit_file( + work, + "src/b.rs", + b"fn b() {}\n", + "add b", + "2026-06-01T12:31:00+02:00", + ); + let newer = sh_git(work, &["rev-parse", "HEAD"]); + commit_file( + work, + "README.md", + b"# periwinkle\n", + "doc", + "2026-06-01T12:33:00+02:00", + ); + let head = sh_git(work, &["rev-parse", "HEAD"]); + sh_git(work, &["push", "-q", &bare, "main"]); + + let value = get_json( + &world, + &format!("/xrpc/sh.tangled.repo.tree?repo={did}&ref=main"), + ) + .await; + let files = value["files"].as_array().unwrap(); + let src = files + .iter() + .find(|file| file["name"] == "src") + .expect("src directory is listed"); + let reported = src["last_commit"]["hash"].as_str().unwrap(); + assert_eq!( + reported, newer, + "the newest commit that touches the subtree is reported, not the oldest" + ); + assert_ne!(reported, older, "not the first commit that created subtree"); + assert_ne!( + reported, head, + "head commit only touched README, never the src subtree" + ); +} + +#[tokio::test] +async fn languages_timeout_yields_a_partial_answer_not_an_error() { + let world = World::new(); + let (did, work) = seeded(&world, "scallop"); + let head = Oid::from_hex(&sh_git(work.path(), &["rev-parse", "HEAD"])).unwrap(); + let repo = world.layout.open(&did).unwrap(); + + let full = + knot_langs::analyze(&repo, head, Some(Instant::now() + Duration::from_secs(60))).unwrap(); + assert!( + full.values().any(|size| size.get() > 0), + "generous budget detects code" + ); + + let expired = Instant::now() + .checked_sub(Duration::from_secs(1)) + .unwrap_or_else(Instant::now); + let partial = knot_langs::analyze(&repo, head, Some(expired)).unwrap(); + assert!( + partial.is_empty(), + "exhausted budget breaks the walk and returns the partial map gathered so far, never an error" + ); +} + +#[tokio::test] +async fn compare_format_patch_keeps_a_non_ascii_author_raw() { + let world = World::new(); + let (did, bare, work) = empty_repo(&world, "mussel"); + let work = work.path(); + commit_file( + work, + "README.md", + b"# mussel\n", + "first", + "2026-06-01T12:30:00+02:00", + ); + let base = sh_git(work, &["rev-parse", "HEAD"]); + std::fs::write(work.join("src.rs"), b"fn main() {}\n").unwrap(); + let author = ("Lýna Þórsdóttir", "lyna@nel.pet"); + git_run(work, "2026-06-01T12:31:00+02:00", author, &["add", "-A"]); + git_run( + work, + "2026-06-01T12:31:00+02:00", + author, + &["commit", "-q", "-m", "café changes"], + ); + let head = sh_git(work, &["rev-parse", "HEAD"]); + sh_git(work, &["push", "-q", &bare, "main"]); + + let value = get_json( + &world, + &format!("/xrpc/sh.tangled.repo.compare?repo={did}&rev1={base}&rev2={head}"), + ) + .await; + let entry = &value["format_patch"][0]; + assert_eq!( + entry["Author"]["Name"], "Lýna Þórsdóttir", + "structured author the appview renders keeps the raw unicode" + ); + assert_eq!(entry["Author"]["Email"], "lyna@nel.pet"); + assert_eq!( + entry["Title"], "café changes", + "structured subject the appview renders keeps the raw unicode" + ); + assert_eq!(entry["RawHeaders"]["Subject"][0], "[PATCH] café changes"); + assert_eq!( + entry["RawHeaders"]["From"][0], + "Lýna Þórsdóttir " + ); + let raw = entry["Raw"].as_str().unwrap(); + assert!( + raw.contains("From: Lýna Þórsdóttir "), + "knot emits the raw UTF-8 author instead of RFC2047 Q-encoding real format-patch uses" + ); + assert!( + !raw.contains("=?UTF-8?") && !raw.contains("=?utf-8?"), + "no MIME word-encoding headers" + ); + assert!(raw.contains("Subject: [PATCH] café changes")); + assert!( + raw.ends_with("-- \nknot"), + "knot signs the patch w/ its own trailer" + ); +} + +#[tokio::test] +async fn diff_reports_structured_fragments_and_stats() { + let world = World::new(); + let (did, work) = seeded(&world, "whelk"); + let head = sh_git(work.path(), &["rev-parse", "HEAD"]); + + let value = get_json( + &world, + &format!("/xrpc/sh.tangled.repo.diff?repo={did}&ref={head}"), + ) + .await; + assert_eq!(value["ref"], head.as_str()); + let diff = &value["diff"]; + assert_eq!(diff["stat"]["files_changed"].as_i64(), Some(1)); + assert_eq!(diff["stat"]["insertions"].as_i64(), Some(1)); + assert_eq!(diff["stat"]["deletions"].as_i64(), Some(1)); + let file = &diff["diff"][0]; + assert_eq!(file["name"]["new"], "README.md"); + assert_eq!(file["is_new"], false); + let fragment = &file["text_fragments"][0]; + assert_eq!(fragment["OldPosition"].as_i64(), Some(1)); + assert_eq!(fragment["Comment"], ""); + let lines = fragment["Lines"].as_array().unwrap(); + assert!( + lines + .iter() + .any(|line| line["Op"].as_i64() == Some(1) && line["Line"] == "hello\n") + ); + assert!( + lines + .iter() + .any(|line| line["Op"].as_i64() == Some(2) && line["Line"] == "hello reef\n") + ); + assert_eq!(diff["commit"]["this"], head.as_str()); +} + +#[tokio::test] +async fn compare_produces_format_patches_and_a_combined_patch() { + let world = World::new(); + let (did, work) = seeded(&world, "conch"); + let head = sh_git(work.path(), &["rev-parse", "HEAD"]); + let base = sh_git(work.path(), &["rev-parse", "HEAD~3"]); + + let value = get_json( + &world, + &format!("/xrpc/sh.tangled.repo.compare?repo={did}&rev1={base}&rev2={head}"), + ) + .await; + assert_eq!(value["rev1"], base.as_str()); + assert_eq!(value["rev2"], head.as_str()); + let patches = value["format_patch"].as_array().unwrap(); + assert_eq!(patches.len(), 3, "three commits separate base from head"); + let first = &patches[0]; + assert_eq!(first["Title"], "add main"); + assert_eq!(first["SubjectPrefix"], "[PATCH] "); + assert_eq!(first["Committer"], serde_json::Value::Null); + assert_eq!(first["CommitterDate"], "0001-01-01T00:00:00Z"); + assert_eq!(first["Author"]["Name"], "nel"); + assert_eq!(first["AuthorDate"], "2026-06-01T12:31:00+02:00"); + assert_eq!(first["RawHeaders"]["Subject"][0], "[PATCH] add main"); + let raw = first["Raw"].as_str().unwrap(); + assert!(raw.starts_with(&format!( + "From {} Mon Sep 17 00:00:00 2001\n", + sh_git(work.path(), &["rev-parse", "HEAD~2"]) + ))); + assert!(raw.contains("Subject: [PATCH] add main")); + assert!(raw.contains("diff --git a/src/main.rs b/src/main.rs")); + assert!(raw.contains("new file mode 100644")); + assert!(first["Files"][0]["NewName"] == "src/main.rs"); + assert!(first["Files"][0]["IsNew"] == true); + + assert!(value["patch"].as_str().unwrap().contains("add logo")); + let combined = value["combined_patch"].as_array().unwrap(); + assert!(combined.iter().any(|file| file["NewName"] == "README.md")); + assert!( + value["combined_patch_raw"] + .as_str() + .unwrap() + .contains("diff --git a/README.md b/README.md") + ); + + assert_eq!( + get_error( + &world, + &format!("/xrpc/sh.tangled.repo.compare?repo={did}&rev1=nope&rev2={head}"), + ) + .await, + (StatusCode::BAD_REQUEST, "RevisionNotFound".to_string()) + ); +} + +#[tokio::test] +async fn archive_conditional_and_range_semantics() { + let world = World::new(); + let (did, _work) = seeded(&world, "nautilus"); + let path = format!("/xrpc/sh.tangled.repo.archive?repo={did}&ref=main"); + + let (status, headers, full) = get(&world, &path).await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + headers.get(header::CONTENT_TYPE).unwrap(), + "application/gzip" + ); + assert_eq!( + headers + .get(header::CONTENT_DISPOSITION) + .unwrap() + .to_str() + .unwrap(), + format!("attachment; filename=\"{did}-main.tar.gz\"") + ); + let link = headers.get(header::LINK).unwrap().to_str().unwrap(); + assert!(link.contains("rel=\"immutable\"")); + assert!(link.contains("/xrpc/sh.tangled.repo.archive?format=tar.gz")); + assert_eq!(headers.get(header::ACCEPT_RANGES).unwrap(), "bytes"); + let last_modified = headers + .get(header::LAST_MODIFIED) + .expect("a pinned modification time backs date revalidation") + .to_str() + .unwrap() + .to_string(); + let etag = headers + .get(header::ETAG) + .unwrap() + .to_str() + .unwrap() + .to_string(); + assert!( + etag.starts_with('"') && etag.ends_with('"'), + "a strong etag is quoted" + ); + assert_eq!(&full[..2], &[0x1f, 0x8b]); + + let mut range = HeaderMap::new(); + range.insert(header::RANGE, "bytes=0-3".parse().unwrap()); + let (status, range_headers, partial) = get_with_headers(&world, &path, range).await; + assert_eq!(status, StatusCode::PARTIAL_CONTENT); + assert_eq!( + range_headers + .get(header::CONTENT_RANGE) + .unwrap() + .to_str() + .unwrap(), + format!("bytes 0-3/{}", full.len()) + ); + assert_eq!( + partial.as_ref(), + &full[..4], + "a resumed range regenerates byte for byte" + ); + + let mut conditional = HeaderMap::new(); + conditional.insert(header::IF_NONE_MATCH, etag.parse().unwrap()); + let (status, cond_headers, conditional_body) = + get_with_headers(&world, &path, conditional).await; + assert_eq!(status, StatusCode::NOT_MODIFIED); + assert_eq!( + cond_headers.get(header::ETAG).unwrap().to_str().unwrap(), + etag + ); + assert!(conditional_body.is_empty(), "a 304 has no body"); + + let (status, _) = get_error( + &world, + &format!("/xrpc/sh.tangled.repo.archive?repo={did}&ref=main&format=tar.bz2"), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); + + let mut etag_match = HeaderMap::new(); + etag_match.insert(header::RANGE, "bytes=0-3".parse().unwrap()); + etag_match.insert(header::IF_RANGE, etag.parse().unwrap()); + let (status, if_range_headers, partial) = get_with_headers(&world, &path, etag_match).await; + assert_eq!( + status, + StatusCode::PARTIAL_CONTENT, + "a matching content etag resumes the range" + ); + assert_eq!( + if_range_headers + .get(header::CONTENT_RANGE) + .unwrap() + .to_str() + .unwrap(), + format!("bytes 0-3/{}", full.len()) + ); + assert_eq!(partial.as_ref(), &full[..4]); + + let mut etag_stale = HeaderMap::new(); + etag_stale.insert(header::RANGE, "bytes=0-3".parse().unwrap()); + etag_stale.insert(header::IF_RANGE, "\"0000\"".parse().unwrap()); + let (status, _, body) = get_with_headers(&world, &path, etag_stale).await; + assert_eq!( + status, + StatusCode::OK, + "a stale content etag falls back to the full body" + ); + assert_eq!(body, full, "the full archive comes back byte for byte"); + + let mut weak = HeaderMap::new(); + weak.insert(header::RANGE, "bytes=0-3".parse().unwrap()); + weak.insert(header::IF_RANGE, format!("W/{etag}").parse().unwrap()); + let (status, _, body) = get_with_headers(&world, &path, weak).await; + assert_eq!( + status, + StatusCode::OK, + "a weak validator never serves a range, per strong-comparison rules" + ); + assert_eq!(body, full); + + let mut date_match = HeaderMap::new(); + date_match.insert(header::RANGE, "bytes=0-3".parse().unwrap()); + date_match.insert(header::IF_RANGE, last_modified.parse().unwrap()); + let (status, date_headers, partial) = get_with_headers(&world, &path, date_match).await; + assert_eq!( + status, + StatusCode::PARTIAL_CONTENT, + "a date matching the pinned last-modified resumes the range" + ); + assert_eq!( + date_headers + .get(header::CONTENT_RANGE) + .unwrap() + .to_str() + .unwrap(), + format!("bytes 0-3/{}", full.len()) + ); + assert_eq!(partial.as_ref(), &full[..4]); + + let mut date_stale = HeaderMap::new(); + date_stale.insert(header::RANGE, "bytes=0-3".parse().unwrap()); + date_stale.insert( + header::IF_RANGE, + "Wed, 21 Oct 2015 07:28:00 GMT".parse().unwrap(), + ); + let (status, _, body) = get_with_headers(&world, &path, date_stale).await; + assert_eq!( + status, + StatusCode::OK, + "a date that doesn't match the pinned last-modified falls back to the full body" + ); + assert_eq!(body, full); + + assert_immutable_round_trip(&world, &headers, &full, &etag).await; +} + +#[tokio::test] +async fn archive_etag_distinguishes_refs_that_share_a_commit() { + let world = World::new(); + let (did, work) = seeded(&world, "scallop"); + let bare = world.layout.repo_path(&did).unwrap(); + sh_git(work.path(), &["branch", "release", "main"]); + sh_git( + work.path(), + &["push", "-q", bare.to_str().unwrap(), "refs/heads/release"], + ); + + let (main_etag, _main_last_modified, main_body) = archive_full(&world, &did).await; + let (status, release_headers, release_body) = get( + &world, + &format!("/xrpc/sh.tangled.repo.archive?repo={did}&ref=release"), + ) + .await; + assert_eq!(status, StatusCode::OK); + let release_etag = release_headers + .get(header::ETAG) + .unwrap() + .to_str() + .unwrap() + .to_string(); + assert_ne!( + main_etag, release_etag, + "two refs at one commit name different archive prefixes, so the strong etag must differ" + ); + assert_ne!( + main_body, release_body, + "the archives use different top-level directories and differ byte for byte" + ); + + let mut conditional = HeaderMap::new(); + conditional.insert(header::IF_NONE_MATCH, main_etag.parse().unwrap()); + let (status, _, _) = get_with_headers( + &world, + &format!("/xrpc/sh.tangled.repo.archive?repo={did}&ref=release"), + conditional, + ) + .await; + assert_eq!( + status, + StatusCode::OK, + "main's etag mustn't satisfy a conditional request for the release archive" + ); +} + +#[tokio::test] +async fn archive_serves_a_sha256_repo_with_a_stable_etag() { + let world = World::sha256(); + let (did, _work) = seeded_with_format(&world, "nautilus", ObjectFormat::SHA256); + + let (status, headers, full) = get( + &world, + &format!("/xrpc/sh.tangled.repo.archive?repo={did}&ref=main"), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + headers.get(header::CONTENT_TYPE).unwrap(), + "application/gzip" + ); + assert_eq!(&full[..2], &[0x1f, 0x8b]); + let etag = headers + .get(header::ETAG) + .unwrap() + .to_str() + .unwrap() + .to_string(); + + assert_immutable_round_trip(&world, &headers, &full, &etag).await; + + let mut conditional = HeaderMap::new(); + conditional.insert(header::IF_NONE_MATCH, etag.parse().unwrap()); + let (status, _, body) = get_with_headers( + &world, + &format!("/xrpc/sh.tangled.repo.archive?repo={did}&ref=main"), + conditional, + ) + .await; + assert_eq!( + status, + StatusCode::NOT_MODIFIED, + "conditional revalidation works under sha256" + ); + assert!(body.is_empty()); +} + +#[tokio::test] +async fn languages_detect_rust_and_markdown_stays_out() { + let world = World::new(); + let (did, _work) = seeded(&world, "uni"); + + let value = get_json( + &world, + &format!("/xrpc/sh.tangled.repo.languages?repo={did}&ref=main"), + ) + .await; + let languages = value["languages"].as_array().unwrap(); + assert_eq!( + languages.len(), + 1, + "only Rust counts: markdown is prose, png is binary" + ); + assert_eq!(languages[0]["name"], "Rust"); + assert_eq!(languages[0]["percentage"].as_i64(), Some(100)); + assert!(languages[0]["size"].as_i64().unwrap() > 0); + assert_eq!(value["totalFiles"].as_i64(), Some(1)); +} + +#[tokio::test] +async fn repo_metadata_resolves_and_fails_closed() { + let world = World::new(); + let (did, _work) = seeded(&world, "cuttle"); + + let value = get_json( + &world, + &format!("/xrpc/sh.tangled.repo.getDefaultBranch?repo={did}"), + ) + .await; + assert_eq!(value["name"], "main"); + assert_eq!(value["hash"], ""); + assert_eq!(value["when"], "1970-01-01T00:00:00Z"); + + let described = get_json( + &world, + &format!("/xrpc/sh.tangled.repo.describeRepo?repoDid={did}"), + ) + .await; + assert_eq!(described["repoDid"], did.as_str()); + assert_eq!(described["ownerDid"], OWNER); + assert_eq!(described["rkey"], "cuttle"); + assert_eq!( + get_error( + &world, + "/xrpc/sh.tangled.repo.describeRepo?repoDid=did:plc:doesnotexist", + ) + .await, + (StatusCode::NOT_FOUND, "RepoNotFound".to_string()) + ); + + let by_owner = get_json( + &world, + &format!("/xrpc/sh.tangled.repo.getDefaultBranch?repo={OWNER}/cuttle"), + ) + .await; + assert_eq!(by_owner["name"], "main"); + assert_eq!( + get_error( + &world, + "/xrpc/sh.tangled.repo.getDefaultBranch?repo=did:plc:unregistered", + ) + .await, + (StatusCode::NOT_FOUND, "RepoNotFound".to_string()) + ); + let (status, _) = get_error(&world, "/xrpc/sh.tangled.repo.getDefaultBranch?repo=oyster").await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!( + get_error( + &world, + &format!("/xrpc/sh.tangled.repo.log?repo={did}&ref=mangrove"), + ) + .await, + (StatusCode::NOT_FOUND, "RefNotFound".to_string()) + ); + + let (status, error) = get_error(&world, "/xrpc/sh.tangled.repo.getDefaultBranch").await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!( + error, "InvalidRequest", + "a structurally unsound query gets the lexicon error shape instead of the runtime's default plaintext" + ); +} + +#[tokio::test] +async fn list_refs_reports_paginates_and_drains() { + let world = World::new(); + let (did, work) = seeded(&world, "whelk"); + let head = sh_git(work.path(), &["rev-parse", "HEAD"]); + + let value = get_json(&world, &format!("/xrpc/sh.tangled.git.listRefs?repo={did}")).await; + assert_eq!( + ref_names(&value, "refs"), + vec![ + "refs/heads/main", + "refs/tags/lightweight", + "refs/tags/v1.0.0" + ] + ); + let main = value["refs"] + .as_array() + .unwrap() + .iter() + .find(|entry| entry["ref"] == "refs/heads/main") + .unwrap(); + assert_eq!(main["sha"], head); + assert_eq!(value["defaultBranch"]["ref"], "refs/heads/main"); + assert_eq!(value["defaultBranch"]["head"], head); + assert!(value["cursor"].is_null()); + + let first = get_json( + &world, + &format!("/xrpc/sh.tangled.git.listRefs?repo={did}&limit=2"), + ) + .await; + assert_eq!(ref_names(&first, "refs").len(), 2); + let cursor = first["cursor"].as_str().unwrap().to_string(); + let second = get_json( + &world, + &format!("/xrpc/sh.tangled.git.listRefs?repo={did}&limit=2&cursor={cursor}"), + ) + .await; + assert_eq!(ref_names(&second, "refs").len(), 1); + assert!(second["cursor"].is_null()); + + let refs = get_json( + &world, + &format!( + "/xrpc/sh.tangled.git.listRefs?repo={did}&cursor={}", + usize::MAX + ), + ) + .await; + assert!(ref_names(&refs, "refs").is_empty()); + assert!(refs["cursor"].is_null()); + let repos = get_json( + &world, + &format!("/xrpc/sh.tangled.sync.listRepos?cursor={}", usize::MAX), + ) + .await; + assert!(repo_dids(&repos).is_empty()); + assert!(repos["cursor"].is_null()); +} + +#[tokio::test] +async fn the_cob_ref_namespace_is_invisible_across_every_read() { + let world = World::new(); + let (did, work) = seeded(&world, "anemone"); + let head = sh_git(work.path(), &["rev-parse", "HEAD"]); + let bare = world.layout.repo_path(&did).unwrap(); + let cob = "refs/cobs/sh.tangled.repo.collaborator/x"; + sh_git(bare.as_path(), &["update-ref", cob, &head]); + + let value = get_json(&world, &format!("/xrpc/sh.tangled.git.listRefs?repo={did}")).await; + assert!( + ref_names(&value, "refs") + .iter() + .all(|name| !name.starts_with("refs/cobs/")), + "reserved cob ref leaked into listRefs" + ); + + let w = &world; + let d = &did; + let named_routes: &[(&str, &str)] = &[ + ("log", ""), + ("tree", ""), + ("blob", "&path=README.md"), + ("diff", ""), + ("archive", ""), + ("languages", ""), + ]; + stream::iter(named_routes) + .for_each(|&(route, suffix)| async move { + let (status, _) = get_error( + w, + &format!("/xrpc/sh.tangled.repo.{route}?repo={d}&ref={cob}{suffix}"), + ) + .await; + assert_eq!( + status, + StatusCode::NOT_FOUND, + "{route} mustn't resolve reserved cobs ref" + ); + }) + .await; + let (status, _) = get_error( + &world, + &format!("/xrpc/sh.tangled.repo.log?repo={did}&ref=cobs/sh.tangled.repo.collaborator/x"), + ) + .await; + assert_eq!( + status, + StatusCode::NOT_FOUND, + "reserved namespace shorthand mustn't resolve either" + ); + let (status, _) = get_error( + &world, + &format!("/xrpc/sh.tangled.repo.compare?repo={did}&rev1={cob}&rev2={head}"), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); + + commit_file( + work.path(), + "secret.txt", + b"hidden\n", + "secret", + "2026-06-01T12:40:00+02:00", + ); + let hidden = sh_git(work.path(), &["rev-parse", "HEAD"]); + sh_git( + work.path(), + &[ + "push", + "-q", + bare.to_str().unwrap(), + "HEAD:refs/cobs/sh.tangled.repo.collaborator/secret", + ], + ); + + let hidden_ref = &hidden; + let hidden_routes: &[(&str, &str)] = &[ + ("log", ""), + ("tree", "&path=secret.txt"), + ("diff", ""), + ("archive", ""), + ("languages", ""), + ]; + stream::iter(hidden_routes) + .for_each(|&(route, suffix)| async move { + let (status, _) = get_error( + w, + &format!("/xrpc/sh.tangled.repo.{route}?repo={d}&ref={hidden_ref}{suffix}"), + ) + .await; + assert_eq!( + status, + StatusCode::NOT_FOUND, + "{route} mustn't serve a commit reachable only through cob ref" + ); + }) + .await; + let (status, _) = get_error( + &world, + &format!("/xrpc/sh.tangled.repo.blob?repo={did}&ref={hidden}&path=secret.txt"), + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + let (status, error) = get_error( + &world, + &format!("/xrpc/sh.tangled.repo.compare?repo={did}&rev1={head}&rev2={hidden}"), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(error, "RevisionNotFound"); + + let still_public = get_json( + &world, + &format!("/xrpc/sh.tangled.repo.log?repo={did}&ref={head}"), + ) + .await; + assert_eq!(still_public["total"].as_i64(), Some(4)); +} + +#[tokio::test] +async fn the_list_reads_reject_malformed_paging_params() { + let world = World::new(); + let (did, _work) = seeded(&world, "barnacle"); + + let queries: Vec = vec![ + format!("/xrpc/sh.tangled.git.listRefs?repo={did}&limit=abc"), + format!("/xrpc/sh.tangled.git.listRefs?repo={did}&cursor=notanint"), + "/xrpc/sh.tangled.sync.listRepos?limit=abc".to_string(), + "/xrpc/sh.tangled.sync.listRepos?cursor=notanint".to_string(), + "/xrpc/sh.tangled.sync.listRepos?order=sideways".to_string(), + ]; + let w = &world; + stream::iter(queries.iter()) + .for_each(|query| async move { + let (status, error) = get_error(w, query).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "query {query}"); + assert_eq!(error, "InvalidRequest", "query {query}"); + }) + .await; + + let clamped = get_json( + &world, + &format!("/xrpc/sh.tangled.git.listRefs?repo={did}&limit=5000"), + ) + .await; + assert_eq!( + ref_names(&clamped, "refs").len(), + 3, + "an oversize limit clamps to the max instead of erroring" + ); +} + +#[tokio::test] +async fn list_repos_lists_hosted_repos_with_order_and_pagination() { + let world = World::new(); + let (mussel, _a) = seeded(&world, "mussel"); + let (nautilus, _b) = seeded(&world, "nautilus"); + let (scallop, _c) = seeded(&world, "scallop"); + + let desc = get_json(&world, "/xrpc/sh.tangled.sync.listRepos").await; + assert_eq!( + repo_dids(&desc), + vec![ + scallop.as_str().to_string(), + nautilus.as_str().to_string(), + mussel.as_str().to_string(), + ] + ); + assert_eq!(desc["repos"][0]["status"], "active"); + assert_eq!(desc["repos"][0]["defaultBranch"]["ref"], "refs/heads/main"); + + let asc = get_json(&world, "/xrpc/sh.tangled.sync.listRepos?order=asc").await; + assert_eq!( + repo_dids(&asc), + vec![ + mussel.as_str().to_string(), + nautilus.as_str().to_string(), + scallop.as_str().to_string(), + ] + ); + + let page = get_json(&world, "/xrpc/sh.tangled.sync.listRepos?order=asc&limit=2").await; + assert_eq!(page["repos"].as_array().unwrap().len(), 2); + let cursor = page["cursor"].as_str().unwrap().to_string(); + let rest = get_json( + &world, + &format!("/xrpc/sh.tangled.sync.listRepos?order=asc&limit=2&cursor={cursor}"), + ) + .await; + assert_eq!(repo_dids(&rest), vec![scallop.as_str().to_string()]); + assert!(rest["cursor"].is_null()); +} + +#[tokio::test] +async fn every_projection_read_fails_closed_while_warming() { + let world = World::warming(); + let did = RepoDid::new("did:plc:limpetfixture").unwrap(); + let cases: Vec<(String, Option<&str>)> = vec![ + ( + "/xrpc/sh.tangled.sync.listRepos".to_string(), + Some("ProjectionWarming"), + ), + ( + format!("/xrpc/sh.tangled.repo.getDefaultBranch?repo={did}"), + Some("ProjectionWarming"), + ), + ( + format!("/xrpc/sh.tangled.repo.describeRepo?repoDid={did}"), + Some("ProjectionWarming"), + ), + ( + "/xrpc/sh.tangled.knot.listMembers?subject=did:web:knot.nel.pet".to_string(), + None, + ), + ( + "/xrpc/sh.tangled.repo.listCollaborators?subject=did:plc:squid".to_string(), + None, + ), + ]; + let w = &world; + stream::iter(cases.iter()) + .for_each(|(path, expected)| async move { + assert_warming(w, path, *expected).await; + }) + .await; +} + +#[tokio::test] +async fn branch_tips_render_edge_shapes() { + let world = World::new(); + let (did, work) = seeded(&world, "trochus"); + let bare = world.layout.repo_path(&did).unwrap(); + let bare_str = bare.to_str().unwrap().to_string(); + + sh_git(work.path(), &["checkout", "-q", "-b", "side", "HEAD~1"]); + commit_file( + work.path(), + "side.txt", + b"side\n", + "side work", + "2026-06-01T12:34:00+02:00", + ); + sh_git(work.path(), &["checkout", "-q", "main"]); + sh_git_at( + work.path(), + "2026-06-01T12:35:00+02:00", + &["merge", "-q", "--no-ff", "-m", "merge side", "side"], + ); + sh_git(work.path(), &["push", "-q", &bare_str, "main"]); + let first_parent = sh_git(work.path(), &["rev-parse", "HEAD^1"]); + let second_parent = sh_git(work.path(), &["rev-parse", "HEAD^2"]); + + let tag_object = sh_git(work.path(), &["rev-parse", "v1.0.0"]); + std::fs::write(bare.join("refs/heads/tagtip"), format!("{tag_object}\n")).unwrap(); + let root_commit = sh_git(work.path(), &["rev-list", "--max-parents=0", "HEAD"]); + std::fs::write(bare.join("refs/heads/roottip"), format!("{root_commit}\n")).unwrap(); + + let value = get_json( + &world, + &format!("/xrpc/sh.tangled.repo.branches?repo={did}"), + ) + .await; + let branches = value["branches"].as_array().unwrap(); + assert_eq!(branches.len(), 3); + + let branch = |name: &str| { + branches + .iter() + .find(|branch| branch["reference"]["name"] == name) + .unwrap() + }; + let parents = |branch: &serde_json::Value| -> Vec { + branch["commit"]["ParentHashes"] + .as_array() + .unwrap() + .iter() + .map(|parent| { + parent + .as_array() + .unwrap() + .iter() + .map(|byte| format!("{:02x}", byte.as_u64().unwrap())) + .collect() + }) + .collect() + }; + + let main = branch("main"); + assert_eq!( + parents(main), + vec![first_parent, second_parent], + "merge tip must report both parents in order" + ); + assert_eq!(main["commit"]["Author"]["Name"], "nel"); + assert!( + parents(branch("roottip")).is_empty(), + "root tip must report no parents" + ); + + let tagtip = branch("tagtip"); + assert!( + parents(tagtip).is_empty(), + "a non-commit tip must report no parents" + ); + assert_eq!(tagtip["reference"]["hash"], tag_object.as_str()); + assert_eq!(tagtip["commit"]["Author"]["Name"], ""); + assert_eq!(tagtip["commit"]["Author"]["When"], "0001-01-01T00:00:00Z"); + assert_eq!(tagtip["commit"]["Message"], "release one"); + assert!( + tagtip["commit"]["TreeHash"] + .as_array() + .unwrap() + .iter() + .all(|byte| byte.as_u64() == Some(0)), + "opaque tip has the zero tree hash" + ); +} + +#[tokio::test] +async fn a_submodule_path_in_the_tree_is_path_not_found() { + let world = World::new(); + let (did, work) = seeded(&world, "razorclam"); + let bare = world + .layout + .repo_path(&did) + .unwrap() + .to_str() + .unwrap() + .to_string(); + let head = sh_git(work.path(), &["rev-parse", "HEAD"]); + sh_git( + work.path(), + &[ + "update-index", + "--add", + "--cacheinfo", + &format!("160000,{head},vendor/dep"), + ], + ); + sh_git(work.path(), &["commit", "-q", "-m", "add gitlink"]); + sh_git(work.path(), &["push", "-q", &bare, "main"]); + + assert_eq!( + get_error( + &world, + &format!("/xrpc/sh.tangled.repo.tree?repo={did}&ref=main&path=vendor/dep"), + ) + .await, + (StatusCode::NOT_FOUND, "PathNotFound".to_string()) + ); +} + +#[tokio::test] +async fn a_blob_past_the_derived_serving_limit_is_a_named_error() { + let world = World::with_response_limit(ResponseLimit::new(1024)); + let (did, bare, work) = empty_repo(&world, "auger"); + commit_file( + work.path(), + "big.txt", + "a".repeat(2_000).as_bytes(), + "big file", + "2026-06-01T12:30:00+02:00", + ); + sh_git(work.path(), &["push", "-q", &bare, "main"]); + + let (status, error) = get_error( + &world, + &format!("/xrpc/sh.tangled.repo.blob?repo={did}&ref=main&path=big.txt"), + ) + .await; + assert_eq!(status, StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!( + error, "BlobTooLarge", + "blob limit is reached before the generic response limit" + ); + + let (status, _, body) = get( + &world, + &format!("/xrpc/sh.tangled.repo.blob?repo={did}&ref=main&path=big.txt&raw=true"), + ) + .await; + assert_eq!(status, StatusCode::OK, "raw serving keeps full limit"); + assert_eq!(body.len(), 2_000); +} + +#[tokio::test] +async fn an_oversized_readme_is_omitted_from_the_tree() { + let world = World::with_response_limit(ResponseLimit::new(2_048)); + let (did, bare, work) = empty_repo(&world, "cowrie"); + commit_file( + work.path(), + "README.md", + format!("# reef\n\n{}\n", "r".repeat(1_000)).as_bytes(), + "huge readme", + "2026-06-01T12:30:00+02:00", + ); + sh_git(work.path(), &["push", "-q", &bare, "main"]); + + let value = get_json( + &world, + &format!("/xrpc/sh.tangled.repo.tree?repo={did}&ref=main"), + ) + .await; + assert_eq!(value["files"][0]["name"], "README.md"); + assert_eq!( + value["readme"]["contents"], "", + "readme past the serving limit is omitted instead of failing the whole tree" + ); +} + +#[tokio::test] +async fn a_comparison_spanning_too_many_commits_is_refused() { + let world = World::new(); + let (did, work) = seeded(&world, "abalone"); + let bare = world + .layout + .repo_path(&did) + .unwrap() + .to_str() + .unwrap() + .to_string(); + let base = sh_git(work.path(), &["rev-parse", "HEAD"]); + (0..501).for_each(|index| { + sh_git( + work.path(), + &["commit", "-q", "--allow-empty", "-m", &format!("c{index}")], + ); + }); + sh_git(work.path(), &["push", "-q", &bare, "main"]); + let head = sh_git(work.path(), &["rev-parse", "HEAD"]); + + let (status, error) = get_error( + &world, + &format!("/xrpc/sh.tangled.repo.compare?repo={did}&rev1={base}&rev2={head}"), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(error, "CompareError"); +} + +#[tokio::test] +async fn archive_rejects_traversal_prefixes_and_sanitizes_the_filename() { + let world = World::new(); + let (did, work) = seeded(&world, "cockle"); + let bare = world + .layout + .repo_path(&did) + .unwrap() + .to_str() + .unwrap() + .to_string(); + + let (status, error) = get_error( + &world, + &format!("/xrpc/sh.tangled.repo.archive?repo={did}&ref=main&prefix=../evil"), + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(error, "InvalidRequest"); + + sh_git(work.path(), &["branch", "a\"b"]); + sh_git(work.path(), &["push", "-q", &bare, "refs/heads/a\"b"]); + let (status, headers, _) = get( + &world, + &format!("/xrpc/sh.tangled.repo.archive?repo={did}&ref=a%22b"), + ) + .await; + assert_eq!(status, StatusCode::OK); + let disposition = headers + .get(header::CONTENT_DISPOSITION) + .unwrap() + .to_str() + .unwrap(); + assert_eq!( + disposition, + format!("attachment; filename=\"{did}-a-b.tar.gz\""), + "quote in the ref name mustn't break the header quoting" + ); +} + +#[tokio::test] +async fn an_archive_larger_than_the_configured_limit_is_refused() { + let world = World::with_archive_limit(ArchiveLimit::new(64)); + let (did, _work) = seeded(&world, "murex"); + + let (status, error) = get_error( + &world, + &format!("/xrpc/sh.tangled.repo.archive?repo={did}&ref=main"), + ) + .await; + assert_eq!(status, StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!(error, "RequestTooLarge"); +} + +#[tokio::test] +async fn an_oversized_read_response_is_refused() { + let world = World::with_response_limit(ResponseLimit::new(256)); + let (did, _work) = seeded(&world, "clam"); + + let (status, error) = get_error( + &world, + &format!("/xrpc/sh.tangled.repo.log?repo={did}&ref=main"), + ) + .await; + assert_eq!(status, StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!(error, "RequestTooLarge"); + + let small = get_json( + &world, + &format!("/xrpc/sh.tangled.repo.getDefaultBranch?repo={did}"), + ) + .await; + assert_eq!(small["name"], "main"); +} + +#[tokio::test] +async fn languages_omit_files_with_zero_size() { + let world = World::new(); + let (did, bare, work) = empty_repo(&world, "topshell"); + commit_file( + work.path(), + "lib.rs", + b"", + "empty rust file", + "2026-06-01T12:30:00+02:00", + ); + sh_git(work.path(), &["push", "-q", &bare, "main"]); + + let value = get_json( + &world, + &format!("/xrpc/sh.tangled.repo.languages?repo={did}&ref=main"), + ) + .await; + assert!( + value["languages"].is_null(), + "zero-byte file mustn't surface as a language with a NaN percentage" + ); + assert!(value.get("totalSize").is_none()); + assert!(value.get("totalFiles").is_none()); +} + +#[tokio::test] +async fn a_compare_patch_round_trips_through_merge_check() { + let world = World::new(); + let (_did, main_sha, feature_sha) = seeded_feature_branch(&world, "periwinkle"); + let registered = RepoDid::new("did:plc:periwinklefixture").unwrap(); + + let compared = get_json( + &world, + &format!( + "/xrpc/sh.tangled.repo.compare?repo={registered}&rev1={main_sha}&rev2={feature_sha}" + ), + ) + .await; + let patch = compared["patch"].as_str().unwrap(); + + let (status, check) = post_json( + &world, + "/xrpc/sh.tangled.repo.mergeCheck", + serde_json::json!({ + "did": OWNER, + "name": "periwinkle", + "branch": "main", + "patch": patch, + }), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + check["is_conflicted"], + serde_json::Value::Bool(false), + "knot's own compare output must pass its own merge check: {check}" + ); + + let (status, stale) = post_json( + &world, + "/xrpc/sh.tangled.repo.mergeCheck", + serde_json::json!({ + "did": OWNER, + "name": "periwinkle", + "branch": "feature", + "patch": patch, + }), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + stale["is_conflicted"], + serde_json::Value::Bool(true), + "re-applying an already-landed patch must conflict: {stale}" + ); +} + +#[tokio::test] +async fn list_members_pages_in_the_wire_shape() { + let world = World::new(); + world.add_member("did:plc:limpet", OWNER, 1_000); + world.add_member("did:plc:scallop", OWNER, 2_000); + world.add_member("did:plc:whelk", OWNER, 3_000); + + let page = get_json( + &world, + "/xrpc/sh.tangled.knot.listMembers?subject=did:web:knot.nel.pet&limit=2", + ) + .await; + let items = page["items"].as_array().unwrap(); + assert_eq!(items.len(), 2, "default order is createdAt descending"); + assert_eq!(items[0]["subject"], "did:plc:whelk"); + assert_eq!(items[0]["addedBy"], OWNER); + assert_eq!(items[0]["createdAt"], "1970-01-01T00:50:00Z"); + assert!( + items[0].get("uri").is_none() && items[0].get("cid").is_none(), + "knot-owned member has no backing record, so uri and cid are omitted" + ); + assert_eq!(items[1]["subject"], "did:plc:scallop"); + + let cursor = page["cursor"].as_str().unwrap(); + let next = get_json( + &world, + &format!( + "/xrpc/sh.tangled.knot.listMembers?subject=did:web:knot.nel.pet&limit=2&cursor={cursor}" + ), + ) + .await; + let items = next["items"].as_array().unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0]["subject"], "did:plc:limpet"); + assert!(next.get("cursor").is_none(), "drained list has no cursor"); + + let ascending = get_json( + &world, + "/xrpc/sh.tangled.knot.listMembers?subject=did:web:knot.nel.pet&order=asc&limit=1", + ) + .await; + assert_eq!(ascending["items"][0]["subject"], "did:plc:limpet"); +} + +#[tokio::test] +async fn list_members_rejects_malformed_params_and_clamps_the_limit() { + let world = World::new(); + world.add_member("did:plc:limpet", OWNER, 1_000); + + let queries: &[&str] = &[ + "limit=abc&subject=did:web:knot.nel.pet", + "cursor=notanint&subject=did:web:knot.nel.pet", + "order=ascending&subject=did:web:knot.nel.pet", + "limit=2", + "subject=knot.nel.pet", + ]; + let w = &world; + stream::iter(queries.iter().copied()) + .for_each(|query| async move { + let (status, _) = + get_error(w, &format!("/xrpc/sh.tangled.knot.listMembers?{query}")).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "query {query}"); + }) + .await; + + let clamped = get_json( + &world, + "/xrpc/sh.tangled.knot.listMembers?subject=did:web:knot.nel.pet&limit=5000", + ) + .await; + assert_eq!(clamped["items"].as_array().unwrap().len(), 1); +} + +#[tokio::test] +async fn list_collaborators_is_scoped_to_the_repo() { + let world = World::new(); + let squid = RepoDid::new("did:plc:squid").unwrap(); + let clam = RepoDid::new("did:plc:clam").unwrap(); + world.layout.create(&squid).unwrap(); + world.layout.create(&clam).unwrap(); + world.register(&squid, "squid"); + world.register(&clam, "clam"); + world.add_collaborator(&squid, "did:plc:lyna", OWNER, 1_000); + world.add_collaborator(&clam, "did:plc:bailey", OWNER, 2_000); + + let page = get_json( + &world, + "/xrpc/sh.tangled.repo.listCollaborators?subject=did:plc:squid", + ) + .await; + let items = page["items"].as_array().unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0]["subject"], "did:plc:lyna"); + assert_eq!(items[0]["addedBy"], OWNER); + assert_eq!(items[0]["createdAt"], "1970-01-01T00:16:40Z"); + assert!(items[0].get("uri").is_none() && items[0].get("cid").is_none()); + + let unknown = get_json( + &world, + "/xrpc/sh.tangled.repo.listCollaborators?subject=did:plc:unhosted", + ) + .await; + assert!( + unknown["items"].as_array().unwrap().is_empty(), + "unhosted repo has no collaborators, the answer is an empty list" + ); + + let (status, _) = get_error( + &world, + "/xrpc/sh.tangled.repo.listCollaborators?subject=notadid", + ) + .await; + assert_eq!(status, StatusCode::BAD_REQUEST); +} + +#[tokio::test] +async fn the_subject_tie_break_stays_ascending_in_both_directions() { + let world = World::new(); + world.add_member("did:plc:whelk", OWNER, 1_000); + world.add_member("did:plc:limpet", OWNER, 1_000); + + let subjects = |page: &serde_json::Value| -> Vec { + page["items"] + .as_array() + .unwrap() + .iter() + .map(|item| item["subject"].as_str().unwrap().to_string()) + .collect() + }; + + let asc = get_json( + &world, + "/xrpc/sh.tangled.knot.listMembers?subject=did:web:knot.nel.pet&order=asc", + ) + .await; + let desc = get_json( + &world, + "/xrpc/sh.tangled.knot.listMembers?subject=did:web:knot.nel.pet&order=desc", + ) + .await; + assert_eq!(subjects(&asc), vec!["did:plc:limpet", "did:plc:whelk"]); + assert_eq!( + subjects(&desc), + vec!["did:plc:limpet", "did:plc:whelk"], + "equal-createdAt entries keep an ascending subject tie-break regardless of sort direction" + ); +} + +#[tokio::test] +async fn repo_error_outranks_paging_in_any_param_order() { + let world = World::new(); + let orders: &[&str] = &[ + "/xrpc/sh.tangled.repo.listCollaborators?subject=notadid&limit=abc", + "/xrpc/sh.tangled.repo.listCollaborators?limit=abc&subject=notadid", + ]; + let w = &world; + stream::iter(orders.iter().copied()) + .for_each(|query| async move { + let (_, error) = get_error(w, query).await; + assert_eq!( + error, "InvalidRepo", + "the repo extractor runs before paging by signature position for {query}" + ); + }) + .await; +} + +#[tokio::test] +async fn service_metadata_endpoints_answer() { + let world = World::new(); + let wire = get_json(&world, "/xrpc/sh.tangled.knot.version").await; + assert_eq!(wire["version"], "v1.15.0"); + assert_eq!(wire["capabilities"], serde_json::json!(["knot-acl"])); + + let owner = get_json(&world, "/xrpc/sh.tangled.owner").await; + assert_eq!(owner["owner"], OWNER); +} + +fn publish_update(world: &World, repo: &str) -> EventCursor { + world.state.events.publish(&GitRefUpdate::new( + RepoDid::new(repo).unwrap(), + Some(OwnerDid::new(OWNER).unwrap()), + AccountDid::new("did:plc:nel").unwrap(), + )) +} + +async fn serve_events(world: &World) -> std::net::SocketAddr { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let router = world.router.clone(); + tokio::spawn(async move { + axum::serve( + listener, + router.into_make_service_with_connect_info::(), + ) + .await + .unwrap(); + }); + addr +} + +type Ws = + tokio_tungstenite::WebSocketStream>; + +fn next_event(ws: &mut Ws) -> Pin + '_>> { + Box::pin(async move { + let received = tokio::time::timeout(std::time::Duration::from_secs(5), ws.next()) + .await + .expect("an event arrives within the timeout") + .expect("the stream stays open") + .expect("the frame is readable"); + match received { + tungstenite::Message::Text(text) => serde_json::from_str(text.as_str()).unwrap(), + _ => next_event(ws).await, + } + }) +} + +#[tokio::test] +async fn the_events_stream_replays_resumes_and_rejects_bad_cursors() { + let world = World::new(); + let first = publish_update(&world, "did:plc:squid"); + publish_update(&world, "did:plc:anemone"); + let addr = serve_events(&world).await; + + let (mut ws, _) = tokio_tungstenite::connect_async(format!("ws://{addr}/events")) + .await + .unwrap(); + let replayed_first = next_event(&mut ws).await; + let replayed_second = next_event(&mut ws).await; + assert_eq!(replayed_first["nsid"], "sh.tangled.git.refUpdate"); + assert_eq!(replayed_first["event"]["repo"], "did:plc:squid"); + assert_eq!(replayed_first["event"]["ownerDid"], OWNER); + assert_eq!(replayed_first["event"]["committerDid"], "did:plc:nel"); + assert_eq!(replayed_first["rkey"].as_str().unwrap().len(), 13); + assert_eq!(replayed_second["event"]["repo"], "did:plc:anemone"); + assert!( + replayed_first["created"].as_i64().unwrap() < replayed_second["created"].as_i64().unwrap() + ); + + publish_update(&world, "did:plc:whelk"); + let live = next_event(&mut ws).await; + assert_eq!(live["event"]["repo"], "did:plc:whelk"); + + let (mut resumed, _) = + tokio_tungstenite::connect_async(format!("ws://{addr}/events?cursor={}", first.get())) + .await + .unwrap(); + let resumed_event = next_event(&mut resumed).await; + assert_eq!( + resumed_event["event"]["repo"], "did:plc:anemone", + "a cursor resumes past the event it names" + ); + + let (mut garbled, _) = + tokio_tungstenite::connect_async(format!("ws://{addr}/events?cursor=banana")) + .await + .unwrap(); + let replayed = next_event(&mut garbled).await; + assert_eq!( + replayed["event"]["repo"], "did:plc:squid", + "a garbled cursor replays from the start" + ); +} + +fn refused(result: Result) { + match result { + Err(tungstenite::Error::Http(response)) => { + assert_eq!(response.status().as_u16(), 503); + } + Err(other) => panic!("expected an http refusal: {other}"), + Ok(_) => panic!("a subscriber past the limit connected"), + } +} + +#[tokio::test] +async fn a_subscriber_beyond_the_events_limit_is_refused() { + let world = World::new(); + let addr = serve_events(&world).await; + let saturated: Vec<_> = (0..16u8) + .map(|octet| { + world + .state + .subscriber_gate + .try_admit(std::net::IpAddr::V4(std::net::Ipv4Addr::new( + 10, 0, 0, octet, + ))) + .expect("distinct peers fill the global limit") + }) + .collect(); + refused(tokio_tungstenite::connect_async(format!("ws://{addr}/events")).await); + drop(saturated); +} + +#[tokio::test] +async fn a_single_peer_cannot_monopolize_the_events_stream() { + let world = World::new(); + let addr = serve_events(&world).await; + let held: Vec<_> = stream::iter(0..4) + .then(|_| async { + tokio_tungstenite::connect_async(format!("ws://{addr}/events")) + .await + .expect("a connection within the per-peer limit is admitted") + .0 + }) + .collect() + .await; + refused(tokio_tungstenite::connect_async(format!("ws://{addr}/events")).await); + drop(held); +} + +#[tokio::test] +async fn set_default_branch_resolves_an_at_uri_repo_and_an_existing_branch() { + let world = World::new(); + let (did, work) = seeded(&world, "coral"); + let bare = world.layout.repo_path(&did).unwrap(); + sh_git(work.path(), &["branch", "release", "main"]); + sh_git( + work.path(), + &["push", "-q", bare.to_str().unwrap(), "refs/heads/release"], + ); + + let (status, _) = post_authed( + &world, + "/xrpc/sh.tangled.repo.setDefaultBranch", + OWNER, + serde_json::json!({ + "repo": format!("at://{OWNER}/sh.tangled.repo/coral"), + "defaultBranch": "release", + }), + ) + .await; + assert_eq!(status, StatusCode::OK); + + let repo = world.layout.open(&did).unwrap(); + assert_eq!( + repo.default_branch().unwrap().as_str(), + "refs/heads/release", + "the default head moved to the requested branch" + ); +} + +#[tokio::test] +async fn delete_branch_removes_a_non_default_branch_then_reports_it_gone() { + let world = World::new(); + let (did, work) = seeded(&world, "kelp"); + let bare = world.layout.repo_path(&did).unwrap(); + sh_git(work.path(), &["branch", "feature", "main"]); + sh_git( + work.path(), + &["push", "-q", bare.to_str().unwrap(), "refs/heads/feature"], + ); + + let at = format!("at://{OWNER}/sh.tangled.repo/kelp"); + let (status, _) = post_authed( + &world, + "/xrpc/sh.tangled.repo.deleteBranch", + OWNER, + serde_json::json!({ "repo": at, "branch": "feature" }), + ) + .await; + assert_eq!(status, StatusCode::OK); + + let (status, body) = post_authed( + &world, + "/xrpc/sh.tangled.repo.deleteBranch", + OWNER, + serde_json::json!({ "repo": at, "branch": "feature" }), + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND, "second delete: {body}"); +} + +#[tokio::test] +async fn bad_post_bodies_are_invalid_request() { + let world = World::new(); + let (_kelp, _wk) = seeded(&world, "kelp"); + let (_barnacle, _wb) = seeded(&world, "barnacle"); + + let cases: &[(&str, serde_json::Value)] = &[ + ( + "/xrpc/sh.tangled.repo.setDefaultBranch", + serde_json::json!({ "repo": "not-an-at-uri", "defaultBranch": "main" }), + ), + ( + "/xrpc/sh.tangled.repo.deleteBranch", + serde_json::json!({ + "repo": format!("at://{OWNER}/sh.tangled.repo/kelp"), + "branch": "bad branch", + }), + ), + ( + "/xrpc/sh.tangled.repo.forkSync", + serde_json::json!({ "did": OWNER, "name": "barnacle", "branch": "bad branch" }), + ), + ( + "/xrpc/sh.tangled.repo.hiddenRef", + serde_json::json!({ "repo": "nope", "forkRef": "feature", "remoteRef": "main" }), + ), + ]; + let w = &world; + stream::iter(cases) + .for_each(|(path, value)| async move { + assert_post_rejected(w, path, OWNER, value.clone()).await; + }) + .await; +} + +#[tokio::test] +async fn merge_applies_a_plain_patch_under_the_supplied_author() { + let world = World::new(); + let (_did, main_sha, feature_sha) = seeded_feature_branch(&world, "mussel"); + let registered = RepoDid::new("did:plc:musselfixture").unwrap(); + + let compared = get_json( + &world, + &format!( + "/xrpc/sh.tangled.repo.compare?repo={registered}&rev1={main_sha}&rev2={feature_sha}" + ), + ) + .await; + let patch = compared["combined_patch_raw"].as_str().unwrap().to_string(); + + let (status, body) = post_authed( + &world, + "/xrpc/sh.tangled.repo.merge", + OWNER, + serde_json::json!({ + "did": OWNER, + "name": "mussel", + "branch": "main", + "patch": patch, + "authorName": "Teq", + "authorEmail": "teq@nel.pet", + "commitMessage": "merged kelp", + }), + ) + .await; + assert_eq!(status, StatusCode::OK, "merge failed: {body}"); + + let log = get_json( + &world, + &format!("/xrpc/sh.tangled.repo.log?repo={registered}&ref=main"), + ) + .await; + let top = &log["commits"][0]; + assert_eq!( + top["author"]["Name"], "Teq", + "the supplied author rode through" + ); + assert!( + top["message"].as_str().unwrap().contains("merged kelp"), + "the supplied commit message rode through: {}", + top["message"] + ); +} + +#[tokio::test] +async fn create_mints_a_did_plc_repo_with_the_requested_default_branch() { + let world = World::new(); + world.add_member(OWNER, OWNER, 1_000); + let (status, body) = post_authed( + &world, + "/xrpc/sh.tangled.repo.create", + OWNER, + serde_json::json!({ "rkey": "squidkey", "name": "squid", "defaultBranch": "trunk" }), + ) + .await; + assert_eq!(status, StatusCode::OK, "create failed: {body}"); + let repo_did = body["repoDid"].as_str().unwrap(); + assert!( + repo_did.starts_with("did:plc:"), + "minted a did:plc: {repo_did}" + ); + let did = RepoDid::new(repo_did).unwrap(); + let repo = world.layout.open(&did).unwrap(); + assert_eq!( + repo.default_branch().unwrap().as_str(), + "refs/heads/trunk", + "the requested default branch became HEAD" + ); +} diff --git a/knot2/third_party/gix-pack/src/find.rs b/knot2/third_party/gix-pack/src/find.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/find.rs @@ -0,0 +1,12 @@ +/// An Entry in a pack providing access to its data. +/// +/// Its commonly retrieved by reading from a pack index file followed by a read from a pack data file. +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[allow(missing_docs)] +pub struct Entry { + /// The pack-data encoded bytes of the pack data entry as present in the pack file, including the header followed by compressed data. + pub data: Vec, + /// The version of the pack file containing `data` + pub version: crate::data::Version, +} diff --git a/knot2/third_party/gix-pack/src/find_traits.rs b/knot2/third_party/gix-pack/src/find_traits.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/find_traits.rs @@ -0,0 +1,288 @@ +use crate::{data, find}; + +/// Describe how object can be located in an object store with built-in facilities to supports packs specifically. +/// +/// ## Notes +/// +/// Find effectively needs [generic associated types][issue] to allow a trait for the returned object type. +/// Until then, we will have to make due with explicit types and give them the potentially added features we want. +/// +/// Furthermore, despite this trait being in `gix-pack`, it leaks knowledge about objects potentially not being packed. +/// This is a necessary trade-off to allow this trait to live in `gix-pack` where it is used in functions to create a pack. +/// +/// [issue]: https://github.com/rust-lang/rust/issues/44265 +pub trait Find { + /// Returns true if the object exists in the database. + fn contains(&self, id: &gix_hash::oid) -> bool; + + /// Find an object matching `id` in the database while placing its raw, decoded data into `buffer`. + /// A `pack_cache` can be used to speed up subsequent lookups, set it to [`crate::cache::Never`] if the + /// workload isn't suitable for caching. + /// + /// Returns `Some((, ))` if it was present in the database, + /// or the error that occurred during lookup or object retrieval. + fn try_find<'a>( + &self, + id: &gix_hash::oid, + buffer: &'a mut Vec, + ) -> Result, Option)>, gix_object::find::Error> { + self.try_find_cached(id, buffer, &mut crate::cache::Never) + } + + /// Like [`Find::try_find()`], but with support for controlling the pack cache. + /// A `pack_cache` can be used to speed up subsequent lookups, set it to [`crate::cache::Never`] if the + /// workload isn't suitable for caching. + /// + /// Returns `Some((, ))` if it was present in the database, + /// or the error that occurred during lookup or object retrieval. + fn try_find_cached<'a>( + &self, + id: &gix_hash::oid, + buffer: &'a mut Vec, + pack_cache: &mut dyn crate::cache::DecodeEntry, + ) -> Result, Option)>, gix_object::find::Error>; + + /// Find the packs location where an object with `id` can be found in the database, or `None` if there is no pack + /// holding the object. + /// + /// _Note_ that this is always None if the object isn't packed even though it exists as loose object. + fn location_by_oid(&self, id: &gix_hash::oid, buf: &mut Vec) -> Option; + + /// Obtain a vector of all offsets, in index order, along with their object id. + fn pack_offsets_and_oid(&self, pack_id: u32) -> Option>; + + /// Return the [`find::Entry`] for `location` if it is backed by a pack. + /// + /// Note that this is only in the interest of avoiding duplicate work during pack generation. + /// Pack locations can be obtained from [`Find::try_find()`]. + /// + /// # Notes + /// + /// Custom implementations might be interested in providing their own meta-data with `object`, + /// which currently isn't possible as the `Locate` trait requires GATs to work like that. + fn entry_by_location(&self, location: &data::entry::Location) -> Option; +} + +mod ext { + use gix_object::{BlobRef, CommitRef, CommitRefIter, Kind, ObjectRef, TagRef, TagRefIter, TreeRef, TreeRefIter}; + + macro_rules! make_obj_lookup { + ($method:ident, $object_variant:path, $object_kind:path, $object_type:ty) => { + /// Like [`find(…)`][Self::find()], but flattens the `Result>` into a single `Result` making a non-existing object an error + /// while returning the desired object type. + fn $method<'a>( + &self, + id: &gix_hash::oid, + buffer: &'a mut Vec, + ) -> Result<($object_type, Option), gix_object::find::existing_object::Error> + { + let id = id.as_ref(); + self.try_find(id, buffer) + .map_err(gix_object::find::existing_object::Error::Find)? + .ok_or_else(|| gix_object::find::existing_object::Error::NotFound { + oid: id.as_ref().to_owned(), + }) + .and_then(|(o, l)| { + o.decode() + .map_err(|err| gix_object::find::existing_object::Error::Decode { + source: err, + oid: id.to_owned(), + }) + .map(|o| (o, l)) + }) + .and_then(|(o, l)| match o { + $object_variant(o) => return Ok((o, l)), + o => Err(gix_object::find::existing_object::Error::ObjectKind { + oid: id.to_owned(), + actual: o.kind(), + expected: $object_kind, + }), + }) + } + }; + } + + macro_rules! make_iter_lookup { + ($method:ident, $object_kind:path, $object_type:ty, $into_iter:tt) => { + /// Like [`find(…)`][Self::find()], but flattens the `Result>` into a single `Result` making a non-existing object an error + /// while returning the desired iterator type. + fn $method<'a>( + &self, + id: &gix_hash::oid, + buffer: &'a mut Vec, + ) -> Result<($object_type, Option), gix_object::find::existing_iter::Error> { + let id = id.as_ref(); + self.try_find(id, buffer) + .map_err(gix_object::find::existing_iter::Error::Find)? + .ok_or_else(|| gix_object::find::existing_iter::Error::NotFound { + oid: id.as_ref().to_owned(), + }) + .and_then(|(o, l)| { + o.$into_iter() + .ok_or_else(|| gix_object::find::existing_iter::Error::ObjectKind { + oid: id.to_owned(), + actual: o.kind, + expected: $object_kind, + }) + .map(|i| (i, l)) + }) + } + }; + } + + /// An extension trait with convenience functions. + pub trait FindExt: super::Find { + /// Like [`try_find(…)`][super::Find::try_find()], but flattens the `Result>` into a single `Result` making a non-existing object an error. + fn find<'a>( + &self, + id: &gix_hash::oid, + buffer: &'a mut Vec, + ) -> Result<(gix_object::Data<'a>, Option), gix_object::find::existing::Error> + { + self.try_find(id, buffer) + .map_err(gix_object::find::existing::Error::Find)? + .ok_or_else(|| gix_object::find::existing::Error::NotFound { + oid: id.as_ref().to_owned(), + }) + } + + make_obj_lookup!(find_commit, ObjectRef::Commit, Kind::Commit, CommitRef<'a>); + make_obj_lookup!(find_tree, ObjectRef::Tree, Kind::Tree, TreeRef<'a>); + make_obj_lookup!(find_tag, ObjectRef::Tag, Kind::Tag, TagRef<'a>); + make_obj_lookup!(find_blob, ObjectRef::Blob, Kind::Blob, BlobRef<'a>); + make_iter_lookup!(find_commit_iter, Kind::Blob, CommitRefIter<'a>, try_into_commit_iter); + make_iter_lookup!(find_tree_iter, Kind::Tree, TreeRefIter<'a>, try_into_tree_iter); + make_iter_lookup!(find_tag_iter, Kind::Tag, TagRefIter<'a>, try_into_tag_iter); + } + + impl FindExt for T {} +} +pub use ext::FindExt; + +mod find_impls { + use std::{ops::Deref, rc::Rc}; + + use gix_hash::oid; + + use crate::{data, find}; + + impl crate::Find for &T + where + T: crate::Find, + { + fn contains(&self, id: &oid) -> bool { + (*self).contains(id) + } + + fn try_find_cached<'a>( + &self, + id: &oid, + buffer: &'a mut Vec, + pack_cache: &mut dyn crate::cache::DecodeEntry, + ) -> Result, Option)>, gix_object::find::Error> { + (*self).try_find_cached(id, buffer, pack_cache) + } + + fn location_by_oid(&self, id: &oid, buf: &mut Vec) -> Option { + (*self).location_by_oid(id, buf) + } + + fn pack_offsets_and_oid(&self, pack_id: u32) -> Option> { + (*self).pack_offsets_and_oid(pack_id) + } + + fn entry_by_location(&self, location: &data::entry::Location) -> Option { + (*self).entry_by_location(location) + } + } + + impl super::Find for std::sync::Arc + where + T: super::Find, + { + fn contains(&self, id: &oid) -> bool { + self.deref().contains(id) + } + + fn try_find_cached<'a>( + &self, + id: &oid, + buffer: &'a mut Vec, + pack_cache: &mut dyn crate::cache::DecodeEntry, + ) -> Result, Option)>, gix_object::find::Error> { + self.deref().try_find_cached(id, buffer, pack_cache) + } + + fn location_by_oid(&self, id: &oid, buf: &mut Vec) -> Option { + self.deref().location_by_oid(id, buf) + } + + fn pack_offsets_and_oid(&self, pack_id: u32) -> Option> { + self.deref().pack_offsets_and_oid(pack_id) + } + + fn entry_by_location(&self, object: &data::entry::Location) -> Option { + self.deref().entry_by_location(object) + } + } + + impl super::Find for Rc + where + T: super::Find, + { + fn contains(&self, id: &oid) -> bool { + self.deref().contains(id) + } + + fn try_find_cached<'a>( + &self, + id: &oid, + buffer: &'a mut Vec, + pack_cache: &mut dyn crate::cache::DecodeEntry, + ) -> Result, Option)>, gix_object::find::Error> { + self.deref().try_find_cached(id, buffer, pack_cache) + } + + fn location_by_oid(&self, id: &oid, buf: &mut Vec) -> Option { + self.deref().location_by_oid(id, buf) + } + + fn pack_offsets_and_oid(&self, pack_id: u32) -> Option> { + self.deref().pack_offsets_and_oid(pack_id) + } + + fn entry_by_location(&self, location: &data::entry::Location) -> Option { + self.deref().entry_by_location(location) + } + } + + impl super::Find for Box + where + T: super::Find, + { + fn contains(&self, id: &oid) -> bool { + self.deref().contains(id) + } + + fn try_find_cached<'a>( + &self, + id: &oid, + buffer: &'a mut Vec, + pack_cache: &mut dyn crate::cache::DecodeEntry, + ) -> Result, Option)>, gix_object::find::Error> { + self.deref().try_find_cached(id, buffer, pack_cache) + } + + fn location_by_oid(&self, id: &oid, buf: &mut Vec) -> Option { + self.deref().location_by_oid(id, buf) + } + + fn pack_offsets_and_oid(&self, pack_id: u32) -> Option> { + self.deref().pack_offsets_and_oid(pack_id) + } + + fn entry_by_location(&self, location: &data::entry::Location) -> Option { + self.deref().entry_by_location(location) + } + } +} diff --git a/knot2/third_party/gix-pack/src/lib.rs b/knot2/third_party/gix-pack/src/lib.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/lib.rs @@ -0,0 +1,121 @@ +//! Git stores all of its data as _Objects_, which are data along with a hash over all data. Storing objects efficiently +//! is what git packs are concerned about. +//! +//! Packs consist of [data files][data::File] and [index files][index::File]. The latter can be generated from a data file +//! and make accessing objects within a pack feasible. +//! +//! A [Bundle] conveniently combines a data pack alongside its index to allow [finding][Find] objects or verifying the pack. +//! Objects returned by `.find(…)` are [objects][gix_object::Data] which know their pack location in order to speed up +//! various common operations like creating new packs from existing ones. +//! +//! When traversing all objects in a pack, a _delta tree acceleration structure_ can be built from pack data or an index +//! in order to decompress packs in parallel and without any waste. +//! ## Feature Flags +#![cfg_attr( + all(doc, feature = "document-features"), + doc = ::document_features::document_features!() +)] +#![cfg_attr(all(doc, feature = "document-features"), feature(doc_cfg))] +#![deny(unsafe_code)] + +use std::{borrow::Cow, ops::Deref, path::Path}; + +/// The default in-memory backing store for index and multi-index files. +#[allow(missing_docs)] +pub struct MMap(Vec); + +impl MMap { + #[allow(missing_docs)] + pub fn map(file: &std::fs::File) -> std::io::Result { + use std::os::unix::fs::FileExt; + let len = usize::try_from(file.metadata()?.len()) + .map_err(|_| std::io::Error::other("file too large to load into memory"))?; + let mut bytes = vec![0u8; len]; + file.read_exact_at(&mut bytes, 0)?; + Ok(MMap(bytes)) + } +} + +impl Deref for MMap { + type Target = [u8]; + + fn deref(&self) -> &[u8] { + &self.0 + } +} + +/// A byte-oriented backing store for pack data and indices. +pub trait FileData: Deref {} + +impl FileData for T where T: Deref {} + +/// +pub mod bundle; +/// A bundle of pack data and the corresponding pack index +pub struct Bundle { + /// The pack file corresponding to `index` + pub pack: data::File, + /// The index file corresponding to `pack` + pub index: index::File, +} + +/// +pub mod find; + +/// +pub mod cache; +/// +pub mod data; + +mod find_traits; +pub use find_traits::{Find, FindExt}; + +/// +pub mod index; +/// +pub mod multi_index; + +/// +pub mod verify; + +mod mmap { + use std::path::Path; + + pub fn read_only(path: &Path) -> std::io::Result { + Ok(super::MMap(std::fs::read(path)?)) + } +} + +/// Return a display-friendly name for pack- or index-related progress messages. +/// +/// Prefer the file name, but fall back to the full path for paths without a terminal component. +fn source_name(path: &Path) -> Cow<'_, str> { + if path.as_os_str().is_empty() { + Cow::Borrowed("") + } else if let Some(name) = path.file_name() { + name.to_string_lossy() + } else { + path.as_os_str().to_string_lossy() + } +} + +#[inline] +fn read_u32(b: &[u8]) -> u32 { + u32::from_be_bytes(b.try_into().unwrap()) +} + +#[inline] +fn read_u64(b: &[u8]) -> u64 { + u64::from_be_bytes(b.try_into().unwrap()) +} + +fn exact_vec(capacity: usize) -> Vec { + let mut v = Vec::new(); + v.reserve_exact(capacity); + v +} + +#[inline] +fn fan_is_monotonically_increasing(fan: &[u32]) -> bool { + !fan.windows(2).any(|window| window[0] > window[1]) +} diff --git a/knot2/third_party/gix-pack/src/verify.rs b/knot2/third_party/gix-pack/src/verify.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/verify.rs @@ -0,0 +1,65 @@ +use std::{path::Path, sync::atomic::AtomicBool}; + +use gix_features::progress::Progress; + +/// +pub mod checksum { + /// Returned by various methods to verify the checksum of a memory mapped file that might also exist on disk. + #[derive(thiserror::Error, Debug)] + #[allow(missing_docs)] + pub enum Error { + #[error("Interrupted by user")] + Interrupted, + #[error("Failed to hash data")] + Hasher(#[from] gix_hash::hasher::Error), + #[error(transparent)] + Verify(#[from] gix_hash::verify::Error), + #[error("Failed to read pack data for checksum verification")] + Io(#[from] std::io::Error), + } +} + +/// Returns the `index` at which the following `index + 1` value is not an increment over the value at `index`. +pub fn fan(data: &[u32]) -> Option { + data.windows(2) + .enumerate() + .find_map(|(win_index, v)| (v[0] > v[1]).then_some(win_index)) +} + +/// Calculate the hash of the given kind by trying to read the file from disk at `data_path` or falling back on the mapped content in `data`. +/// `Ok(expected)` or [`checksum::Error::Verify`] is returned if the hash matches or mismatches. +/// If the [`checksum::Error::Interrupted`] is returned, the operation was interrupted. +pub fn checksum_on_disk_or_mmap( + data_path: &Path, + data: &[u8], + expected: gix_hash::ObjectId, + object_hash: gix_hash::Kind, + progress: &mut dyn Progress, + should_interrupt: &AtomicBool, +) -> Result { + let data_len_without_trailer = data.len() - object_hash.len_in_bytes(); + let actual = match gix_hash::bytes_of_file( + data_path, + data_len_without_trailer as u64, + object_hash, + progress, + should_interrupt, + ) { + Ok(id) => id, + Err(gix_hash::io::Error::Io(err)) if err.kind() == std::io::ErrorKind::Interrupted => { + return Err(checksum::Error::Interrupted); + } + Err(gix_hash::io::Error::Io(_io_err)) => { + let start = std::time::Instant::now(); + let mut hasher = gix_hash::hasher(object_hash); + hasher.update(&data[..data_len_without_trailer]); + progress.inc_by(data_len_without_trailer); + progress.show_throughput(start); + hasher.try_finalize()? + } + Err(gix_hash::io::Error::Hasher(err)) => return Err(checksum::Error::Hasher(err)), + }; + + actual.verify(&expected)?; + Ok(actual) +} diff --git a/knot2/crates/knot-atproto/fuzz/fuzz_targets/did_document.rs b/knot2/crates/knot-atproto/fuzz/fuzz_targets/did_document.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-atproto/fuzz/fuzz_targets/did_document.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + knot_atproto::fuzz::did_document(data); +}); diff --git a/knot2/crates/knot-atproto/fuzz/fuzz_targets/pubkey.rs b/knot2/crates/knot-atproto/fuzz/fuzz_targets/pubkey.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-atproto/fuzz/fuzz_targets/pubkey.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + knot_atproto::fuzz::pubkey(data); +}); diff --git a/knot2/crates/knot-cobs/fuzz/fuzz_targets/cob_change.rs b/knot2/crates/knot-cobs/fuzz/fuzz_targets/cob_change.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-cobs/fuzz/fuzz_targets/cob_change.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + knot_cobs::fuzz::change_decode(data); +}); diff --git a/knot2/crates/knot-cobs/fuzz/fuzz_targets/cob_ref.rs b/knot2/crates/knot-cobs/fuzz/fuzz_targets/cob_ref.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-cobs/fuzz/fuzz_targets/cob_ref.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + knot_cobs::fuzz::ref_parse(data); +}); diff --git a/knot2/crates/knot-cobs/tests/common/mod.rs b/knot2/crates/knot-cobs/tests/common/mod.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-cobs/tests/common/mod.rs @@ -0,0 +1,213 @@ +#![allow(dead_code)] + +use knot_cob::{ChangePayload, CobError, CobHome, CobId, CobStore}; +use knot_cobs::{Grant, Members, MembersChange, MembersCob, Registration, RegistryChange, Rename}; +use knot_git::{Layout, RefUpdate, Repo}; +use knot_runtime::{K256Signer, SeededEntropy, Signer}; +use knot_types::{ + AccountDid, ActorId, Oid, OwnerDid, RefName, RepoDid, RepoName, RepoRkey, TypeName, UnixSeconds, +}; +use tempfile::TempDir; + +pub fn fixture() -> (TempDir, Repo) { + let dir = tempfile::tempdir().unwrap(); + let repo = Layout::new(dir.path()) + .create(&RepoDid::new("did:plc:squid").unwrap()) + .unwrap(); + (dir, repo) +} + +pub fn signer(seed: u64) -> K256Signer { + K256Signer::generate(&SeededEntropy::new(seed)) +} + +pub fn home() -> CobHome { + CobHome::from(&RepoDid::new("did:plc:squid").unwrap()) +} + +pub fn account(suffix: &str) -> AccountDid { + AccountDid::new(format!("did:plc:{suffix}")).unwrap() +} + +pub fn grant(subject: &str, added_by: &str, at: i64) -> Grant { + Grant { + subject: account(subject), + added_by: account(added_by), + created_at: UnixSeconds::new(at), + } +} + +pub fn at(seconds: i64) -> UnixSeconds { + UnixSeconds::new(seconds) +} + +pub fn owner_of(seed: u64) -> ActorId { + ActorId::from_secp256k1(signer(seed).public_key().as_bytes()) +} + +pub fn rkey(value: &str) -> RepoRkey { + RepoRkey::new(value).unwrap() +} + +pub fn did(suffix: &str) -> T +where + T::Err: std::fmt::Debug, +{ + format!("did:plc:{suffix}").parse().unwrap() +} + +pub fn registration(owner: &str, key: &str, repo_id: &str, ts: i64) -> Registration { + Registration { + owner: OwnerDid::new(format!("did:plc:{owner}")).unwrap(), + rkey: rkey(key), + name: RepoName::new(key).unwrap(), + repo: RepoDid::new(format!("did:plc:{repo_id}")).unwrap(), + created_at: at(ts), + } +} + +pub fn rename(owner_id: &str, key: &str, repo_id: &str) -> Rename { + Rename { + owner: did::(owner_id), + rkey: rkey(key), + name: RepoName::new(key).unwrap(), + repo: did::(repo_id), + } +} + +pub fn reopen(repo: Repo) -> Repo { + let path = repo.path().to_path_buf(); + drop(repo); + Repo::open(path).unwrap() +} + +pub fn members_store( + seed: u64, + steps: &[(MembersChange, i64)], +) -> (TempDir, Repo, K256Signer, CobId) { + let (dir, repo) = fixture(); + let key = signer(seed); + let store = CobStore::new(&repo); + let (first, first_at) = &steps[0]; + let created = store.create(&home(), first, &key, at(*first_at)).unwrap(); + steps[1..].iter().for_each(|(change, ts)| { + store + .update(&home(), created.object, change, &key, at(*ts)) + .unwrap(); + }); + (dir, repo, key, created.object) +} + +pub fn build_members(seed: u64, steps: &[(MembersChange, i64)]) -> Members { + let (_dir, repo, _key, object) = members_store(seed, steps); + CobStore::new(&repo) + .get::(object) + .unwrap() + .into_state() +} + +pub fn write_cob_commit( + repo: &Repo, + type_name: &TypeName, + payload: &[u8], + parents: &[Oid], + author: &ActorId, + timestamp: i64, +) -> Oid { + let git = repo.git(); + let payload_oid = git.write_blob(payload).unwrap().detach(); + let tree = gix::objs::Tree { + entries: vec![gix::objs::tree::Entry { + mode: gix::objs::tree::EntryKind::Blob.into(), + filename: "payload".into(), + oid: payload_oid, + }], + }; + let revision = git.write_object(tree).unwrap().detach(); + let identity = gix::actor::Signature { + name: "knot".into(), + email: "noreply@knot".into(), + time: gix::date::Time::new(timestamp, 0), + }; + let commit = gix::objs::Commit { + tree: revision, + parents: parents.iter().map(|oid| oid.object_id()).collect(), + author: identity.clone(), + committer: identity, + encoding: None, + message: Vec::new().into(), + extra_headers: vec![ + ("cob-type".into(), type_name.as_str().into()), + ("cob-author".into(), author.as_str().into()), + ("cob-sig".into(), "00".into()), + ], + }; + Oid::from(git.write_object(commit).unwrap().detach()) +} + +pub fn cob_ref(type_name: &TypeName, object: CobId) -> RefName { + RefName::new(format!( + "refs/cobs/{}/{}", + type_name.as_str(), + object.oid().to_hex() + )) + .unwrap() +} + +pub fn forked_members_object( + seed: u64, + root: (MembersChange, i64), + left: (MembersChange, i64), + right: (MembersChange, i64), + merge: (MembersChange, i64), +) -> (TempDir, Repo, CobId) { + let (dir, repo) = fixture(); + let nsid = MembersChange::type_name(); + let author = ActorId::from_secp256k1(signer(seed).public_key().as_bytes()); + let enc = |c: &MembersChange| c.encode().unwrap(); + + let root_oid = write_cob_commit(&repo, &nsid, &enc(&root.0), &[], &author, root.1); + let object = CobId::new(root_oid); + let left_oid = write_cob_commit(&repo, &nsid, &enc(&left.0), &[root_oid], &author, left.1); + let right_oid = write_cob_commit(&repo, &nsid, &enc(&right.0), &[root_oid], &author, right.1); + let merge_oid = write_cob_commit( + &repo, + &nsid, + &enc(&merge.0), + &[left_oid, right_oid], + &author, + merge.1, + ); + repo.update_ref(&RefUpdate::Create { + name: cob_ref(&nsid, object), + new: merge_oid, + }) + .unwrap(); + (dir, repo, object) +} + +pub fn forked_members( + seed: u64, + root: (MembersChange, i64), + left: (MembersChange, i64), + right: (MembersChange, i64), + merge: (MembersChange, i64), +) -> Result { + let (_dir, repo, object) = forked_members_object(seed, root, left, right, merge); + CobStore::new(&repo) + .get::(object) + .map(|object| object.into_state()) +} + +pub fn registry_with(repo: &Repo, key: &K256Signer, name: &str, repo_id: &str) -> CobId { + let store = CobStore::new(repo); + store + .create( + &home(), + &RegistryChange::Register(registration("nel", name, repo_id, 1)), + key, + at(1), + ) + .unwrap() + .object +} diff --git a/knot2/crates/knot-edge/fuzz/fuzz_targets/spki.rs b/knot2/crates/knot-edge/fuzz/fuzz_targets/spki.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-edge/fuzz/fuzz_targets/spki.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + knot_edge::fuzz::spki_of_certificate(data); +}); diff --git a/knot2/crates/knot-edge/fuzz/fuzz_targets/spki_pin.rs b/knot2/crates/knot-edge/fuzz/fuzz_targets/spki_pin.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-edge/fuzz/fuzz_targets/spki_pin.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + knot_edge::fuzz::spki_pin(data); +}); diff --git a/knot2/crates/knot-git/fuzz/fuzz_targets/patch.rs b/knot2/crates/knot-git/fuzz/fuzz_targets/patch.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-git/fuzz/fuzz_targets/patch.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + knot_git::fuzz::patch(data); +}); diff --git a/knot2/crates/knot-git/src/bitmap/bitset.rs b/knot2/crates/knot-git/src/bitmap/bitset.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-git/src/bitmap/bitset.rs @@ -0,0 +1,113 @@ +use super::BitPosition; +use crate::error::GitError; + +#[derive(Clone)] +pub(crate) struct Bitset { + words: Vec, +} + +impl Bitset { + pub(crate) fn zeros(num_bits: usize) -> Self { + Self { + words: vec![0u64; num_bits.div_ceil(64)], + } + } + + pub(crate) fn from_ewah( + vector: &gix_bitmap::ewah::Vec, + num_bits: usize, + ) -> Result { + if vector.num_bits() > num_bits { + return Err(GitError::Backend( + "bitmap entry is wider than the object count".to_string(), + )); + } + let mut bits = Self::zeros(num_bits); + let complete = vector.for_each_set_bit(|index| { + (index < num_bits).then(|| bits.set(BitPosition::new(index as u32))) + }); + match complete { + Some(()) => Ok(bits), + None => Err(GitError::Backend("malformed ewah bitmap".to_string())), + } + } + + pub(crate) fn set(&mut self, index: BitPosition) { + let index = index.get() as usize; + self.words[index / 64] |= 1u64 << (index % 64); + } + + pub(crate) fn union_with(&mut self, other: &Bitset) { + self.words + .iter_mut() + .zip(&other.words) + .for_each(|(slot, bits)| *slot |= *bits); + } + + pub(crate) fn difference_indices<'a>( + &'a self, + other: &'a Bitset, + ) -> impl Iterator + 'a { + self.words + .iter() + .zip(&other.words) + .enumerate() + .flat_map(|(word, (present, absent))| WordBits { + remaining: present & !absent, + base: (word as u32) * 64, + }) + } +} + +struct WordBits { + remaining: u64, + base: u32, +} + +impl Iterator for WordBits { + type Item = BitPosition; + + fn next(&mut self) -> Option { + (self.remaining != 0).then(|| { + let offset = self.remaining.trailing_zeros(); + self.remaining &= self.remaining - 1; + BitPosition::new(self.base + offset) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn set_bits(bits: &Bitset, other: &Bitset) -> Vec { + bits.difference_indices(other) + .map(BitPosition::get) + .collect() + } + + #[test] + fn difference_yields_ascending_set_minus_set() { + let mut want = Bitset::zeros(130); + [1u32, 64, 65, 129] + .into_iter() + .for_each(|bit| want.set(BitPosition::new(bit))); + let mut have = Bitset::zeros(130); + [64u32, 129] + .into_iter() + .for_each(|bit| have.set(BitPosition::new(bit))); + assert_eq!(set_bits(&want, &have), vec![1, 65]); + } + + #[test] + fn union_accumulates_both_operands() { + let mut acc = Bitset::zeros(70); + let mut other = Bitset::zeros(70); + acc.set(BitPosition::new(3)); + other.set(BitPosition::new(3)); + other.set(BitPosition::new(69)); + acc.union_with(&other); + let empty = Bitset::zeros(70); + assert_eq!(set_bits(&acc, &empty), vec![3, 69]); + } +} diff --git a/knot2/crates/knot-git/src/bitmap/midx.rs b/knot2/crates/knot-git/src/bitmap/midx.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-git/src/bitmap/midx.rs @@ -0,0 +1,110 @@ +use std::path::{Path, PathBuf}; + +use knot_types::Oid; + +use crate::objects::{Haves, Wants}; + +use super::reader; +use super::revindex::{Order, OrderTable}; +use super::writer; +use crate::error::GitError; +use crate::repo::Repo; + +// hashtag easter egg +const RIDX_SIGNATURE: u32 = 0x5249_4458; +const MIDX_ALLOC_LIMIT_BYTES: usize = 16 * 1024 * 1024; + +fn midx_path(objects_dir: &Path) -> PathBuf { + objects_dir.join("pack").join("multi-pack-index") +} + +fn sidecar(objects_dir: &Path, checksum: &gix_hash::ObjectId, ext: &str) -> PathBuf { + objects_dir + .join("pack") + .join(format!("multi-pack-index-{}.{ext}", checksum.to_hex())) +} + +pub(super) fn write(repo: &Repo) -> Result { + let kind = repo.object_format().kind(); + let objects_dir = repo.objects_dir(); + let file = match gix_pack::multi_index::File::at( + midx_path(&objects_dir), + Some(MIDX_ALLOC_LIMIT_BYTES), + ) { + Ok(file) => file, + Err(_) => return Ok(false), + }; + let order = OrderTable::from_file(&file); + if order.len() == 0 { + return Ok(false); + } + + let _boost = knot_resource::saturate(); + let types = writer::type_index_bits(repo, &order)?; + let selected = writer::selected_entries(repo, &order)?; + if selected.is_empty() { + return Ok(false); + } + + let checksum = file.checksum(); + let bytes = writer::assemble(kind, &checksum, &types, &selected)?; + + write_rev( + &sidecar(&objects_dir, &checksum, "rev"), + &order, + kind, + &checksum, + )?; + writer::install(&sidecar(&objects_dir, &checksum, "bitmap"), &bytes)?; + Ok(true) +} + +fn write_rev( + path: &Path, + order: &OrderTable, + kind: gix::hash::Kind, + checksum: &gix_hash::ObjectId, +) -> Result<(), GitError> { + let hash_id: u32 = match kind { + gix::hash::Kind::Sha256 => 2, + _ => 1, + }; + let mut out = Vec::new(); + out.extend_from_slice(&RIDX_SIGNATURE.to_be_bytes()); + out.extend_from_slice(&1u32.to_be_bytes()); + out.extend_from_slice(&hash_id.to_be_bytes()); + order + .index_positions_in_bit_order() + .iter() + .for_each(|position| out.extend_from_slice(&position.get().to_be_bytes())); + out.extend_from_slice(checksum.as_slice()); + let mut hasher = gix_hash::hasher(kind); + hasher.update(&out); + let digest = hasher + .try_finalize() + .map_err(|error| GitError::Backend(format!("midx revindex checksum: {error}")))?; + out.extend_from_slice(digest.as_slice()); + writer::install(path, &out) +} + +pub(super) fn reachable( + repo: &Repo, + wants: Wants<'_>, + haves: Haves<'_>, +) -> Result>, GitError> { + let kind = repo.object_format().kind(); + let objects_dir = repo.objects_dir(); + let file = match gix_pack::multi_index::File::at( + midx_path(&objects_dir), + Some(MIDX_ALLOC_LIMIT_BYTES), + ) { + Ok(file) => file, + Err(_) => return Ok(None), + }; + let Ok(bytes) = std::fs::read(sidecar(&objects_dir, &file.checksum(), "bitmap")) else { + return Ok(None); + }; + let order = OrderTable::from_file(&file); + let maps = reader::parse(&bytes, kind, order.len())?; + super::resolve(repo, &order, &maps, wants, haves) +} diff --git a/knot2/crates/knot-git/src/bitmap/mod.rs b/knot2/crates/knot-git/src/bitmap/mod.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-git/src/bitmap/mod.rs @@ -0,0 +1,170 @@ +use std::path::{Path, PathBuf}; + +use knot_types::{ObjectCount, Oid}; + +use crate::error::GitError; +use crate::objects::{Haves, Wants}; +use crate::repo::Repo; + +mod bitset; +mod midx; +mod reader; +mod revindex; +mod writer; + +use bitset::Bitset; +use revindex::{Order, OrderTable}; + +knot_types::scalar_newtype! { + pub(crate) struct BitPosition(u32); + pub(crate) struct IndexPosition(u32) => ordered; + pub(crate) struct BitmapEntryOffset(u64); +} + +pub fn write_bitmap(repo: &Repo, pack_idx: &Path) -> Result { + writer::write(repo, pack_idx) +} + +pub fn write_midx_bitmap(repo: &Repo) -> Result { + midx::write(repo) +} + +pub fn reachable_via_bitmap( + repo: &Repo, + wants: Wants<'_>, + haves: Haves<'_>, +) -> Result>, GitError> { + if let Some((_, found)) = single_pack(repo, wants, haves)? { + return Ok(Some(found)); + } + midx::reachable(repo, wants, haves) +} + +fn single_pack( + repo: &Repo, + wants: Wants<'_>, + haves: Haves<'_>, +) -> Result)>, GitError> { + let kind = repo.object_format().kind(); + let Some(pack_idx) = bitmapped_pack(&repo.objects_dir()) else { + return Ok(None); + }; + let Ok(bytes) = std::fs::read(pack_idx.with_extension("bitmap")) else { + return Ok(None); + }; + let index = gix_pack::index::File::at(&pack_idx, kind) + .map_err(|error| GitError::Backend(format!("open pack index: {error}")))?; + let rev = OrderTable::from_index(&index); + let count = rev.len(); + let maps = reader::parse(&bytes, kind, count)?; + Ok(resolve(repo, &rev, &maps, wants, haves)? + .map(|reachable| (ObjectCount::new(count), reachable))) +} + +pub(super) fn resolve( + repo: &Repo, + rev: &impl Order, + maps: &reader::Bitmaps<'_>, + wants: Wants<'_>, + haves: Haves<'_>, +) -> Result>, GitError> { + let (Some(want), Some(have)) = ( + accumulate(repo, rev, maps, wants.as_slice())?, + accumulate(repo, rev, maps, haves.as_slice())?, + ) else { + return Ok(None); + }; + let reachable = want + .difference_indices(&have) + .map(|bit| rev.oid_at_bit(bit)) + .collect(); + Ok(Some(reachable)) +} + +pub fn verbatim_clone_pack( + repo: &Repo, + wants: Wants<'_>, +) -> Result, GitError> { + let Some(pack_idx) = bitmapped_pack(&repo.objects_dir()) else { + return Ok(None); + }; + let Some((num_objects, reachable)) = single_pack(repo, wants, Haves::new(&[]))? else { + return Ok(None); + }; + if ObjectCount::new(reachable.len()) != num_objects { + return Ok(None); + } + Ok(std::fs::File::open(pack_idx.with_extension("pack")).ok()) +} + +fn bitmapped_pack(objects_dir: &Path) -> Option { + let pack_dir = objects_dir.join("pack"); + let mut idxs: Vec = std::fs::read_dir(&pack_dir) + .ok()? + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| path.extension().is_some_and(|ext| ext == "idx")) + .collect(); + idxs.sort(); + idxs.into_iter() + .find(|idx| idx.with_extension("bitmap").exists()) +} + +fn accumulate( + repo: &Repo, + rev: &impl Order, + maps: &reader::Bitmaps<'_>, + oids: &[Oid], +) -> Result, GitError> { + oids.iter() + .try_fold(Some(Bitset::zeros(rev.len())), |acc, oid| { + let Some(mut acc) = acc else { + return Ok(None); + }; + match contribution(repo, rev, maps, *oid)? { + Some(part) => { + acc.union_with(&part); + Ok(Some(acc)) + } + None => Ok(None), + } + }) +} + +fn contribution( + repo: &Repo, + rev: &impl Order, + maps: &reader::Bitmaps<'_>, + oid: Oid, +) -> Result, GitError> { + let Some((commit, tags)) = peel_commit_chain(repo, oid) else { + return Ok(None); + }; + let Some(position) = rev.index_of(commit) else { + return Ok(None); + }; + let Some(base) = maps.bitmap(position)? else { + return Ok(None); + }; + let bits = tags.iter().try_fold(base, |mut bits, tag| { + let position = rev.index_of(*tag)?; + bits.set(rev.bit_at_index(position)); + Some(bits) + }); + Ok(bits) +} + +fn peel_commit_chain(repo: &Repo, oid: Oid) -> Option<(Oid, Vec)> { + let object = repo.git().find_object(oid.object_id()).ok()?; + match object.kind { + gix::object::Kind::Commit => Some((oid, Vec::new())), + gix::object::Kind::Tag => { + let target = object.try_into_tag().ok()?.target_id().ok()?.detach(); + peel_commit_chain(repo, Oid::from(target)).map(|(commit, mut tags)| { + tags.push(oid); + (commit, tags) + }) + } + _ => None, + } +} diff --git a/knot2/crates/knot-git/src/bitmap/reader.rs b/knot2/crates/knot-git/src/bitmap/reader.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-git/src/bitmap/reader.rs @@ -0,0 +1,114 @@ +use super::bitset::Bitset; +use super::{BitmapEntryOffset, IndexPosition}; +use crate::error::GitError; + +const OPT_LOOKUP_TABLE: u16 = 0x10; +const TRIPLET_LEN: usize = 16; + +pub(crate) struct Bitmaps<'a> { + body: &'a [u8], + table: Vec<(IndexPosition, BitmapEntryOffset)>, + num_objects: usize, +} + +impl Bitmaps<'_> { + pub(crate) fn bitmap(&self, position: IndexPosition) -> Result, GitError> { + match self.table.binary_search_by_key(&position, |(pos, _)| *pos) { + Ok(index) => self.decode_at(position, self.table[index].1).map(Some), + Err(_) => Ok(None), + } + } + + fn decode_at( + &self, + commit_pos: IndexPosition, + offset: BitmapEntryOffset, + ) -> Result { + let start = usize::try_from(offset.get()) + .ok() + .filter(|start| *start <= self.body.len()) + .ok_or_else(|| GitError::Backend("bitmap entry offset out of range".to_string()))?; + let entry = &self.body[start..]; + if entry.len() < 6 { + return Err(GitError::Backend( + "truncated bitmap entry header".to_string(), + )); + } + if u32::from_be_bytes([entry[0], entry[1], entry[2], entry[3]]) != commit_pos.get() { + return Err(GitError::Backend( + "bitmap lookup points at the wrong commit".to_string(), + )); + } + if entry[4] != 0 { + return Err(GitError::Backend( + "xor-compressed bitmap entries are unsupported".to_string(), + )); + } + let (vector, _) = decode(&entry[6..])?; + Bitset::from_ewah(&vector, self.num_objects) + } +} + +pub(crate) fn parse( + bytes: &[u8], + kind: gix::hash::Kind, + num_objects: usize, +) -> Result, GitError> { + let raw = kind.len_in_bytes(); + let header_len = 12 + raw; + if bytes.len() < header_len + raw || &bytes[..4] != b"BITM" { + return Err(GitError::Backend("bitmap header isn't BITM".to_string())); + } + let version = u16::from_be_bytes([bytes[4], bytes[5]]); + if version != 1 { + return Err(GitError::Backend(format!( + "unsupported bitmap version {version}" + ))); + } + let flags = u16::from_be_bytes([bytes[6], bytes[7]]); + if flags & OPT_LOOKUP_TABLE == 0 { + return Err(GitError::Backend( + "bitmap lacks the lookup table extension".to_string(), + )); + } + let entry_count = u32::from_be_bytes([bytes[8], bytes[9], bytes[10], bytes[11]]) as usize; + + let (body, trailer) = bytes.split_at(bytes.len() - raw); + let mut hasher = gix_hash::hasher(kind); + hasher.update(body); + let digest = hasher + .try_finalize() + .map_err(|error| GitError::Backend(format!("bitmap checksum: {error}")))?; + if digest.as_slice() != trailer { + return Err(GitError::Backend("bitmap checksum mismatch".to_string())); + } + + let table_len = entry_count + .checked_mul(TRIPLET_LEN) + .filter(|len| header_len + len <= body.len()) + .ok_or_else(|| GitError::Backend("bitmap lookup table overflows the file".to_string()))?; + let region = &body[body.len() - table_len..]; + let mut table: Vec<(IndexPosition, BitmapEntryOffset)> = (0..entry_count) + .map(|index| { + let base = index * TRIPLET_LEN; + let commit_pos = u32::from_be_bytes(region[base..base + 4].try_into().unwrap()); + let offset = u64::from_be_bytes(region[base + 4..base + 12].try_into().unwrap()); + ( + IndexPosition::new(commit_pos), + BitmapEntryOffset::new(offset), + ) + }) + .collect(); + table.sort_unstable_by_key(|(commit_pos, _)| *commit_pos); + + Ok(Bitmaps { + body, + table, + num_objects, + }) +} + +fn decode(data: &[u8]) -> Result<(gix_bitmap::ewah::Vec, &[u8]), GitError> { + gix_bitmap::ewah::decode(data) + .map_err(|error| GitError::Backend(format!("ewah decode: {error:?}"))) +} diff --git a/knot2/crates/knot-git/src/bitmap/revindex.rs b/knot2/crates/knot-git/src/bitmap/revindex.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-git/src/bitmap/revindex.rs @@ -0,0 +1,93 @@ +use std::collections::HashMap; + +use knot_types::Oid; + +use super::{BitPosition, IndexPosition}; + +pub(crate) trait Order { + fn len(&self) -> usize; + fn index_of(&self, oid: Oid) -> Option; + fn bit_at_index(&self, position: IndexPosition) -> BitPosition; + fn oid_at_bit(&self, bit: BitPosition) -> Oid; +} + +pub(crate) struct OrderTable { + oids: Vec, + bit_to_index: Vec, + index_to_bit: Vec, + by_oid: HashMap, +} + +impl OrderTable { + fn build(oids: Vec, placement: impl Fn(u32) -> K) -> Self { + let count = oids.len() as u32; + let mut order: Vec = (0..count).collect(); + order.sort_by_key(|position| placement(*position)); + let bit_to_index: Vec = order + .iter() + .map(|position| IndexPosition::new(*position)) + .collect(); + let index_to_bit = order.iter().enumerate().fold( + vec![BitPosition::new(0); oids.len()], + |mut table, (bit, position)| { + table[*position as usize] = BitPosition::new(bit as u32); + table + }, + ); + let by_oid: HashMap = oids + .iter() + .enumerate() + .map(|(position, oid)| (*oid, IndexPosition::new(position as u32))) + .collect(); + Self { + oids, + bit_to_index, + index_to_bit, + by_oid, + } + } + + pub(crate) fn from_index(index: &gix_pack::index::File) -> Self { + let count = index.num_objects(); + let oids: Vec = (0..count) + .map(|position| Oid::from(index.oid_at_index(position).to_owned())) + .collect(); + let offsets: Vec = (0..count) + .map(|position| index.pack_offset_at_index(position)) + .collect(); + Self::build(oids, |position| offsets[position as usize]) + } + + pub(crate) fn from_file(file: &gix_pack::multi_index::File) -> Self { + let count = file.num_objects(); + let oids: Vec = (0..count) + .map(|position| Oid::from(file.oid_at_index(position).to_owned())) + .collect(); + let placement: Vec<(u32, u64)> = (0..count) + .map(|position| file.pack_id_and_pack_offset_at_index(position)) + .collect(); + Self::build(oids, |position| placement[position as usize]) + } + + pub(crate) fn index_positions_in_bit_order(&self) -> &[IndexPosition] { + &self.bit_to_index + } +} + +impl Order for OrderTable { + fn len(&self) -> usize { + self.oids.len() + } + + fn index_of(&self, oid: Oid) -> Option { + self.by_oid.get(&oid).copied() + } + + fn bit_at_index(&self, position: IndexPosition) -> BitPosition { + self.index_to_bit[position.get() as usize] + } + + fn oid_at_bit(&self, bit: BitPosition) -> Oid { + self.oids[self.bit_to_index[bit.get() as usize].get() as usize] + } +} diff --git a/knot2/crates/knot-git/src/bitmap/writer.rs b/knot2/crates/knot-git/src/bitmap/writer.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-git/src/bitmap/writer.rs @@ -0,0 +1,277 @@ +use std::collections::HashSet; +use std::path::Path; + +use knot_types::Oid; + +use super::revindex::{Order, OrderTable}; +use super::{BitPosition, BitmapEntryOffset, IndexPosition}; +use crate::error::GitError; +use crate::objects::{Haves, Wants}; +use crate::repo::Repo; + +const OPT_FULL_DAG: u16 = 0x1; +const OPT_LOOKUP_TABLE: u16 = 0x10; + +const CAT_COMMIT: u8 = 0; +const CAT_TREE: u8 = 1; +const CAT_BLOB: u8 = 2; +const CAT_TAG: u8 = 3; + +pub(super) struct TypeBits { + commits: Vec, + trees: Vec, + blobs: Vec, + tags: Vec, +} + +pub(super) struct Selected { + commit_pos: IndexPosition, + bits: Vec, +} + +pub(crate) fn write(repo: &Repo, pack_idx: &Path) -> Result { + let kind = repo.object_format().kind(); + let index = gix_pack::index::File::at(pack_idx, kind) + .map_err(|error| GitError::Backend(format!("open pack index: {error}")))?; + let rev = OrderTable::from_index(&index); + if rev.len() == 0 { + return Ok(false); + } + + let _boost = knot_resource::saturate(); + let types = type_index_bits(repo, &rev)?; + let selected = selected_entries(repo, &rev)?; + if selected.is_empty() { + return Ok(false); + } + + let pack_path = pack_idx.with_extension("pack"); + let checksum = gix_pack::data::File::at(&pack_path, kind) + .map_err(|error| GitError::Backend(format!("open pack data: {error}")))? + .checksum(); + + let bytes = assemble(kind, &checksum, &types, &selected)?; + install(&pack_idx.with_extension("bitmap"), &bytes)?; + Ok(true) +} + +pub(super) fn type_index_bits(repo: &Repo, rev: &R) -> Result { + let categories = categories_in_bit_order(repo, rev)?; + let select = |target: u8| { + categories + .iter() + .map(|value| *value == target) + .collect::>() + }; + Ok(TypeBits { + commits: select(CAT_COMMIT), + trees: select(CAT_TREE), + blobs: select(CAT_BLOB), + tags: select(CAT_TAG), + }) +} + +fn categories_in_bit_order(repo: &Repo, rev: &R) -> Result, GitError> { + let path = repo.path().to_owned(); + knot_resource::map_spans(rev.len(), |start, end| { + let local = Repo::open(&path)?; + (start..end) + .map(|bit| object_category(&local, rev.oid_at_bit(BitPosition(bit as u32)))) + .collect::, GitError>>() + }) +} + +fn object_category(repo: &Repo, oid: Oid) -> Result { + match repo.git().try_find_header(oid.object_id()) { + Ok(Some(header)) => Ok(match header.kind() { + gix::object::Kind::Commit => CAT_COMMIT, + gix::object::Kind::Tree => CAT_TREE, + gix::object::Kind::Blob => CAT_BLOB, + gix::object::Kind::Tag => CAT_TAG, + }), + Ok(None) => Err(GitError::ObjectNotFound(oid)), + Err(error) => Err(GitError::Corrupt { + oid, + message: error.to_string(), + }), + } +} + +fn one_selected( + repo: &Repo, + rev: &R, + commit_pos: IndexPosition, + commit: Oid, +) -> Result { + let closure = repo.select_pack_objects(Wants::new(&[commit]), Haves::new(&[]))?; + Ok(Selected { + commit_pos, + bits: closure_bits(rev, &closure)?, + }) +} + +pub(super) fn selected_entries( + repo: &Repo, + rev: &R, +) -> Result, GitError> { + let mut seen: HashSet = HashSet::new(); + let commits: Vec<(IndexPosition, Oid)> = repo + .references()? + .into_iter() + .filter_map(|record| peel_to_commit(repo, record.target)) + .filter_map(|commit| rev.index_of(commit).map(|position| (position, commit))) + .filter(|(_, commit)| seen.insert(*commit)) + .collect(); + + let path = repo.path().to_owned(); + let mut entries: Vec = knot_resource::map_chunks(&commits, |batch| { + let local = Repo::open(&path)?; + batch + .iter() + .map(|(commit_pos, commit)| one_selected(&local, rev, *commit_pos, *commit)) + .collect::, GitError>>() + })?; + entries.sort_by_key(|entry| entry.commit_pos); + Ok(entries) +} + +fn peel_to_commit(repo: &Repo, oid: Oid) -> Option { + let object = repo.git().find_object(oid.object_id()).ok()?; + match object.kind { + gix::object::Kind::Commit => Some(oid), + gix::object::Kind::Tag => { + let target = object.try_into_tag().ok()?.target_id().ok()?.detach(); + peel_to_commit(repo, Oid::from(target)) + } + _ => None, + } +} + +fn closure_bits(rev: &impl Order, closure: &[Oid]) -> Result, GitError> { + closure + .iter() + .try_fold(vec![false; rev.len()], |mut bits, oid| { + let position = rev.index_of(*oid).ok_or_else(|| { + GitError::Backend(format!( + "closure object {oid} is absent from the pack bitmap" + )) + })?; + bits[rev.bit_at_index(position).get() as usize] = true; + Ok(bits) + }) +} + +pub(super) fn assemble( + kind: gix::hash::Kind, + checksum: &gix_hash::ObjectId, + types: &TypeBits, + selected: &[Selected], +) -> Result, GitError> { + let flags: u16 = OPT_FULL_DAG | OPT_LOOKUP_TABLE; + let mut out = Vec::new(); + out.extend_from_slice(b"BITM"); + out.extend_from_slice(&1u16.to_be_bytes()); + out.extend_from_slice(&flags.to_be_bytes()); + out.extend_from_slice(&(selected.len() as u32).to_be_bytes()); + out.extend_from_slice(checksum.as_slice()); + + write_ewah(&mut out, &types.commits)?; + write_ewah(&mut out, &types.trees)?; + write_ewah(&mut out, &types.blobs)?; + write_ewah(&mut out, &types.tags)?; + + let offsets: Vec<(IndexPosition, BitmapEntryOffset)> = selected + .iter() + .map(|entry| { + let at = BitmapEntryOffset::new(out.len() as u64); + out.extend_from_slice(&entry.commit_pos.get().to_be_bytes()); + out.push(0); + out.push(0); + write_ewah(&mut out, &entry.bits)?; + Ok::<_, GitError>((entry.commit_pos, at)) + }) + .collect::>()?; + + offsets.iter().for_each(|(commit_pos, offset)| { + out.extend_from_slice(&commit_pos.get().to_be_bytes()); + out.extend_from_slice(&offset.get().to_be_bytes()); + out.extend_from_slice(&0xffff_ffffu32.to_be_bytes()); + }); + + let mut hasher = gix_hash::hasher(kind); + hasher.update(&out); + let digest = hasher + .try_finalize() + .map_err(|error| GitError::Backend(format!("bitmap checksum: {error}")))?; + out.extend_from_slice(digest.as_slice()); + Ok(out) +} + +fn write_ewah(out: &mut Vec, bits: &[bool]) -> Result<(), GitError> { + let vector = gix_bitmap::ewah::Vec::from_bits(bits) + .ok_or_else(|| GitError::Backend("ewah bit count exceeds u32".to_string()))?; + vector + .write_to(out) + .map_err(|error| GitError::Backend(format!("ewah write: {error}"))) +} + +pub(super) fn install(path: &Path, bytes: &[u8]) -> Result<(), GitError> { + knot_resource::atomic_write_bytes(path, bytes, knot_resource::FileMode::Inherited).map_err( + |error| GitError::Maintenance(format!("write bitmap {}: {}", path.display(), error.source)), + ) +} + +#[cfg(test)] +mod tests { + use super::super::{BitPosition, IndexPosition}; + use super::*; + + struct FakeOrder { + oids: Vec, + } + + impl Order for FakeOrder { + fn len(&self) -> usize { + self.oids.len() + } + + fn index_of(&self, oid: Oid) -> Option { + self.oids + .iter() + .position(|candidate| *candidate == oid) + .map(|position| IndexPosition::new(position as u32)) + } + + fn bit_at_index(&self, position: IndexPosition) -> BitPosition { + BitPosition::new(position.get()) + } + + fn oid_at_bit(&self, bit: BitPosition) -> Oid { + self.oids[bit.get() as usize] + } + } + + fn oid(byte: u8) -> Oid { + Oid::from_hex(&format!("{byte:02x}").repeat(20)).unwrap() + } + + #[test] + fn closure_bits_sets_one_bit_per_present_object() { + let order = FakeOrder { + oids: vec![oid(1), oid(2), oid(3)], + }; + let bits = closure_bits(&order, &[oid(1), oid(3)]).unwrap(); + assert_eq!(bits, vec![true, false, true]); + } + + #[test] + fn closure_bits_errors_when_an_object_is_outside_the_pack() { + let order = FakeOrder { + oids: vec![oid(1), oid(2)], + }; + assert!( + closure_bits(&order, &[oid(1), oid(9)]).is_err(), + "an incomplete closure must fail the bitmap rather than write a partial one" + ); + } +} diff --git a/knot2/crates/knot-git/tests/common/mod.rs b/knot2/crates/knot-git/tests/common/mod.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-git/tests/common/mod.rs @@ -0,0 +1,18 @@ +#![allow(dead_code, unused_imports)] + +use knot_git::Layout; +use knot_types::RepoDid; + +pub use knot_fixtures::{ + available as git_available, commit as commit_file, must as git_ok, run as git, +}; + +pub fn seeded() -> (tempfile::TempDir, tempfile::TempDir, Layout, RepoDid) { + let scan = tempfile::tempdir().unwrap(); + let work = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + let did = RepoDid::new("did:plc:squid").unwrap(); + layout.create(&did).unwrap(); + git_ok(work.path(), &["init", "-q", "-b", "main"]); + (scan, work, layout, did) +} diff --git a/knot2/crates/knot-index/tests/common/mod.rs b/knot2/crates/knot-index/tests/common/mod.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-index/tests/common/mod.rs @@ -0,0 +1,179 @@ +#![allow(dead_code)] + +use std::path::PathBuf; + +use knot_cob::{CobHome, CobId, CobStore}; +use knot_cobs::{CollaboratorsChange, Grant, MembersChange, Registration, RegistryChange, Removal}; +use knot_git::{Layout, Repo}; +use knot_index::Index; +use knot_runtime::{K256Signer, SeededEntropy}; +use knot_types::{AccountDid, KnotId, OwnerDid, RepoDid, RepoName, RepoRkey, UnixSeconds}; +use tempfile::TempDir; + +pub fn acc(suffix: &str) -> AccountDid { + AccountDid::new(format!("did:plc:{suffix}")).unwrap() +} + +pub fn own(suffix: &str) -> OwnerDid { + OwnerDid::new(format!("did:plc:{suffix}")).unwrap() +} + +pub fn repo_did(suffix: &str) -> RepoDid { + RepoDid::new(format!("did:plc:{suffix}")).unwrap() +} + +pub fn rkey(value: &str) -> RepoRkey { + RepoRkey::new(value).unwrap() +} + +pub fn at(seconds: i64) -> UnixSeconds { + UnixSeconds::new(seconds) +} + +pub fn meta_home() -> CobHome { + CobHome::from(&KnotId::new("did:web:knot.nel.pet").unwrap()) +} + +pub fn grant(subject: &str, added_by: &str, seconds: i64) -> Grant { + Grant { + subject: acc(subject), + added_by: acc(added_by), + created_at: at(seconds), + } +} + +pub fn registration(owner_id: &str, key: &str, repo: &RepoDid, seconds: i64) -> Registration { + Registration { + owner: own(owner_id), + rkey: rkey(key), + name: RepoName::new(key).unwrap(), + repo: repo.clone(), + created_at: at(seconds), + } +} + +pub struct World { + _dir: TempDir, + pub meta_path: PathBuf, + pub layout: Layout, + pub signer: K256Signer, +} + +impl World { + pub fn new() -> Self { + Self::seeded(1) + } + + pub fn seeded(seed: u64) -> Self { + let dir = tempfile::tempdir().unwrap(); + let meta_path = dir.path().join("meta"); + Repo::create(&meta_path).unwrap(); + let layout = Layout::new(dir.path().join("repos")); + let signer = K256Signer::generate(&SeededEntropy::new(seed)); + Self { + _dir: dir, + meta_path, + layout, + signer, + } + } + + pub fn index(&self) -> Index { + Index::new(&self.meta_path, self.layout.clone()) + } + + pub fn seed_members(&self) -> CobId { + let meta = Repo::open(&self.meta_path).unwrap(); + let store = CobStore::new(&meta); + let created = store + .create( + &meta_home(), + &MembersChange::Add(grant("nel", "nel", 1)), + &self.signer, + at(1), + ) + .unwrap(); + store + .update( + &meta_home(), + created.object, + &MembersChange::Add(grant("olaren", "nel", 2)), + &self.signer, + at(2), + ) + .unwrap(); + created.object + } + + pub fn add_member(&self, object: CobId, subject: &str, seconds: i64) { + let meta = Repo::open(&self.meta_path).unwrap(); + let store = CobStore::new(&meta); + store + .update( + &meta_home(), + object, + &MembersChange::Add(grant(subject, "nel", seconds)), + &self.signer, + at(seconds), + ) + .unwrap(); + } + + pub fn seed_registry(&self, repo: &RepoDid) -> CobId { + let meta = Repo::open(&self.meta_path).unwrap(); + let store = CobStore::new(&meta); + store + .create( + &meta_home(), + &RegistryChange::Register(registration("nel", "anemone", repo, 1)), + &self.signer, + at(1), + ) + .unwrap() + .object + } + + pub fn register_extra(&self, repo: &RepoDid, key: &str, registry: CobId) { + let meta = Repo::open(&self.meta_path).unwrap(); + let store = CobStore::new(&meta); + store + .update( + &meta_home(), + registry, + &RegistryChange::Register(registration("nel", key, repo, 2)), + &self.signer, + at(2), + ) + .unwrap(); + } + + pub fn seed_collaborator(&self, repo: &RepoDid, subject: &str) -> CobId { + let git = self.layout.create(repo).unwrap(); + let store = CobStore::new(&git); + store + .create( + &CobHome::from(repo), + &CollaboratorsChange::Add(grant(subject, "nel", 1)), + &self.signer, + at(1), + ) + .unwrap() + .object + } + + pub fn remove_collaborator(&self, repo: &RepoDid, object: CobId, subject: &str, seconds: i64) { + let git = self.layout.open(repo).unwrap(); + let store = CobStore::new(&git); + store + .update( + &CobHome::from(repo), + object, + &CollaboratorsChange::Remove(Removal { + subject: acc(subject), + }), + &self.signer, + at(seconds), + ) + .unwrap(); + } +} diff --git a/knot2/crates/knot-lfs/fuzz/fuzz_targets/batch.rs b/knot2/crates/knot-lfs/fuzz/fuzz_targets/batch.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-lfs/fuzz/fuzz_targets/batch.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + knot_lfs::fuzz::batch(data); +}); diff --git a/knot2/crates/knot-lfs/fuzz/fuzz_targets/pointer.rs b/knot2/crates/knot-lfs/fuzz/fuzz_targets/pointer.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-lfs/fuzz/fuzz_targets/pointer.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + knot_lfs::fuzz::pointer(data); +}); diff --git a/knot2/crates/knot-lfs/fuzz/fuzz_targets/transfer.rs b/knot2/crates/knot-lfs/fuzz/fuzz_targets/transfer.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-lfs/fuzz/fuzz_targets/transfer.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + knot_lfs::fuzz::transfer(data); +}); diff --git a/knot2/crates/knot-lfs/tests/common/mod.rs b/knot2/crates/knot-lfs/tests/common/mod.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-lfs/tests/common/mod.rs @@ -0,0 +1,98 @@ +#![allow(dead_code)] + +use std::path::{Path, PathBuf}; +use std::time::{Duration, SystemTime}; + +use gix_packetline::blocking_io::encode; +use knot_lfs::{LfsOid, LfsSize}; +use knot_types::RepoDid; +use sha2::{Digest, Sha256}; + +pub const SQUID: &str = "did:plc:squid"; +pub const PKT_DATA_MAX: usize = 65516; +pub const PEAK_CEILING: u64 = 512 * 1024 * 1024; +pub const GROWTH_SLACK: u64 = 64 * 1024 * 1024; + +pub fn repo() -> RepoDid { + RepoDid::new(SQUID).unwrap() +} + +pub fn oid_of(bytes: &[u8]) -> LfsOid { + LfsOid::from_digest(Sha256::digest(bytes).into()) +} + +pub fn incompressible(len: usize, seed: u64) -> Vec { + let mut state = seed | 1; + (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + (state & 0xff) as u8 + }) + .collect() +} + +pub fn pointer_blob(oid: &LfsOid, size: LfsSize) -> Vec { + format!("version https://git-lfs.github.com/spec/v1\noid sha256:{oid}\nsize {size}\n") + .into_bytes() +} + +pub fn object_path(store_dir: &Path, oid: &LfsOid) -> PathBuf { + store_dir + .join("plc/sq/uid") + .join(&oid.as_str()[0..2]) + .join(&oid.as_str()[2..4]) + .join(oid.as_str()) +} + +pub fn backdate(path: &Path, past: Duration) { + std::fs::OpenOptions::new() + .write(true) + .open(path) + .unwrap() + .set_modified(SystemTime::now() - past) + .unwrap(); +} + +pub fn put_text(buf: &mut Vec, line: &str) { + encode::data_to_write(format!("{line}\n").as_bytes(), &mut *buf).unwrap(); +} + +pub fn upload_script(body: &[u8]) -> (LfsOid, Vec) { + let oid = oid_of(body); + let mut script = Vec::with_capacity(body.len() + 4096); + put_text(&mut script, &format!("put-object {oid}")); + put_text(&mut script, &format!("size={}", body.len())); + encode::delim_to_write(&mut script).unwrap(); + body.chunks(PKT_DATA_MAX).for_each(|chunk| { + encode::data_to_write(chunk, &mut script).unwrap(); + }); + encode::flush_to_write(&mut script).unwrap(); + put_text(&mut script, &format!("verify-object {oid}")); + put_text(&mut script, &format!("size={}", body.len())); + encode::flush_to_write(&mut script).unwrap(); + put_text(&mut script, "quit"); + encode::flush_to_write(&mut script).unwrap(); + (oid, script) +} + +pub fn download_script(oid: &LfsOid) -> Vec { + let mut script = Vec::new(); + put_text(&mut script, &format!("get-object {oid}")); + encode::flush_to_write(&mut script).unwrap(); + put_text(&mut script, "quit"); + encode::flush_to_write(&mut script).unwrap(); + script +} + +pub fn rss_bytes() -> u64 { + const PAGE_BYTES: u64 = 4096; + let statm = std::fs::read_to_string("/proc/self/statm").expect("/proc/self/statm is readable"); + statm + .split_whitespace() + .nth(1) + .and_then(|pages| pages.parse::().ok()) + .map(|pages| pages * PAGE_BYTES) + .expect("statm lists the resident page count") +} diff --git a/knot2/crates/knot-maintenance/tests/common/mod.rs b/knot2/crates/knot-maintenance/tests/common/mod.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-maintenance/tests/common/mod.rs @@ -0,0 +1,223 @@ +#![allow(dead_code, unused_imports)] + +use std::collections::BTreeSet; +use std::ops::Range; +use std::path::Path; + +use knot_git::{ + EntryKind, Identity, Layout, NewCommit, RefUpdate, Repo, StagedAction, StagedChange, +}; +use knot_maintenance::{GeometricFactor, ObjectCount, Options, PruneGrace, ReflogRetention}; +use knot_types::{AuthorName, BranchName, Email, ObjectFormat, Oid, RefName, RepoDid, UnixSeconds}; + +pub const EMPTY_TREE_SHA1: &str = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"; +pub const EMPTY_TREE_SHA256: &str = + "6ef19b41225c5369f1c104d45d8d85efa9b057b53b14b4b9b939dd74decc5321"; + +pub fn empty_tree(format: ObjectFormat) -> &'static str { + if format == ObjectFormat::SHA256 { + EMPTY_TREE_SHA256 + } else { + EMPTY_TREE_SHA1 + } +} + +pub fn now() -> UnixSeconds { + UnixSeconds::new(1_700_000_500) +} + +pub use knot_fixtures::available as git_available; + +pub fn identity() -> Identity { + Identity { + name: AuthorName::new("nel"), + email: Email::new("nel@oyster.cafe"), + time: UnixSeconds::new(1_700_000_000), + offset_seconds: 0, + } +} + +pub fn options() -> Options { + Options { + repack_max_objects: ObjectCount::new(1_000_000), + geometric_factor: GeometricFactor::full_repack(), + prune_grace: PruneGrace::from_secs(0), + reflog_floor: ReflogRetention::from_secs(i64::MAX as u64 / 4), + commit_graph: true, + multi_pack_index: true, + bitmap: true, + } +} + +pub fn create_repo(scan: &Path, format: ObjectFormat, did: &str) -> Repo { + Layout::new(scan) + .with_object_format(format) + .with_default_branch(BranchName::new("main").unwrap()) + .create(&RepoDid::new(did).unwrap()) + .unwrap() +} + +pub fn commit_on(repo: &Repo, empty_tree: &str, body: u8, parents: Vec) -> Oid { + let tree = repo + .write_staged_tree( + Oid::from_hex(empty_tree).unwrap(), + &[StagedChange { + path: knot_types::RepoPath::new(format!("file{body}.txt")).unwrap(), + action: StagedAction::Put { + content: vec![body, body, body], + kind: EntryKind::Blob, + }, + }], + ) + .unwrap(); + repo.write_commit(&NewCommit { + tree, + parents, + author: identity(), + committer: identity(), + message: format!("commit {body}"), + extra_headers: Vec::new(), + }) + .unwrap() +} + +pub fn commit(repo: &Repo, body: u8, parents: Vec) -> Oid { + commit_on(repo, EMPTY_TREE_SHA1, body, parents) +} + +pub fn chain(repo: &Repo, empty_tree: &str, bodies: Range, start: Option) -> Oid { + bodies + .fold(start, |parent, body| { + let parents = parent.map(|tip| vec![tip]).unwrap_or_default(); + Some(commit_on(repo, empty_tree, body, parents)) + }) + .expect("a non-empty body range yields a tip") +} + +pub fn set_ref(repo: &Repo, name: &str, new: Oid) { + let refname = RefName::new(name).unwrap(); + let update = match repo.find_ref(&refname).unwrap() { + Some(old) => RefUpdate::Update { + name: refname, + old, + new, + }, + None => RefUpdate::Create { name: refname, new }, + }; + repo.update_ref(&update).unwrap(); +} + +pub fn delete_ref(repo: &Repo, name: &str) { + let refname = RefName::new(name).unwrap(); + let old = repo.find_ref(&refname).unwrap().unwrap(); + repo.update_ref(&RefUpdate::Delete { name: refname, old }) + .unwrap(); +} + +pub fn set_reflog_seconds(repo: &Repo, name: &str, seconds: &[i64]) { + let path = repo.git().git_dir().join("logs").join(name); + let text = std::fs::read_to_string(&path).expect("reflog file exists"); + let lines: Vec<&str> = text.lines().collect(); + assert_eq!( + lines.len(), + seconds.len(), + "set_reflog_seconds needs one timestamp per reflog line" + ); + let rewritten = lines + .iter() + .zip(seconds) + .map(|(line, secs)| rewrite_reflog_seconds(line, *secs)) + .collect::>() + .join("\n"); + std::fs::write(&path, format!("{rewritten}\n")).expect("rewrite reflog"); +} + +fn rewrite_reflog_seconds(line: &str, secs: i64) -> String { + let (meta, message) = line + .split_once('\t') + .expect("reflog line has a message tab"); + let tokens: Vec<&str> = meta.split(' ').collect(); + let tz = tokens.last().expect("reflog line has a timezone"); + let head = tokens[..tokens.len() - 2].join(" "); + format!("{head} {secs} {tz}\t{message}") +} + +pub fn git(repo: &Repo, args: &[&str]) -> (bool, String) { + let out = knot_fixtures::command(repo.git().git_dir()) + .args(args) + .output() + .expect("git runs"); + ( + out.status.success(), + String::from_utf8_lossy(&out.stderr).into_owned(), + ) +} + +pub fn fsck(repo: &Repo) -> (bool, String) { + git(repo, &["fsck", "--no-dangling", "--no-progress"]) +} + +pub fn fsck_clean(repo: &Repo) -> bool { + fsck(repo).0 +} + +pub fn assert_fsck_clean(repo: &Repo) { + let (clean, stderr) = fsck(repo); + assert!(clean, "fsck failed: {stderr}"); +} + +pub fn reachable_objects(repo: &Repo) -> BTreeSet { + let out = knot_fixtures::command(repo.git().git_dir()) + .args(["rev-list", "--objects", "--all"]) + .output() + .expect("git rev-list runs"); + assert!(out.status.success()); + String::from_utf8_lossy(&out.stdout) + .lines() + .filter_map(|line| line.split_whitespace().next()) + .map(str::to_string) + .collect() +} + +pub fn midx_verifies(repo: &Repo) -> Option<(bool, String)> { + let midx = repo.git().git_dir().join("objects/pack/multi-pack-index"); + midx.exists() + .then(|| git(repo, &["multi-pack-index", "verify"])) +} + +fn pack_entries(repo: &Repo) -> impl Iterator { + std::fs::read_dir(repo.git().git_dir().join("objects/pack")) + .into_iter() + .flatten() + .filter_map(Result::ok) + .map(|entry| entry.path()) +} + +pub fn idx_stems(repo: &Repo) -> BTreeSet { + pack_entries(repo) + .filter(|path| path.extension().is_some_and(|ext| ext == "idx")) + .filter_map(|path| { + path.file_stem() + .and_then(|stem| stem.to_str()) + .map(str::to_string) + }) + .collect() +} + +pub fn has_cruft_pack(repo: &Repo) -> bool { + pack_entries(repo).any(|path| path.extension().is_some_and(|ext| ext == "mtimes")) +} + +pub fn has_bitmap(repo: &Repo) -> bool { + pack_entries(repo).any(|path| path.extension().is_some_and(|ext| ext == "bitmap")) +} + +pub fn has_midx_bitmap(repo: &Repo) -> bool { + pack_entries(repo) + .filter_map(|path| { + path.file_name() + .and_then(|n| n.to_str()) + .map(str::to_string) + }) + .any(|name| name.starts_with("multi-pack-index-") && name.ends_with(".bitmap")) +} diff --git a/knot2/crates/knot-pack/fuzz/fuzz_targets/pack.rs b/knot2/crates/knot-pack/fuzz/fuzz_targets/pack.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/fuzz/fuzz_targets/pack.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + knot_pack::fuzz::pack(data); +}); diff --git a/knot2/crates/knot-pack/fuzz/fuzz_targets/pkt.rs b/knot2/crates/knot-pack/fuzz/fuzz_targets/pkt.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/fuzz/fuzz_targets/pkt.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + knot_pack::fuzz::pkt(data); +}); diff --git a/knot2/crates/knot-pack/fuzz/fuzz_targets/receive_commands.rs b/knot2/crates/knot-pack/fuzz/fuzz_targets/receive_commands.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/fuzz/fuzz_targets/receive_commands.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + knot_pack::fuzz::receive_commands(data); +}); diff --git a/knot2/crates/knot-pack/fuzz/fuzz_targets/upload_args.rs b/knot2/crates/knot-pack/fuzz/fuzz_targets/upload_args.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/fuzz/fuzz_targets/upload_args.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + knot_pack::fuzz::upload_args(data); +}); diff --git a/knot2/crates/knot-pack/tests/common/mod.rs b/knot2/crates/knot-pack/tests/common/mod.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-pack/tests/common/mod.rs @@ -0,0 +1,337 @@ +#![allow(dead_code, unused_imports)] + +use std::collections::BTreeSet; +use std::io::Write; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::sync::Arc; + +use axum::Router; +use knot_git::{Layout, Repo}; +use knot_pack::{RepoLookup, RepoResolver, RepoTarget}; +use knot_types::{ObjectFormat, RepoDid}; + +pub use knot_fixtures::{commit, must, run as git}; + +pub fn pkt(payload: &[u8]) -> Vec { + let mut out = format!("{:04x}", payload.len() + 4).into_bytes(); + out.extend_from_slice(payload); + out +} + +pub fn pack_objects(cwd: &Path, oids: &[String]) -> Vec { + let mut child = knot_fixtures::command(cwd) + .args(["pack-objects", "--stdout", "-q"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn pack-objects"); + child + .stdin + .take() + .unwrap() + .write_all(oids.join("\n").as_bytes()) + .unwrap(); + let out = child.wait_with_output().unwrap(); + assert!(out.status.success(), "pack-objects failed"); + out.stdout +} + +pub fn pack_objects_tuned(cwd: &Path, oids: &[String], ofs: bool) -> Vec { + let args: &[&str] = if ofs { + &[ + "pack-objects", + "--stdout", + "-q", + "--delta-base-offset", + "--depth=50", + "--window=250", + ] + } else { + &[ + "-c", + "pack.useDeltaBaseOffset=false", + "pack-objects", + "--stdout", + "-q", + "--depth=50", + "--window=250", + ] + }; + let mut child = knot_fixtures::command(cwd) + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn pack-objects"); + child + .stdin + .take() + .unwrap() + .write_all(oids.join("\n").as_bytes()) + .unwrap(); + let out = child.wait_with_output().unwrap(); + assert!(out.status.success(), "pack-objects failed"); + out.stdout +} + +pub fn index_into_bare(extra: &[&str], pack: &[u8]) -> bool { + let bare = tempfile::tempdir().unwrap(); + knot_fixtures::must( + bare.path(), + &["init", "--bare", "-q", bare.path().to_str().unwrap()], + ); + let args: Vec<&str> = std::iter::once("index-pack") + .chain(extra.iter().copied()) + .chain(std::iter::once("--stdin")) + .collect(); + knot_fixtures::feed(bare.path(), &args, pack).0 +} + +fn zlib(data: &[u8]) -> Vec { + let mut encoder = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::fast()); + encoder.write_all(data).unwrap(); + encoder.finish().unwrap() +} + +fn base128(value: u64) -> Vec { + let low = (value & 0x7f) as u8; + let rest = value >> 7; + if rest == 0 { + vec![low] + } else { + std::iter::once(low | 0x80).chain(base128(rest)).collect() + } +} + +fn obj_header(obj_type: u8, size: usize) -> Vec { + fn tail(size: usize) -> Vec { + if size == 0 { + Vec::new() + } else { + let byte = (size & 0x7f) as u8; + let rest = size >> 7; + let cont = if rest > 0 { 0x80 } else { 0 }; + std::iter::once(byte | cont).chain(tail(rest)).collect() + } + } + let rest = size >> 4; + let cont = if rest > 0 { 0x80 } else { 0 }; + std::iter::once((obj_type << 4) | (size & 0x0f) as u8 | cont) + .chain(tail(rest)) + .collect() +} + +fn ofs_distance(distance: u64) -> Vec { + fn prefix(value: u64) -> Vec { + if value == 0 { + Vec::new() + } else { + let reduced = value - 1; + prefix(reduced >> 7) + .into_iter() + .chain(std::iter::once(0x80 | (reduced & 0x7f) as u8)) + .collect() + } + } + prefix(distance >> 7) + .into_iter() + .chain(std::iter::once((distance & 0x7f) as u8)) + .collect() +} + +pub fn delta_bomb_pack(declared_result_bytes: u64) -> Vec { + let base = b"hi"; + let mut entry0 = obj_header(3, base.len()); + entry0.extend(zlib(base)); + + let delta_stream: Vec = base128(base.len() as u64) + .into_iter() + .chain(base128(declared_result_bytes)) + .chain([0x90, 0x02]) + .collect(); + let mut entry1 = obj_header(6, delta_stream.len()); + entry1.extend(ofs_distance(entry0.len() as u64)); + entry1.extend(zlib(&delta_stream)); + + let mut pack = b"PACK".to_vec(); + pack.extend_from_slice(&2u32.to_be_bytes()); + pack.extend_from_slice(&2u32.to_be_bytes()); + pack.extend_from_slice(&entry0); + pack.extend_from_slice(&entry1); + + let mut hasher = gix_hash::hasher(gix_hash::Kind::Sha1); + hasher.update(&pack); + let checksum = hasher.try_finalize().unwrap(); + pack.extend_from_slice(checksum.as_bytes()); + pack +} + +pub fn receive_request(refname: &str, old: &str, new: &str, pack: &[u8]) -> Vec { + let mut first = format!("{old} {new} {refname}").into_bytes(); + first.push(0); + first.extend_from_slice(b"report-status\n"); + let mut req = pkt(&first); + req.extend_from_slice(b"0000"); + req.extend_from_slice(pack); + req +} + +pub fn unsideband(resp: &[u8]) -> Vec { + let mut out = Vec::new(); + let mut pos = 0usize; + let mut in_pack = false; + while pos + 4 <= resp.len() { + let len = std::str::from_utf8(&resp[pos..pos + 4]) + .ok() + .and_then(|hex| usize::from_str_radix(hex, 16).ok()) + .unwrap_or(0); + pos += 4; + if len < 4 { + continue; + } + let end = (pos + len - 4).min(resp.len()); + let payload = &resp[pos..end]; + pos = end; + if payload == b"packfile\n" { + in_pack = true; + } else if in_pack && payload.first() == Some(&1) { + out.extend_from_slice(&payload[1..]); + } + } + out +} + +pub fn object_set(dir: &Path) -> BTreeSet { + must( + dir, + &[ + "cat-file", + "--batch-all-objects", + "--batch-check=%(objectname)", + ], + ) + .lines() + .map(|line| line.trim().to_string()) + .filter(|line| !line.is_empty()) + .collect() +} + +pub fn incompressible(seed: u64, len: usize) -> Vec { + let mut state = seed.wrapping_mul(0x9E37_79B9_7F4A_7C15).wrapping_add(1); + (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + (state & 0xff) as u8 + }) + .collect() +} + +pub fn serve_dids() -> Arc { + Arc::new(|target: &RepoTarget| match target { + RepoTarget::Did(did) => RepoLookup::Hosted(did.clone()), + RepoTarget::OwnerRkey(_, _) => RepoLookup::Unhosted, + }) +} + +pub async fn spawn(router: Router, bind: &str) -> SocketAddr { + let listener = tokio::net::TcpListener::bind(bind).await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, router).await.unwrap(); + }); + addr +} + +pub fn advance_via_receive(bare: &Path, work: &Path, old: &str, new: &str) { + let oids: Vec = must(work, &["rev-list", "--objects", new, "--not", old]) + .lines() + .filter_map(|line| line.split_whitespace().next()) + .map(str::to_string) + .collect(); + let request = receive_request("refs/heads/main", old, new, &pack_objects(work, &oids)); + let repo = Repo::open(bare).expect("open knot bare"); + let report = knot_pack::receive_pack(&repo, &request).expect("knot receive"); + assert!( + String::from_utf8_lossy(&report).contains("ok refs/heads/main"), + "knot must accept a receive that advances main" + ); +} + +pub fn seed_branches_and_tag(work: &Path, bares: [&Path; 2], format: ObjectFormat) { + let fmt = format!("--object-format={}", format.capability()); + std::fs::create_dir_all(work).unwrap(); + must(work, &["init", &fmt, "-q", "-b", "main"]); + std::fs::write(work.join("README.md"), "seed\n").unwrap(); + must(work, &["add", "-A"]); + must(work, &["commit", "-q", "-m", "c1"]); + let c1 = must(work, &["rev-parse", "HEAD"]); + std::fs::write(work.join("src.txt"), "more\n").unwrap(); + must(work, &["add", "-A"]); + must(work, &["commit", "-q", "-m", "c2"]); + must(work, &["checkout", "-q", "-b", "dev", &c1]); + std::fs::write(work.join("dev.txt"), "branch\n").unwrap(); + must(work, &["add", "-A"]); + must(work, &["commit", "-q", "-m", "c3"]); + must(work, &["checkout", "-q", "main"]); + must(work, &["tag", "-a", "v1", "-m", "release"]); + bares.into_iter().for_each(|bare| { + must( + work, + &["push", "-q", bare.to_str().unwrap(), "main", "dev", "v1"], + ); + must(bare, &["symbolic-ref", "HEAD", "refs/heads/main"]); + }); +} + +pub fn seeded(layout: &Layout, did: &RepoDid) -> (Repo, tempfile::TempDir, String, Vec) { + let bare = layout.create(did).unwrap(); + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + must(work, &["init", "-q", "-b", "main"]); + commit(work, "a.txt", "x\n", "c1"); + let c1 = must(work, &["rev-parse", "HEAD"]); + let oids: Vec = must(work, &["rev-list", "--objects", &c1]) + .lines() + .map(|line| line.split_whitespace().next().unwrap().to_string()) + .collect(); + let pack = pack_objects(work, &oids); + (bare, work_dir, c1, pack) +} + +pub struct Stand { + pub scan: tempfile::TempDir, + pub scratch: tempfile::TempDir, + pub layout: Layout, + pub addr: SocketAddr, + pub bare: PathBuf, +} + +pub async fn stand(did: &RepoDid) -> Stand { + let scan = tempfile::tempdir().unwrap(); + let layout = Layout::new(scan.path()); + layout.create(did).unwrap(); + let bare = layout.repo_path(did).unwrap(); + let addr = spawn( + knot_pack::router( + layout.clone(), + serve_dids(), + std::sync::Arc::new(knot_runtime::SystemClock), + ), + "[::1]:0", + ) + .await; + let scratch = tempfile::tempdir().unwrap(); + Stand { + scan, + scratch, + layout, + addr, + bare, + } +} diff --git a/knot2/crates/knot-sim/tests/common/mod.rs b/knot2/crates/knot-sim/tests/common/mod.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-sim/tests/common/mod.rs @@ -0,0 +1,303 @@ +#![allow(dead_code)] + +use std::net::SocketAddr; +use std::num::{NonZeroU32, NonZeroU64}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; + +use bytes::{Buf, Bytes}; +use http::{Method, StatusCode}; +use knot_edge::{ + BodyInactivityTimeout, BurstSize, CertSource, EdgeConfig, EdgeGuards, HeaderTimeout, + IdleTimeout, ListenLimits, MaxInflightRequests, RequestTimeout, RequestsPerSecond, + RequiresFullHandshake, StaticCertPaths, TlsSetup, WriteRequestTimeout, ZeroRttRoutes, +}; +use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}; +use rustls::crypto::aws_lc_rs; +use rustls::pki_types::{CertificateDer, ServerName, UnixTime}; +use rustls::{DigitallySignedStruct, SignatureScheme}; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; + +pub fn nz32(value: u32) -> NonZeroU32 { + NonZeroU32::new(value).unwrap() +} + +pub fn nz64(value: u64) -> NonZeroU64 { + NonZeroU64::new(value).unwrap() +} + +pub fn pkt(payload: &[u8]) -> Vec { + assert!( + payload.len() + 4 <= 0xFFF0, + "pkt-line payload exceeds the 65516-byte maximum" + ); + let mut out = format!("{:04x}", payload.len() + 4).into_bytes(); + out.extend_from_slice(payload); + out +} + +pub fn free_port() -> u16 { + std::net::TcpListener::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap() + .port() +} + +pub fn write_self_signed(dir: &Path) -> (PathBuf, PathBuf, Vec) { + let generated = + rcgen::generate_simple_self_signed(vec!["localhost".to_string(), "127.0.0.1".to_string()]) + .unwrap(); + let cert_path = dir.join("cert.pem"); + let key_path = dir.join("key.pem"); + std::fs::write(&cert_path, generated.cert.pem()).unwrap(); + std::fs::write(&key_path, generated.signing_key.serialize_pem()).unwrap(); + let der = generated.cert.der().as_ref().to_vec(); + (cert_path, key_path, der) +} + +pub fn edge_config(addr: SocketAddr, cert: PathBuf, key: PathBuf) -> EdgeConfig { + EdgeConfig { + http_addr: knot_edge::PublicBind::new(addr), + limits: ListenLimits::new( + HeaderTimeout::from_millis(nz64(30_000)), + IdleTimeout::from_millis(nz64(120_000)), + nz32(1024), + ), + guards: EdgeGuards::new( + RequestsPerSecond::new(nz32(1_000_000)), + BurstSize::new(nz32(1_000_000)), + MaxInflightRequests::new(nz32(10_000)), + RequestTimeout::from_millis(nz64(120_000)), + BodyInactivityTimeout::from_millis(nz64(120_000)), + WriteRequestTimeout::from_millis(nz64(1_800_000)), + None, + ), + tls: Some(TlsSetup { + source: CertSource::Static(StaticCertPaths { + cert_path: knot_edge::CertChainPath::new(cert), + key_path: knot_edge::PrivateKeyPath::new(key), + }), + http3: true, + internal: None, + }), + } +} + +pub struct Edge { + pub addr: SocketAddr, + pub shutdown: CancellationToken, + pub task: JoinHandle>, + pub client: quinn::Endpoint, +} + +async fn probe_identity( + endpoint: &quinn::Endpoint, + addr: SocketAddr, + expected_cert: &[u8], +) -> Option { + let connecting = endpoint.connect(addr, "localhost").ok()?; + let connection = tokio::time::timeout(Duration::from_millis(250), connecting) + .await + .ok()? + .ok()?; + let ours = connection + .peer_identity() + .and_then(|identity| identity.downcast::>>().ok()) + .map(|certs| { + certs + .first() + .is_some_and(|cert| cert.as_ref() == expected_cert) + }) + .unwrap_or(false); + connection.close(0u32.into(), b"probe done"); + Some(ours) +} + +async fn await_ready( + endpoint: &quinn::Endpoint, + addr: SocketAddr, + task: &mut JoinHandle>, + expected_cert: &[u8], +) -> bool { + for _ in 0..200 { + if task.is_finished() { + return false; + } + if let Some(ours) = probe_identity(endpoint, addr, expected_cert).await { + return ours; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + false +} + +pub async fn serve_edge( + certdir: &Path, + build: impl Fn() -> (RequiresFullHandshake, ZeroRttRoutes), +) -> Edge { + for _ in 0..8 { + let addr: SocketAddr = format!("127.0.0.1:{}", free_port()).parse().unwrap(); + let (cert, key, cert_der) = write_self_signed(certdir); + let (app, advertisement) = build(); + let shutdown = CancellationToken::new(); + let mut task = tokio::spawn(knot_edge::serve( + edge_config(addr, cert, key), + app, + advertisement, + shutdown.clone(), + )); + let client = h3_client(); + if await_ready(&client, addr, &mut task, &cert_der).await { + return Edge { + addr, + shutdown, + task, + client, + }; + } + client.close(0u32.into(), b"stand up retry"); + shutdown.cancel(); + let _ = task.await; + } + panic!("couldn't bind a free TCP+UDP port for the edge after several attempts"); +} + +#[derive(Debug)] +struct AcceptAnyServerCert; + +impl ServerCertVerifier for AcceptAnyServerCert { + fn verify_server_cert( + &self, + _end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + _server_name: &ServerName<'_>, + _ocsp_response: &[u8], + _now: UnixTime, + ) -> Result { + Ok(ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature( + message, + cert, + dss, + &aws_lc_rs::default_provider().signature_verification_algorithms, + ) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature( + message, + cert, + dss, + &aws_lc_rs::default_provider().signature_verification_algorithms, + ) + } + + fn supported_verify_schemes(&self) -> Vec { + aws_lc_rs::default_provider() + .signature_verification_algorithms + .supported_schemes() + } +} + +pub fn h3_client() -> quinn::Endpoint { + let mut crypto = + rustls::ClientConfig::builder_with_provider(Arc::new(aws_lc_rs::default_provider())) + .with_protocol_versions(&[&rustls::version::TLS13]) + .unwrap() + .dangerous() + .with_custom_certificate_verifier(Arc::new(AcceptAnyServerCert)) + .with_no_client_auth(); + crypto.alpn_protocols = vec![b"h3".to_vec()]; + crypto.resumption = rustls::client::Resumption::disabled(); + let quic = quinn::crypto::rustls::QuicClientConfig::try_from(crypto).unwrap(); + let mut endpoint = quinn::Endpoint::client("127.0.0.1:0".parse().unwrap()).unwrap(); + endpoint.set_default_client_config(quinn::ClientConfig::new(Arc::new(quic))); + endpoint +} + +pub async fn drain( + stream: &mut h3::client::RequestStream, Bytes>, +) -> Vec { + let mut out = Vec::new(); + while let Some(mut chunk) = stream.recv_data().await.unwrap() { + out.extend_from_slice(&chunk.copy_to_bytes(chunk.remaining())); + } + out +} + +pub async fn finish_request( + stream: &mut h3::client::RequestStream, Bytes>, +) { + match stream.finish().await { + Ok(()) => (), + Err(h3::error::StreamError::RemoteTerminate { code, .. }) + if code == h3::error::Code::H3_NO_ERROR => {} + Err(error) => panic!("finishing the request stream failed: {error}"), + } +} + +pub async fn h3_request( + edge: &Edge, + method: Method, + uri: String, + headers: &[(&str, &str)], + body: Option, + warmup: Option<&str>, +) -> (StatusCode, Vec) { + let connection = edge + .client + .connect(edge.addr, "localhost") + .unwrap() + .await + .unwrap(); + let quic = connection.clone(); + let (mut driver, mut sender) = h3::client::new(h3_quinn::Connection::new(connection)) + .await + .unwrap(); + let drive = tokio::spawn(async move { + let _ = std::future::poll_fn(|cx| driver.poll_close(cx)).await; + }); + + if let Some(warmup) = warmup { + let request = http::Request::get(warmup).body(()).unwrap(); + let mut stream = sender.send_request(request).await.unwrap(); + finish_request(&mut stream).await; + let _ = stream.recv_response().await.unwrap(); + drain(&mut stream).await; + } + + let request = headers + .iter() + .fold( + http::Request::builder().method(method).uri(uri), + |builder, (name, value)| builder.header(*name, *value), + ) + .body(()) + .unwrap(); + let mut stream = sender.send_request(request).await.unwrap(); + if let Some(body) = body { + stream.send_data(body).await.unwrap(); + } + finish_request(&mut stream).await; + let status = stream.recv_response().await.unwrap().status(); + let out = drain(&mut stream).await; + quic.close(0u32.into(), b"done"); + drive.abort(); + (status, out) +} diff --git a/knot2/crates/knot-xrpc/tests/common/mod.rs b/knot2/crates/knot-xrpc/tests/common/mod.rs new file mode 100644 --- /dev/null +++ b/knot2/crates/knot-xrpc/tests/common/mod.rs @@ -0,0 +1,688 @@ +#![allow(dead_code)] + +use std::collections::BTreeSet; +use std::path::Path; +use std::sync::Arc; + +use std::sync::atomic::{AtomicU64, Ordering}; + +use axum::Router; +use axum::body::{Body, Bytes}; +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use http::{HeaderMap, StatusCode, header}; +use k256::ecdsa::signature::Signer; +use k256::ecdsa::{Signature, SigningKey}; +use tower::ServiceExt; + +use knot_atproto::Atproto; +use knot_cob::{CobHome, CobStore}; +use knot_cobs::{ + CollaboratorsChange, CollaboratorsCob, Grant, MembersChange, MembersCob, Registration, + RegistryChange, RepoRegistryCob, register_repo, +}; +use knot_git::{Layout, Repo}; +use knot_index::Index; +use knot_runtime::{ + FakeHttp, HttpRequest, HttpResponse, ManualClock, NetworkError, OsEntropy, UnixMicros, +}; +use knot_secrets::{MasterKey, SealedStore}; +use knot_types::{ + AccountDid, AuthorName, Email, KnotHostname, KnotId, ObjectFormat, Oid, OwnerDid, RepoDid, + RepoName, RepoRkey, UnixSeconds, +}; +use knot_xrpc::{ + ArchiveLimit, Budgets, ByteLimits, CobLocks, GlobalQuota, LimitConfig, PerActorQuota, + PreAuthLimiter, Reservations, ResponseLimit, XrpcState, +}; + +pub const KNOT_HOST: &str = "knot.nel.pet"; +pub const OWNER: &str = "did:web:olaren.dev"; + +pub type Responder = Box Result + Send + Sync>; + +pub struct World { + _dir: tempfile::TempDir, + pub layout: Layout, + pub lfs_dir: std::path::PathBuf, + pub router: Router, + pub state: Arc, ManualClock>>, +} + +impl World { + pub fn new() -> Self { + Self::build(true, ByteLimits::default(), ObjectFormat::SHA1) + } + + pub fn unshed() -> Self { + Self::build_with_limits( + true, + ByteLimits::default(), + ObjectFormat::SHA1, + LimitConfig::unmetered(), + ) + } + + pub fn sha256() -> Self { + Self::build(true, ByteLimits::default(), ObjectFormat::SHA256) + } + + pub fn warming() -> Self { + Self::build(false, ByteLimits::default(), ObjectFormat::SHA1) + } + + pub fn with_response_limit(response: ResponseLimit) -> Self { + Self::build( + true, + ByteLimits { + response, + ..ByteLimits::default() + }, + ObjectFormat::SHA1, + ) + } + + pub fn with_archive_limit(archive: ArchiveLimit) -> Self { + Self::build( + true, + ByteLimits { + archive, + ..ByteLimits::default() + }, + ObjectFormat::SHA1, + ) + } + + fn build(rebuilt: bool, byte_limits: ByteLimits, object_format: ObjectFormat) -> Self { + Self::build_with_limits(rebuilt, byte_limits, object_format, LimitConfig::default()) + } + + fn build_with_limits( + rebuilt: bool, + byte_limits: ByteLimits, + object_format: ObjectFormat, + limits: LimitConfig, + ) -> Self { + let dir = tempfile::tempdir().unwrap(); + let scan_path = dir.path().join("repos"); + std::fs::create_dir_all(&scan_path).unwrap(); + let knot = KnotId::new(format!("did:web:{KNOT_HOST}")).unwrap(); + let layout = Layout::new(&scan_path) + .with_object_format(object_format) + .reserving_meta(&knot) + .unwrap(); + layout.bootstrap_meta(&knot).unwrap(); + let meta_path = layout.meta_path(&knot).unwrap(); + let index = Arc::new(Index::new(meta_path.clone(), layout.clone())); + if rebuilt { + index.rebuild().unwrap(); + } + + let responder: Responder = Box::new(|request| { + let host = request.url.host_str().unwrap_or_default(); + let did = match host { + "plc.directory" => request.url.path().trim_start_matches('/').to_string(), + host if request.url.path().ends_with("/.well-known/did.json") => { + format!("did:web:{host}") + } + _ => String::new(), + }; + let body = match did.starts_with("did:") { + true => did_doc_for(&did), + false => Bytes::new(), + }; + Ok(HttpResponse { + status: StatusCode::OK, + headers: http::HeaderMap::new(), + body, + }) + }); + let atproto = Arc::new(Atproto::new( + FakeHttp::new(responder), + ManualClock::new(UnixMicros::new(1_000_000_000)), + knot.clone(), + knot_atproto::PlcDirectory::new(url::Url::parse("https://plc.directory/").unwrap()) + .unwrap(), + )); + let secrets = Arc::new( + SealedStore::open( + dir.path().join("keys.sealed"), + &MasterKey::new([7u8; 32]).unwrap(), + Box::new(OsEntropy), + ) + .unwrap(), + ); + secrets.ensure(&knot).unwrap(); + + let lfs_store = dir.path().join("lfs"); + std::fs::create_dir_all(&lfs_store).unwrap(); + let lfs_handle = knot_lfs::LfsHandle::open( + knot_lfs::LfsStorePath::new(&lfs_store), + knot_lfs::LfsSize::new(64 * 1024 * 1024), + knot_lfs::FreeSpaceFloor::new(0), + ) + .unwrap(); + + let state = Arc::new(XrpcState { + layout: layout.clone(), + index, + atproto, + secrets, + entropy: Arc::new(OsEntropy), + ci_logs: None, + admins: BTreeSet::new(), + admission: knot_types::AdmissionPolicy::Closed, + knot_did: knot, + knot_hostname: KnotHostname::new(KNOT_HOST).unwrap(), + meta_path, + knot_service_url: knot_types::KnotServiceUrl::new(format!("https://{KNOT_HOST}")) + .unwrap(), + limiter: Arc::new(PreAuthLimiter::with_config(limits)), + cob_locks: Arc::new(CobLocks::default()), + reservations: Arc::new(Reservations::new( + knot_xrpc::ReservationTtl::new(1_000_000), + PerActorQuota::new(16), + GlobalQuota::new(16), + )), + trusted_proxy_header: None, + committer: knot_xrpc::Committer { + name: AuthorName::new("Tangled"), + email: Email::new("noreply@tangled.sh"), + }, + byte_limits, + budgets: Budgets::default(), + git_http: Arc::new(FakeHttp::new(|_request: &HttpRequest| { + Err(NetworkError::Connect( + "no git upstream is served in this test".to_string(), + )) + })), + pack_limits: knot_pack::PackLimits::default(), + service_owner: AccountDid::new(OWNER).unwrap(), + events: Arc::new(knot_events::EventLog::new( + ManualClock::new(UnixMicros::new(1_000_000_000)), + knot_events::ReplayBounds::new( + knot_events::ReplayEvents::new(1024).unwrap(), + knot_events::ReplayBytes::new(16 << 20).unwrap(), + ), + )), + subscriber_gate: Arc::new(knot_events::SubscriberGate::new( + knot_events::GlobalSubscriberLimit::new(16), + knot_events::PerPeerSubscriberLimit::new(4), + )), + maintenance: knot_maintenance::MaintenanceHandle::disabled(), + appview: knot_types::AppviewEndpoint::new("https://tangled.test").unwrap(), + slots: knot_resource::Slots::testing(8), + lfs: Some(knot_xrpc::LfsWeb::new(lfs_handle, 8)), + catalog: Arc::new(knot_messages::Catalog::defaults()), + }); + let router = knot_xrpc::router(Arc::clone(&state)); + Self { + _dir: dir, + layout, + lfs_dir: lfs_store, + router, + state, + } + } + + pub fn register(&self, did: &RepoDid, rkey: &str) { + let meta = Repo::open(&self.state.meta_path).unwrap(); + let store = CobStore::new(&meta); + let home = CobHome::from(&self.state.knot_did); + let signer = self.state.secrets.signer(&self.state.knot_did).unwrap(); + let registration = Registration { + owner: OwnerDid::new(OWNER).unwrap(), + rkey: RepoRkey::new(rkey).unwrap(), + name: RepoName::new(rkey).unwrap(), + repo: did.clone(), + created_at: UnixSeconds::new(1_000), + }; + match store.list::().unwrap().as_slice() { + [] => { + store + .create( + &home, + &RegistryChange::Register(registration), + &signer, + UnixSeconds::new(1_000), + ) + .unwrap(); + } + [object] => { + register_repo( + &store, + &home, + *object, + registration, + &signer, + UnixSeconds::new(1_000), + ) + .unwrap(); + } + many => panic!("{} registry objects", many.len()), + } + self.state.index.refresh_registry().unwrap(); + } + + pub fn add_member(&self, subject: &str, added_by: &str, at: i64) { + let meta = Repo::open(&self.state.meta_path).unwrap(); + let store = CobStore::new(&meta); + let home = CobHome::from(&self.state.knot_did); + let signer = self.state.secrets.signer(&self.state.knot_did).unwrap(); + let change = MembersChange::Add(grant(subject, added_by, at)); + match store.list::().unwrap().as_slice() { + [] => { + store + .create(&home, &change, &signer, UnixSeconds::new(at)) + .unwrap(); + } + [object] => { + store + .update(&home, *object, &change, &signer, UnixSeconds::new(at)) + .unwrap(); + } + many => panic!("{} members objects", many.len()), + } + self.state.index.refresh_members().unwrap(); + } + + pub fn add_collaborator(&self, repo: &RepoDid, subject: &str, added_by: &str, at: i64) { + let git = self.layout.open(repo).unwrap(); + let store = CobStore::new(&git); + let home = CobHome::from(repo); + let signer = self.state.secrets.signer(&self.state.knot_did).unwrap(); + let change = CollaboratorsChange::Add(grant(subject, added_by, at)); + match store.list::().unwrap().as_slice() { + [] => { + store + .create(&home, &change, &signer, UnixSeconds::new(at)) + .unwrap(); + } + [object] => { + store + .update(&home, *object, &change, &signer, UnixSeconds::new(at)) + .unwrap(); + } + many => panic!("{} collaborators objects", many.len()), + } + self.state.index.refresh_collaborators(repo).unwrap(); + } +} + +fn grant(subject: &str, added_by: &str, at: i64) -> Grant { + Grant { + subject: AccountDid::new(subject).unwrap(), + added_by: AccountDid::new(added_by).unwrap(), + created_at: UnixSeconds::new(at), + } +} + +fn actor_key() -> SigningKey { + SigningKey::from_bytes(&[9u8; 32].into()).unwrap() +} + +fn did_doc_for(did: &str) -> Bytes { + let sec1 = actor_key() + .verifying_key() + .to_encoded_point(true) + .as_bytes() + .to_vec(); + let multikey = knot_types::crypto::multikey(0xe7, &sec1); + let body = serde_json::json!({ + "id": did, + "alsoKnownAs": [], + "verificationMethod": [{ + "id": format!("{did}#atproto"), + "type": "Multikey", + "controller": did, + "publicKeyMultibase": multikey + }], + "service": [{ + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": "https://pds.oyster.cafe" + }] + }); + Bytes::from(serde_json::to_vec(&body).unwrap()) +} + +static JTI: AtomicU64 = AtomicU64::new(0); + +fn service_jwt(nsid: &str, actor: &str) -> String { + let nonce = JTI.fetch_add(1, Ordering::SeqCst); + let header = URL_SAFE_NO_PAD.encode(br#"{"alg":"ES256K","typ":"JWT"}"#); + let claims = serde_json::json!({ + "iss": actor, + "aud": format!("did:web:{KNOT_HOST}"), + "exp": 1_001, + "iat": 999, + "jti": format!("nonce-{nonce}"), + "lxm": nsid, + }); + let payload = URL_SAFE_NO_PAD.encode(serde_json::to_vec(&claims).unwrap()); + let signing_input = format!("{header}.{payload}"); + let signature: Signature = actor_key().sign(signing_input.as_bytes()); + format!( + "{signing_input}.{}", + URL_SAFE_NO_PAD.encode(signature.to_bytes()) + ) +} + +pub async fn post_authed( + world: &World, + path: &str, + actor: &str, + value: serde_json::Value, +) -> (StatusCode, serde_json::Value) { + let nsid = path + .strip_prefix("/xrpc/") + .expect("post_authed path names an xrpc method"); + let token = service_jwt(nsid, actor); + let request = http::Request::builder() + .method("POST") + .uri(path) + .header(header::CONTENT_TYPE, "application/json") + .header(header::AUTHORIZATION, format!("Bearer {token}")) + .body(Body::from(serde_json::to_vec(&value).unwrap())) + .unwrap(); + let response = world.router.clone().oneshot(request).await.unwrap(); + let status = response.status(); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + (status, serde_json::from_slice(&body).unwrap()) +} + +pub async fn post_json( + world: &World, + path: &str, + value: serde_json::Value, +) -> (StatusCode, serde_json::Value) { + let request = http::Request::builder() + .method("POST") + .uri(path) + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from(serde_json::to_vec(&value).unwrap())) + .unwrap(); + let response = world.router.clone().oneshot(request).await.unwrap(); + let status = response.status(); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + (status, serde_json::from_slice(&body).unwrap()) +} + +pub fn git_run(cwd: &Path, when: &str, author: (&str, &str), args: &[&str]) -> String { + let output = knot_fixtures::command_at(cwd, when) + .args(args) + .env("GIT_AUTHOR_NAME", author.0) + .env("GIT_AUTHOR_EMAIL", author.1) + .env("GIT_COMMITTER_NAME", author.0) + .env("GIT_COMMITTER_EMAIL", author.1) + .output() + .expect("git is available"); + assert!( + output.status.success(), + "git {args:?} failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).unwrap().trim().to_string() +} + +pub fn sh_git_at(cwd: &Path, when: &str, args: &[&str]) -> String { + git_run(cwd, when, ("nel", "nel@oyster.cafe"), args) +} + +pub fn sh_git(cwd: &Path, args: &[&str]) -> String { + sh_git_at(cwd, "2026-06-01T12:30:00+02:00", args) +} + +pub fn commit_file(work: &Path, file: &str, contents: &[u8], message: &str, when: &str) { + let target = work.join(file); + if let Some(parent) = target.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(target, contents).unwrap(); + sh_git_at(work, when, &["add", "-A"]); + sh_git_at(work, when, &["commit", "-q", "-m", message]); +} + +pub fn seeded(world: &World, rkey: &str) -> (RepoDid, tempfile::TempDir) { + seeded_with_format(world, rkey, ObjectFormat::SHA1) +} + +pub fn seeded_with_format( + world: &World, + rkey: &str, + object_format: ObjectFormat, +) -> (RepoDid, tempfile::TempDir) { + let did = RepoDid::new(format!("did:plc:{rkey}fixture")).unwrap(); + world.layout.create(&did).unwrap(); + world.register(&did, rkey); + let bare = world.layout.repo_path(&did).unwrap(); + let work_dir = tempfile::tempdir().unwrap(); + let work = work_dir.path(); + let init = match object_format == ObjectFormat::SHA256 { + true => vec!["init", "-q", "--object-format=sha256", "-b", "main"], + false => vec!["init", "-q", "-b", "main"], + }; + sh_git(work, &init); + commit_file( + work, + "README.md", + b"# coral\n\nhello\n", + "first", + "2026-06-01T12:30:00+02:00", + ); + commit_file( + work, + "src/main.rs", + b"fn main() {\n println!(\"reef\");\n}\n", + "add main", + "2026-06-01T12:31:00+02:00", + ); + commit_file( + work, + "logo.png", + b"\x89PNG\r\n\x1a\n0000binarybytes\x00\x01", + "add logo", + "2026-06-01T12:32:00+02:00", + ); + sh_git(work, &["tag", "lightweight"]); + sh_git_at( + work, + "2026-06-01T12:32:30+02:00", + &["tag", "-a", "v1.0.0", "-m", "release one"], + ); + commit_file( + work, + "README.md", + b"# coral\n\nhello reef\n", + "update readme", + "2026-06-01T12:33:00+02:00", + ); + sh_git( + work, + &["push", "-q", "--tags", bare.to_str().unwrap(), "main"], + ); + (did, work_dir) +} + +pub fn empty_repo(world: &World, rkey: &str) -> (RepoDid, String, tempfile::TempDir) { + let did = RepoDid::new(format!("did:plc:{rkey}fixture")).unwrap(); + world.layout.create(&did).unwrap(); + world.register(&did, rkey); + let bare = world + .layout + .repo_path(&did) + .unwrap() + .to_str() + .unwrap() + .to_string(); + let work_dir = tempfile::tempdir().unwrap(); + sh_git(work_dir.path(), &["init", "-q", "-b", "main"]); + (did, bare, work_dir) +} + +pub fn seeded_feature_branch(world: &World, rkey: &str) -> (RepoDid, Oid, Oid) { + let (did, bare, work_dir) = empty_repo(world, rkey); + let work = work_dir.path(); + commit_file( + work, + "reef.txt", + b"one\ntwo\n", + "base", + "2026-06-01T12:30:00+02:00", + ); + sh_git(work, &["checkout", "-q", "-b", "feature"]); + commit_file( + work, + "reef.txt", + b"one\nTWO\n", + "capitalize two\n\nbecause waves", + "2026-06-01T12:31:00+02:00", + ); + commit_file( + work, + "kelp.txt", + b"frond\n", + "add kelp", + "2026-06-01T12:32:00+02:00", + ); + sh_git(work, &["push", "-q", &bare, "main", "feature"]); + let main = Oid::from_hex(&sh_git(work, &["rev-parse", "main"])).unwrap(); + let feature = Oid::from_hex(&sh_git(work, &["rev-parse", "feature"])).unwrap(); + (did, main, feature) +} + +pub async fn get_with_headers( + world: &World, + path_and_query: &str, + headers: HeaderMap, +) -> (StatusCode, HeaderMap, Bytes) { + let mut request = http::Request::builder() + .method("GET") + .uri(path_and_query) + .body(Body::empty()) + .unwrap(); + request.headers_mut().extend(headers); + let response = world.router.clone().oneshot(request).await.unwrap(); + let status = response.status(); + let response_headers = response.headers().clone(); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + (status, response_headers, body) +} + +pub async fn get(world: &World, path_and_query: &str) -> (StatusCode, HeaderMap, Bytes) { + get_with_headers(world, path_and_query, HeaderMap::new()).await +} + +pub async fn get_json(world: &World, path_and_query: &str) -> serde_json::Value { + let (status, _, body) = get(world, path_and_query).await; + assert_eq!( + status, + StatusCode::OK, + "GET {path_and_query} failed: {}", + String::from_utf8_lossy(&body) + ); + serde_json::from_slice(&body).unwrap() +} + +pub async fn get_error(world: &World, path_and_query: &str) -> (StatusCode, String) { + let (status, _, body) = get(world, path_and_query).await; + assert!(!status.is_success(), "GET {path_and_query} unexpectedly ok"); + let value: serde_json::Value = serde_json::from_slice(&body).unwrap(); + (status, value["error"].as_str().unwrap().to_string()) +} + +pub fn ref_names(value: &serde_json::Value, key: &str) -> Vec { + value[key] + .as_array() + .unwrap() + .iter() + .map(|entry| entry["ref"].as_str().unwrap().to_string()) + .collect() +} + +pub fn repo_dids(value: &serde_json::Value) -> Vec { + value["repos"] + .as_array() + .unwrap() + .iter() + .map(|entry| entry["repo"].as_str().unwrap().to_string()) + .collect() +} + +pub async fn archive_full(world: &World, did: &RepoDid) -> (String, String, Bytes) { + let (status, headers, body) = get( + world, + &format!("/xrpc/sh.tangled.repo.archive?repo={did}&ref=main"), + ) + .await; + assert_eq!(status, StatusCode::OK); + let etag = headers + .get(header::ETAG) + .unwrap() + .to_str() + .unwrap() + .to_string(); + let last_modified = headers + .get(header::LAST_MODIFIED) + .unwrap() + .to_str() + .unwrap() + .to_string(); + (etag, last_modified, body) +} + +pub async fn assert_immutable_round_trip( + world: &World, + headers: &HeaderMap, + full: &Bytes, + etag: &str, +) { + let link = headers.get(header::LINK).unwrap().to_str().unwrap(); + let immutable = link + .trim_start_matches('<') + .split('>') + .next() + .unwrap() + .strip_prefix(&format!("https://{KNOT_HOST}")) + .expect("the immutable link points at this knot"); + let (status, immutable_headers, immutable_body) = get(world, immutable).await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + &immutable_body, full, + "following the immutable link regenerates the very bytes it was attached to" + ); + assert_eq!( + immutable_headers + .get(header::ETAG) + .unwrap() + .to_str() + .unwrap(), + etag, + "the immutable link shares the etag of the response that advertised it" + ); +} + +pub async fn assert_warming(world: &World, path: &str, expected_error: Option<&str>) { + let (status, error) = get_error(world, path).await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE, "path {path}"); + if let Some(expected) = expected_error { + assert_eq!(error, expected, "path {path}"); + } +} + +pub async fn assert_post_rejected( + world: &World, + path: &str, + actor: &str, + value: serde_json::Value, +) { + let (status, body) = post_authed(world, path, actor, value).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{path}: {body}"); + assert_eq!(body["error"], "InvalidRequest", "{path}"); +} diff --git a/knot2/third_party/gix-pack/src/bundle/find.rs b/knot2/third_party/gix-pack/src/bundle/find.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/bundle/find.rs @@ -0,0 +1,71 @@ +use gix_features::zlib; + +impl crate::Bundle { + /// Find an object with the given [`ObjectId`](gix_hash::ObjectId) and place its data into `out`. + /// `inflate` is used to decompress objects, and will be reset before first use, but not after the last use. + /// + /// [`cache`](crate::cache::DecodeEntry) is used to accelerate the lookup. + /// + /// **Note** that ref deltas are automatically resolved within this pack only, which makes this implementation unusable + /// for thin packs, which by now are expected to be resolved already. + pub fn find<'a>( + &self, + id: &gix_hash::oid, + out: &'a mut Vec, + inflate: &mut zlib::Inflate, + cache: &mut dyn crate::cache::DecodeEntry, + ) -> Result, crate::data::entry::Location)>, crate::data::decode::Error> { + let idx = match self.index.lookup(id) { + Some(idx) => idx, + None => return Ok(None), + }; + self.get_object_by_index(idx, out, inflate, cache).map(Some) + } + + /// Special-use function to get an object given an index previously returned from + /// [index::File::](crate::index::File::lookup()). + /// `inflate` is used to decompress objects, and will be reset before first use, but not after the last use. + /// + /// # Panics + /// + /// If `index` is out of bounds. + pub fn get_object_by_index<'a>( + &self, + idx: u32, + out: &'a mut Vec, + inflate: &mut zlib::Inflate, + cache: &mut dyn crate::cache::DecodeEntry, + ) -> Result<(gix_object::Data<'a>, crate::data::entry::Location), crate::data::decode::Error> { + let ofs = self.index.pack_offset_at_index(idx); + let pack_entry = self.pack.entry(ofs)?; + let header_size = pack_entry.header_size(); + self.pack + .decode_entry( + pack_entry, + out, + inflate, + &|id, _out| { + let idx = self.index.lookup(id)?; + self.pack + .entry(self.index.pack_offset_at_index(idx)) + .ok() + .map(crate::data::decode::entry::ResolvedBase::InPack) + }, + cache, + ) + .map(move |r| { + ( + gix_object::Data { + kind: r.kind, + data: out.as_slice(), + object_hash: self.pack.object_hash(), + }, + crate::data::entry::Location { + pack_id: self.pack.id, + pack_offset: ofs, + entry_size: r.compressed_size + header_size, + }, + ) + }) + } +} diff --git a/knot2/third_party/gix-pack/src/bundle/init.rs b/knot2/third_party/gix-pack/src/bundle/init.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/bundle/init.rs @@ -0,0 +1,46 @@ +use std::path::{Path, PathBuf}; + +use crate::Bundle; + +/// Returned by [`Bundle::at()`] +#[derive(thiserror::Error, Debug)] +#[allow(missing_docs)] +pub enum Error { + #[error("An 'idx' extension is expected of an index file: '{0}'")] + InvalidPath(PathBuf), + #[error(transparent)] + Pack(#[from] crate::data::header::decode::Error), + #[error(transparent)] + Index(#[from] crate::index::init::Error), +} + +/// Initialization +impl Bundle { + /// Create a `Bundle` from `path`, which is either a pack file _(*.pack)_ or an index file _(*.idx)_. + /// + /// The corresponding complementary file is expected to be present. + /// + /// The `object_hash` is a way to read (and write) the same file format with different hashes, as the hash kind + /// isn't stored within the file format itself. + pub fn at(path: impl AsRef, object_hash: gix_hash::Kind) -> Result { + Self::at_inner(path.as_ref(), object_hash) + } + + fn at_inner(path: &Path, object_hash: gix_hash::Kind) -> Result { + let ext = path + .extension() + .and_then(std::ffi::OsStr::to_str) + .ok_or_else(|| Error::InvalidPath(path.to_owned()))?; + Ok(match ext { + "idx" => Self { + index: crate::index::File::at(path, object_hash)?, + pack: crate::data::File::at(path.with_extension("pack"), object_hash)?, + }, + "pack" => Self { + pack: crate::data::File::at(path, object_hash)?, + index: crate::index::File::at(path.with_extension("idx"), object_hash)?, + }, + _ => return Err(Error::InvalidPath(path.to_owned())), + }) + } +} diff --git a/knot2/third_party/gix-pack/src/bundle/mod.rs b/knot2/third_party/gix-pack/src/bundle/mod.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/bundle/mod.rs @@ -0,0 +1,56 @@ +/// +pub mod init; + +mod find; +/// +#[cfg(all(not(feature = "wasm"), feature = "streaming-input"))] +pub mod write; + +/// +pub mod verify { + use std::sync::atomic::AtomicBool; + + use gix_features::progress::DynNestedProgress; + + /// + pub mod integrity { + /// Returned by [`Bundle::verify_integrity()`][crate::Bundle::verify_integrity()]. + pub struct Outcome { + /// The computed checksum of the index which matched the stored one. + pub actual_index_checksum: gix_hash::ObjectId, + /// The packs traversal outcome + pub pack_traverse_outcome: crate::index::traverse::Statistics, + } + } + + use crate::Bundle; + + impl Bundle { + /// Similar to [`crate::index::File::verify_integrity()`] but more convenient to call as the presence of the + /// pack file is a given. + pub fn verify_integrity( + &self, + progress: &mut dyn DynNestedProgress, + should_interrupt: &AtomicBool, + options: crate::index::verify::integrity::Options, + ) -> Result> + where + C: crate::cache::DecodeEntry, + F: Fn() -> C + Send + Clone, + { + self.index + .verify_integrity( + Some(crate::index::verify::PackContext { + data: &self.pack, + options, + }), + progress, + should_interrupt, + ) + .map(|o| integrity::Outcome { + actual_index_checksum: o.actual_index_checksum, + pack_traverse_outcome: o.pack_traverse_statistics.expect("pack is set"), + }) + } + } +} diff --git a/knot2/third_party/gix-pack/src/cache/lru.rs b/knot2/third_party/gix-pack/src/cache/lru.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/cache/lru.rs @@ -0,0 +1,241 @@ +use super::DecodeEntry; + +#[cfg(feature = "pack-cache-lru-dynamic")] +mod memory { + use std::num::NonZeroUsize; + + use clru::WeightScale; + + use super::DecodeEntry; + use crate::cache::set_vec_to_slice; + + struct Entry { + data: Vec, + kind: gix_object::Kind, + compressed_size: usize, + } + + type Key = (u32, u64); + struct CustomScale; + + impl WeightScale for CustomScale { + fn weight(&self, _key: &Key, value: &Entry) -> usize { + value.data.len() + } + } + + /// An LRU cache with hash map backing and an eviction rule based on the memory usage for object data in bytes. + pub struct MemoryCappedHashmap { + inner: clru::CLruCache, + free_list: Vec>, + debug: gix_features::cache::Debug, + } + + impl MemoryCappedHashmap { + /// Return a new instance which evicts least recently used items if it uses more than `memory_cap_in_bytes` + /// object data. + pub fn new(memory_cap_in_bytes: usize) -> MemoryCappedHashmap { + MemoryCappedHashmap { + inner: clru::CLruCache::with_config( + clru::CLruCacheConfig::new(NonZeroUsize::new(memory_cap_in_bytes).expect("non zero")) + .with_scale(CustomScale), + ), + free_list: Vec::new(), + debug: gix_features::cache::Debug::new(format!("MemoryCappedHashmap({memory_cap_in_bytes}B)")), + } + } + } + + impl DecodeEntry for MemoryCappedHashmap { + fn put(&mut self, pack_id: u32, offset: u64, data: &[u8], kind: gix_object::Kind, compressed_size: usize) { + self.debug.put(); + let Some(data) = set_vec_to_slice(self.free_list.pop().unwrap_or_default(), data) else { + return; + }; + let res = self.inner.put_with_weight( + (pack_id, offset), + Entry { + data, + kind, + compressed_size, + }, + ); + match res { + Ok(Some(previous_entry)) => self.free_list.push(previous_entry.data), + Ok(None) => {} + Err((_key, value)) => self.free_list.push(value.data), + } + } + + fn get(&mut self, pack_id: u32, offset: u64, out: &mut Vec) -> Option<(gix_object::Kind, usize)> { + let res = self.inner.get(&(pack_id, offset)).and_then(|e| { + set_vec_to_slice(out, &e.data)?; + Some((e.kind, e.compressed_size)) + }); + if res.is_some() { + self.debug.hit(); + } else { + self.debug.miss(); + } + res + } + } +} + +#[cfg(feature = "pack-cache-lru-dynamic")] +pub use memory::MemoryCappedHashmap; + +#[cfg(feature = "pack-cache-lru-static")] +mod _static { + use super::DecodeEntry; + use crate::cache::set_vec_to_slice; + struct Entry { + pack_id: u32, + offset: u64, + data: Vec, + kind: gix_object::Kind, + compressed_size: usize, + } + + /// A cache using a least-recently-used implementation capable of storing the `SIZE` most recent objects. + /// The cache must be small as the search is 'naive' and the underlying data structure is a linked list. + /// Values of 64 seem to improve performance. + pub struct StaticLinkedList { + inner: uluru::LRUCache, + last_evicted: Vec, + debug: gix_features::cache::Debug, + /// the amount of bytes we are currently holding, taking into account the capacities of all Vecs we keep. + mem_used: usize, + /// The total amount of memory we should be able to hold with all entries combined. + mem_limit: usize, + } + + impl StaticLinkedList { + /// Create a new list with a memory limit of `mem_limit` in bytes. If 0, there is no memory limit. + pub fn new(mem_limit: usize) -> Self { + StaticLinkedList { + inner: Default::default(), + last_evicted: Vec::new(), + debug: gix_features::cache::Debug::new(format!("StaticLinkedList<{SIZE}>")), + mem_used: 0, + mem_limit: if mem_limit == 0 { usize::MAX } else { mem_limit }, + } + } + } + + impl Default for StaticLinkedList { + fn default() -> Self { + Self::new(96 * 1024 * 1024) + } + } + + impl DecodeEntry for StaticLinkedList { + fn put(&mut self, pack_id: u32, offset: u64, data: &[u8], kind: gix_object::Kind, compressed_size: usize) { + // We cannot possibly hold this much. + if data.len() > self.mem_limit { + return; + } + // If we could hold it but are at limit, all we can do is make space. + let mem_free = self.mem_limit - self.mem_used; + if data.len() > mem_free { + // prefer freeing free-lists instead of clearing our cache + let free_list_cap = self.last_evicted.len(); + self.last_evicted = Vec::new(); + // still not enough? clear everything + if data.len() > mem_free + free_list_cap { + self.inner.clear(); + self.mem_used = 0; + } else { + self.mem_used -= free_list_cap; + } + } + self.debug.put(); + let mut v = std::mem::take(&mut self.last_evicted); + self.mem_used -= v.capacity(); + if set_vec_to_slice(&mut v, data).is_none() { + return; + } + self.mem_used += v.capacity(); + if let Some(previous) = self.inner.insert(Entry { + offset, + pack_id, + data: v, + kind, + compressed_size, + }) { + // No need to adjust capacity as we already counted it. + self.last_evicted = previous.data; + } + } + + fn get(&mut self, pack_id: u32, offset: u64, out: &mut Vec) -> Option<(gix_object::Kind, usize)> { + let res = self.inner.lookup(|e: &mut Entry| { + if e.pack_id == pack_id && e.offset == offset { + set_vec_to_slice(&mut *out, &e.data)?; + Some((e.kind, e.compressed_size)) + } else { + None + } + }); + if res.is_some() { + self.debug.hit(); + } else { + self.debug.miss(); + } + res + } + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn no_limit() { + let c = StaticLinkedList::<10>::new(0); + assert_eq!( + c.mem_limit, + usize::MAX, + "zero is automatically turned into a large limit that is equivalent to unlimited" + ); + } + + #[test] + fn journey() { + let mut c = StaticLinkedList::<10>::new(100); + assert_eq!(c.mem_limit, 100); + assert_eq!(c.mem_used, 0); + + // enough memory for normal operation + let mut last_mem_used = 0; + for _ in 0..10 { + c.put(0, 0, &[0], gix_object::Kind::Blob, 1); + assert!(c.mem_used > last_mem_used); + last_mem_used = c.mem_used; + } + assert_eq!(c.mem_used, 80, "there is a minimal vec size"); + assert_eq!(c.inner.len(), 10); + assert_eq!(c.last_evicted.len(), 0); + + c.put(0, 0, &(0..20).collect::>(), gix_object::Kind::Blob, 1); + assert_eq!(c.inner.len(), 10); + assert_eq!(c.mem_used, 80 + 20); + assert_eq!(c.last_evicted.len(), 1); + + c.put(0, 0, &(0..50).collect::>(), gix_object::Kind::Blob, 1); + assert_eq!(c.inner.len(), 1, "cache clearance wasn't necessary"); + assert_eq!(c.last_evicted.len(), 0, "the free list was cleared"); + assert_eq!(c.mem_used, 50); + + c.put(0, 0, &(0..101).collect::>(), gix_object::Kind::Blob, 1); + assert_eq!( + c.inner.len(), + 1, + "objects that won't ever fit within the memory limit are ignored" + ); + } + } +} + +#[cfg(feature = "pack-cache-lru-static")] +pub use _static::StaticLinkedList; diff --git a/knot2/third_party/gix-pack/src/cache/mod.rs b/knot2/third_party/gix-pack/src/cache/mod.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/cache/mod.rs @@ -0,0 +1,71 @@ +use std::ops::DerefMut; + +use gix_object::Kind; + +/// A trait to model putting objects at a given pack `offset` into a cache, and fetching them. +/// +/// It is used to speed up [pack traversals][crate::index::File::traverse()]. +pub trait DecodeEntry { + /// Store a fully decoded object at `offset` of `kind` with `compressed_size` and `data` in the cache. + /// + /// It is up to the cache implementation whether that actually happens or not. + fn put(&mut self, pack_id: u32, offset: u64, data: &[u8], kind: gix_object::Kind, compressed_size: usize); + /// Attempt to fetch the object at `offset` and store its decoded bytes in `out`, as previously stored with [`DecodeEntry::put()`], and return + /// its (object `kind`, `decompressed_size`) + fn get(&mut self, pack_id: u32, offset: u64, out: &mut Vec) -> Option<(gix_object::Kind, usize)>; +} + +/// A cache that stores nothing and retrieves nothing, thus it _never_ caches. +#[derive(Default)] +pub struct Never; + +impl DecodeEntry for Never { + fn put(&mut self, _pack_id: u32, _offset: u64, _data: &[u8], _kind: gix_object::Kind, _compressed_size: usize) {} + fn get(&mut self, _pack_id: u32, _offset: u64, _out: &mut Vec) -> Option<(gix_object::Kind, usize)> { + None + } +} + +impl DecodeEntry for Box { + fn put(&mut self, pack_id: u32, offset: u64, data: &[u8], kind: Kind, compressed_size: usize) { + self.deref_mut().put(pack_id, offset, data, kind, compressed_size); + } + + fn get(&mut self, pack_id: u32, offset: u64, out: &mut Vec) -> Option<(Kind, usize)> { + self.deref_mut().get(pack_id, offset, out) + } +} + +/// A way of storing and retrieving entire objects to and from a cache. +pub trait Object { + /// Put the object going by `id` of `kind` with `data` into the cache. + fn put(&mut self, id: gix_hash::ObjectId, kind: gix_object::Kind, data: &[u8]); + + /// Try to retrieve the object named `id` and place its data into `out` if available and return `Some(kind)` if found. + fn get(&mut self, id: &gix_hash::ObjectId, out: &mut Vec) -> Option; +} + +/// Various implementations of [`DecodeEntry`] using least-recently-used algorithms. +#[cfg(any(feature = "pack-cache-lru-dynamic", feature = "pack-cache-lru-static"))] +pub mod lru; + +pub mod object; + +/// +pub mod delta; + +/// Replaces content of the given `Vec` with the slice. The vec will have the same length +/// as the slice. The vec can be either `&mut Vec` or `Vec`. +/// Returns `None` if no memory could be allocated. +#[cfg(any( + feature = "pack-cache-lru-static", + feature = "pack-cache-lru-dynamic", + feature = "object-cache-dynamic" +))] +fn set_vec_to_slice>>(mut vec: V, source: &[u8]) -> Option { + let out = vec.borrow_mut(); + out.clear(); + out.try_reserve(source.len()).ok()?; + out.extend_from_slice(source); + Some(vec) +} diff --git a/knot2/third_party/gix-pack/src/cache/object.rs b/knot2/third_party/gix-pack/src/cache/object.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/cache/object.rs @@ -0,0 +1,111 @@ +//! This module is a bit 'misplaced' if spelled out like '`gix_pack::cache::object::`*' but is best placed here for code reuse and +//! general usefulness. +use crate::cache; + +#[cfg(feature = "object-cache-dynamic")] +mod memory { + use std::num::NonZeroUsize; + + use clru::WeightScale; + + use crate::{cache, cache::set_vec_to_slice}; + + struct Entry { + data: Vec, + kind: gix_object::Kind, + } + + type Key = gix_hash::ObjectId; + + struct CustomScale; + + impl WeightScale for CustomScale { + fn weight(&self, key: &Key, value: &Entry) -> usize { + value.data.len() + std::mem::size_of::() + key.as_bytes().len() + } + } + + /// An LRU cache with hash map backing and an eviction rule based on the memory usage for object data in bytes. + pub struct MemoryCappedHashmap { + inner: clru::CLruCache, + free_list: Vec>, + debug: gix_features::cache::Debug, + } + + impl MemoryCappedHashmap { + /// The amount of bytes we can hold in total, or the value we saw in `new(…)`. + pub fn capacity(&self) -> usize { + self.inner.capacity() + } + /// Return a new instance which evicts least recently used items if it uses more than `memory_cap_in_bytes` + /// object data. + pub fn new(memory_cap_in_bytes: usize) -> MemoryCappedHashmap { + MemoryCappedHashmap { + inner: clru::CLruCache::with_config( + clru::CLruCacheConfig::new(NonZeroUsize::new(memory_cap_in_bytes).expect("non zero")) + .with_hasher(gix_hashtable::hash::Builder) + .with_scale(CustomScale), + ), + free_list: Vec::new(), + debug: gix_features::cache::Debug::new(format!("MemoryCappedObjectHashmap({memory_cap_in_bytes}B)")), + } + } + } + + impl cache::Object for MemoryCappedHashmap { + /// Put the object going by `id` of `kind` with `data` into the cache. + fn put(&mut self, id: gix_hash::ObjectId, kind: gix_object::Kind, data: &[u8]) { + self.debug.put(); + let Some(data) = set_vec_to_slice(self.free_list.pop().unwrap_or_default(), data) else { + return; + }; + let res = self.inner.put_with_weight(id, Entry { data, kind }); + match res { + Ok(Some(previous_entry)) => self.free_list.push(previous_entry.data), + Ok(None) => {} + Err((_key, value)) => self.free_list.push(value.data), + } + } + + /// Try to retrieve the object named `id` and place its data into `out` if available and return `Some(kind)` if found. + fn get(&mut self, id: &gix_hash::ObjectId, out: &mut Vec) -> Option { + let res = self.inner.get(id).and_then(|e| { + set_vec_to_slice(out, &e.data)?; + Some(e.kind) + }); + if res.is_some() { + self.debug.hit(); + } else { + self.debug.miss(); + } + res + } + } +} +#[cfg(feature = "object-cache-dynamic")] +pub use memory::MemoryCappedHashmap; + +/// A cache implementation that doesn't do any caching. +pub struct Never; + +impl cache::Object for Never { + /// Noop + fn put(&mut self, _id: gix_hash::ObjectId, _kind: gix_object::Kind, _data: &[u8]) {} + + /// Noop + fn get(&mut self, _id: &gix_hash::ObjectId, _out: &mut Vec) -> Option { + None + } +} + +impl cache::Object for Box { + fn put(&mut self, id: gix_hash::ObjectId, kind: gix_object::Kind, data: &[u8]) { + use std::ops::DerefMut; + self.deref_mut().put(id, kind, data); + } + + fn get(&mut self, id: &gix_hash::ObjectId, out: &mut Vec) -> Option { + use std::ops::DerefMut; + self.deref_mut().get(id, out) + } +} diff --git a/knot2/third_party/gix-pack/src/data/delta.rs b/knot2/third_party/gix-pack/src/data/delta.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/delta.rs @@ -0,0 +1,125 @@ +/// +pub mod apply { + /// Returned when failing to apply deltas. + #[derive(thiserror::Error, Debug)] + #[allow(missing_docs)] + pub enum Error { + #[error("Corrupt delta data: {message}")] + Corrupt { message: &'static str }, + #[error("Encountered unsupported command code: 0")] + UnsupportedCommandCode, + #[error("Delta copy from base: byte slices must match")] + DeltaCopyBaseSliceMismatch, + #[error("Delta copy data: byte slices must match")] + DeltaCopyDataSliceMismatch, + } +} + +/// Given the decompressed pack delta `d`, decode a size in bytes (either the base object size or the result object size) +/// Equivalent to [this canonical git function](https://github.com/git/git/blob/311531c9de557d25ac087c1637818bd2aad6eb3a/delta.h#L89) +pub(crate) fn decode_header_size(d: &[u8]) -> Result<(u64, usize), apply::Error> { + let mut shift = 0; + let mut size = 0u64; + let mut consumed = 0; + for cmd in d.iter() { + if shift >= u64::BITS { + return Err(apply::Error::Corrupt { + message: "delta header size uses more bits than fit into u64", + }); + } + consumed += 1; + size |= (u64::from(*cmd) & 0x7f) << shift; + shift += 7; + if *cmd & 0x80 == 0 { + return Ok((size, consumed)); + } + } + Err(apply::Error::Corrupt { + message: "delta header size is truncated", + }) +} + +pub(crate) fn apply(base: &[u8], mut target: &mut [u8], data: &[u8]) -> Result<(), apply::Error> { + fn next_byte(data: &[u8], i: &mut usize) -> Result { + let byte = *data.get(*i).ok_or(apply::Error::Corrupt { + message: "delta copy instruction is truncated", + })?; + *i += 1; + Ok(byte) + } + + let mut i = 0; + while let Some(cmd) = data.get(i) { + i += 1; + match cmd { + cmd if cmd & 0b1000_0000 != 0 => { + let (mut ofs, mut size): (u32, u32) = (0, 0); + if cmd & 0b0000_0001 != 0 { + ofs = u32::from(next_byte(data, &mut i)?); + } + if cmd & 0b0000_0010 != 0 { + ofs |= u32::from(next_byte(data, &mut i)?) << 8; + } + if cmd & 0b0000_0100 != 0 { + ofs |= u32::from(next_byte(data, &mut i)?) << 16; + } + if cmd & 0b0000_1000 != 0 { + ofs |= u32::from(next_byte(data, &mut i)?) << 24; + } + if cmd & 0b0001_0000 != 0 { + size = u32::from(next_byte(data, &mut i)?); + } + if cmd & 0b0010_0000 != 0 { + size |= u32::from(next_byte(data, &mut i)?) << 8; + } + if cmd & 0b0100_0000 != 0 { + size |= u32::from(next_byte(data, &mut i)?) << 16; + } + if size == 0 { + size = 0x10000; // 65536 + } + let ofs = ofs as usize; + let end = ofs.checked_add(size as usize).ok_or(apply::Error::Corrupt { + message: "delta copy range overflows", + })?; + std::io::Write::write( + &mut target, + base.get(ofs..end).ok_or(apply::Error::Corrupt { + message: "delta copy range exceeds base object size", + })?, + ) + .map_err(|_e| apply::Error::DeltaCopyBaseSliceMismatch)?; + } + 0 => { + return Err(apply::Error::Corrupt { + message: "delta command 0 is reserved and invalid", + }); + } + size => { + let end = i.checked_add(*size as usize).ok_or(apply::Error::Corrupt { + message: "delta insert range overflows", + })?; + std::io::Write::write( + &mut target, + data.get(i..end).ok_or(apply::Error::Corrupt { + message: "delta insert data is truncated", + })?, + ) + .map_err(|_e| apply::Error::DeltaCopyDataSliceMismatch)?; + i = end; + } + } + } + debug_assert_eq!( + i, + data.len(), + "delta instructions were not consumed completely, should be impossible" + ); + if !target.is_empty() { + return Err(apply::Error::Corrupt { + message: "delta instructions produced fewer bytes than promised", + }); + } + + Ok(()) +} diff --git a/knot2/third_party/gix-pack/src/data/header.rs b/knot2/third_party/gix-pack/src/data/header.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/header.rs @@ -0,0 +1,55 @@ +use crate::data; + +pub(crate) const N32_SIZE: usize = std::mem::size_of::(); + +/// Parses the first 12 bytes of a pack file, returning the pack version as well as the number of objects contained in the pack. +pub fn decode(data: &[u8; 12]) -> Result<(data::Version, u32), decode::Error> { + let mut ofs = 0; + if &data[ofs..ofs + b"PACK".len()] != b"PACK" { + return Err(decode::Error::Corrupt("Pack data type not recognized".into())); + } + ofs += N32_SIZE; + let kind = match crate::read_u32(&data[ofs..ofs + N32_SIZE]) { + 2 => data::Version::V2, + 3 => data::Version::V3, + v => return Err(decode::Error::UnsupportedVersion(v)), + }; + ofs += N32_SIZE; + let num_objects = crate::read_u32(&data[ofs..ofs + N32_SIZE]); + + Ok((kind, num_objects)) +} + +/// Write a pack data header at `version` with `num_objects` and return a buffer. +pub fn encode(version: data::Version, num_objects: u32) -> [u8; 12] { + use crate::data::Version::*; + let mut buf = [0u8; 12]; + buf[..4].copy_from_slice(b"PACK"); + buf[4..8].copy_from_slice( + &match version { + V2 => 2u32, + V3 => 3, + } + .to_be_bytes()[..], + ); + buf[8..].copy_from_slice(&num_objects.to_be_bytes()[..]); + buf +} + +/// +pub mod decode { + /// Returned by [`decode()`][super::decode()]. + #[derive(thiserror::Error, Debug)] + #[allow(missing_docs)] + pub enum Error { + #[error("Could not open pack file at '{path}'")] + Io { + source: std::io::Error, + path: std::path::PathBuf, + }, + #[error("{0}")] + Corrupt(String), + #[error("Unsupported pack version: {0}")] + UnsupportedVersion(u32), + } +} diff --git a/knot2/third_party/gix-pack/src/data/mod.rs b/knot2/third_party/gix-pack/src/data/mod.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/mod.rs @@ -0,0 +1,170 @@ +//! a pack data file +use std::path::Path; + +/// The offset to an entry into the pack data file, relative to its beginning. +pub type Offset = u64; + +/// An identifier to uniquely identify all packs loaded within a known context or namespace. +pub type Id = u32; + +/// An representing an full- or delta-object within a pack +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Entry { + /// The entry's header + pub header: entry::Header, + /// The decompressed size of the entry in bytes. + /// + /// Note that for non-delta entries this will be the size of the object itself. + pub decompressed_size: u64, + /// absolute offset to compressed object data in the pack, just behind the entry's header + pub data_offset: Offset, +} + +mod file; +pub use file::{Header, decode, verify}; +/// +pub mod header; + +/// +pub mod init { + pub use super::header::decode::Error; +} + +/// +pub mod entry; + +/// +#[cfg(feature = "streaming-input")] +pub mod input; + +/// Utilities to encode pack data entries and write them to a `Write` implementation to resemble a pack data file. +#[cfg(feature = "generate")] +pub mod output; + +/// A slice into a pack file denoting a pack entry. +/// +/// An entry can be decoded into an object. +pub type EntryRange = std::ops::Range; + +/// Supported versions of a pack data file +#[derive(Default, PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum Version { + /// The default pack data version. + /// + /// This is the version generated by Git and by `gix-pack` writers. + #[default] + V2, + /// A pack data version accepted by Git and recognized by `gix-pack` readers. + /// + /// Git does not generate this version, and `gix-pack` writers currently reject it. + /// Entries are decoded with the same layout as [`V2`](Version::V2); the difference + /// visible to this crate is the version number stored in the pack header. + V3, +} + +/// A pack data file, read from disk on demand rather than held in memory. +#[allow(missing_docs)] +pub struct File { + file: std::fs::File, + len: usize, + path: std::path::PathBuf, + pub id: Id, + version: Version, + num_objects: u32, + hash_len: usize, + object_hash: gix_hash::Kind, + alloc_limit_bytes: Option, +} + +/// Information about the pack data file itself +impl File { + /// The pack data version of this file + pub fn version(&self) -> Version { + self.version + } + /// The number of objects stored in this pack data file + pub fn num_objects(&self) -> u32 { + self.num_objects + } + /// The length of all pack data, including the pack header and the pack trailer + pub fn data_len(&self) -> usize { + self.len + } + /// The kind of hash we use internally. + pub fn object_hash(&self) -> gix_hash::Kind { + self.object_hash + } + /// The maximum size of a single allocation caused by user-controlled on-disk pack data. + /// + /// A value of `None` means no additional limit is enforced. + pub fn alloc_limit_bytes(&self) -> Option { + self.alloc_limit_bytes + } + /// The position of the byte one past the last pack entry, or in other terms, the first byte of the trailing hash. + pub fn pack_end(&self) -> usize { + self.len - self.hash_len + } + + /// The path to the pack data file on disk + pub fn path(&self) -> &Path { + &self.path + } + + pub(crate) fn read_exact_at(&self, offset: usize, buf: &mut [u8]) -> std::io::Result<()> { + use std::os::unix::fs::FileExt; + self.file.read_exact_at(buf, offset as u64) + } + + pub(crate) fn read_span(&self, slice: EntryRange) -> Option> { + let start = usize::try_from(slice.start).ok()?; + let end = usize::try_from(slice.end).ok()?; + if start > end || end > self.len { + return None; + } + let mut buf = vec![0u8; end - start]; + self.read_exact_at(start, &mut buf).ok()?; + Some(buf) + } + + #[allow(missing_docs)] + pub fn read_into(&self, slice: EntryRange, buf: &mut Vec) -> bool { + let (Ok(start), Ok(end)) = (usize::try_from(slice.start), usize::try_from(slice.end)) else { + return false; + }; + if start > end || end > self.len { + return false; + } + buf.clear(); + buf.resize(end - start, 0); + self.read_exact_at(start, buf).is_ok() + } + + pub(crate) fn materialized(&self) -> std::io::Result { + crate::MMap::map(&self.file) + } + + /// Returns the pack data at the given slice if its range is contained in the pack data. + pub fn entry_slice(&self, slice: EntryRange) -> Option> { + self.read_span(slice) + } + + /// Returns the CRC32 of the pack data indicated by `pack_offset` and the `size` of the data. + /// + /// _Note:_ finding the right size is only possible by decompressing + /// the pack entry beforehand, or by using the (to be sorted) offsets stored in an index file. + /// + /// # Panics + /// + /// If `pack_offset` or `size` are pointing to a range outside of the pack data. + pub fn entry_crc32(&self, pack_offset: Offset, size: usize) -> u32 { + let buf = self + .read_span(pack_offset..pack_offset + size as u64) + .expect("entry range within pack data"); + gix_features::hash::crc32(&buf) + } +} + +/// +pub mod delta; diff --git a/knot2/third_party/gix-pack/src/index/access.rs b/knot2/third_party/gix-pack/src/index/access.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/index/access.rs @@ -0,0 +1,297 @@ +use std::{mem::size_of, ops::Range}; + +use crate::{ + data, + index::{self, EntryIndex, FAN_LEN, PrefixLookupResult}, +}; + +const N32_SIZE: usize = size_of::(); +const N64_SIZE: usize = size_of::(); +const V1_HEADER_SIZE: usize = FAN_LEN * N32_SIZE; +const V2_HEADER_SIZE: usize = N32_SIZE * 2 + FAN_LEN * N32_SIZE; +const N32_HIGH_BIT: u32 = 1 << 31; + +/// Represents an entry within a pack index file, effectively mapping object [`IDs`][gix_hash::ObjectId] to pack data file locations. +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Entry { + /// The ID of the object + pub oid: gix_hash::ObjectId, + /// The offset to the object's header in the pack data file + pub pack_offset: data::Offset, + /// The CRC32 hash over all bytes of the pack data entry. + /// + /// This can be useful for direct copies of pack data entries from one pack to another with insurance there was no bit rot. + /// _Note_: Only available in index version 2 or newer + pub crc32: Option, +} + +/// Iteration and access +impl index::File +where + T: crate::FileData, +{ + fn iter_v1(&self) -> impl Iterator + '_ { + match self.version { + index::Version::V1 => self.data[V1_HEADER_SIZE..] + .chunks_exact(N32_SIZE + self.hash_len) + .take(self.num_objects as usize) + .map(|c| { + let (ofs, oid) = c.split_at(N32_SIZE); + Entry { + oid: gix_hash::ObjectId::from_bytes_or_panic(oid), + pack_offset: u64::from(crate::read_u32(ofs)), + crc32: None, + } + }), + _ => panic!("Cannot use iter_v1() on index of type {:?}", self.version), + } + } + + fn iter_v2(&self) -> impl Iterator + '_ { + let pack64_offset = self.offset_pack_offset64_v2(); + let oids = self.data[V2_HEADER_SIZE..] + .chunks_exact(self.hash_len) + .take(self.num_objects as usize); + let crcs = self.data[self.offset_crc32_v2()..] + .chunks_exact(N32_SIZE) + .take(self.num_objects as usize); + let offsets = self.data[self.offset_pack_offset_v2()..] + .chunks_exact(N32_SIZE) + .take(self.num_objects as usize); + assert_eq!(oids.len(), crcs.len()); + assert_eq!(crcs.len(), offsets.len()); + match self.version { + index::Version::V2 => izip!(oids, crcs, offsets).map(move |(oid, crc32, ofs32)| Entry { + oid: gix_hash::ObjectId::from_bytes_or_panic(oid), + pack_offset: self.pack_offset_from_offset_v2(ofs32, pack64_offset), + crc32: Some(crate::read_u32(crc32)), + }), + _ => panic!("Cannot use iter_v2() on index of type {:?}", self.version), + } + } + + /// Returns the object hash at the given index in our list of (sorted) sha1 hashes. + /// The index ranges from 0 to `self.num_objects()` + /// + /// # Panics + /// + /// If `index` is out of bounds. + pub fn oid_at_index(&self, index: EntryIndex) -> &gix_hash::oid { + let index = index as usize; + let start = match self.version { + index::Version::V2 => V2_HEADER_SIZE + index * self.hash_len, + index::Version::V1 => V1_HEADER_SIZE + index * (N32_SIZE + self.hash_len) + N32_SIZE, + }; + gix_hash::oid::from_bytes_unchecked(&self.data[start..][..self.hash_len]) + } + + /// Returns the offset into our pack data file at which to start reading the object at `index`. + /// + /// # Panics + /// + /// If `index` is out of bounds. + pub fn pack_offset_at_index(&self, index: EntryIndex) -> data::Offset { + let index = index as usize; + match self.version { + index::Version::V2 => { + let start = self.offset_pack_offset_v2() + index * N32_SIZE; + self.pack_offset_from_offset_v2(&self.data[start..][..N32_SIZE], self.offset_pack_offset64_v2()) + } + index::Version::V1 => { + let start = V1_HEADER_SIZE + index * (N32_SIZE + self.hash_len); + u64::from(crate::read_u32(&self.data[start..][..N32_SIZE])) + } + } + } + + /// Returns the CRC32 of the object at the given `index`. + /// + /// _Note_: These are always present for index version 2 or higher. + /// # Panics + /// + /// If `index` is out of bounds. + pub fn crc32_at_index(&self, index: EntryIndex) -> Option { + let index = index as usize; + match self.version { + index::Version::V2 => { + let start = self.offset_crc32_v2() + index * N32_SIZE; + Some(crate::read_u32(&self.data[start..start + N32_SIZE])) + } + index::Version::V1 => None, + } + } + + /// Returns the `index` of the given hash for use with the [`oid_at_index()`][index::File::oid_at_index()], + /// [`pack_offset_at_index()`][index::File::pack_offset_at_index()] or [`crc32_at_index()`][index::File::crc32_at_index()]. + // NOTE: pretty much the same things as in `multi_index::File::lookup`, change things there + // as well. + pub fn lookup(&self, id: impl AsRef) -> Option { + lookup(id.as_ref(), &self.fan, &|idx| self.oid_at_index(idx)) + } + + /// Given a `prefix`, find an object that matches it uniquely within this index and return `Some(Ok(entry_index))`. + /// If there is more than one object matching the object `Some(Err(())` is returned. + /// + /// Finally, if no object matches the index, the return value is `None`. + /// + /// Pass `candidates` to obtain the set of entry-indices matching `prefix`, with the same return value as + /// one would have received if it remained `None`. It will be empty if no object matched the `prefix`. + /// + // NOTE: pretty much the same things as in `index::File::lookup`, change things there + // as well. + pub fn lookup_prefix( + &self, + prefix: gix_hash::Prefix, + candidates: Option<&mut Range>, + ) -> Option { + lookup_prefix( + prefix, + candidates, + &self.fan, + &|idx| self.oid_at_index(idx), + self.num_objects, + ) + } + + /// An iterator over all [`Entries`][Entry] of this index file. + pub fn iter<'a>(&'a self) -> Box + 'a> { + match self.version { + index::Version::V2 => Box::new(self.iter_v2()), + index::Version::V1 => Box::new(self.iter_v1()), + } + } + + /// Return a vector of ascending offsets into our respective pack data file. + /// + /// Useful to control an iteration over all pack entries in a cache-friendly way. + pub fn sorted_offsets(&self) -> Vec { + let mut ofs: Vec<_> = match self.version { + index::Version::V1 => self.iter().map(|e| e.pack_offset).collect(), + index::Version::V2 => { + let offset32_start = &self.data[self.offset_pack_offset_v2()..]; + let offsets32 = offset32_start.chunks_exact(N32_SIZE).take(self.num_objects as usize); + assert_eq!(self.num_objects as usize, offsets32.len()); + let pack_offset_64_start = self.offset_pack_offset64_v2(); + offsets32 + .map(|offset| self.pack_offset_from_offset_v2(offset, pack_offset_64_start)) + .collect() + } + }; + ofs.sort_unstable(); + ofs + } + + #[inline] + fn offset_crc32_v2(&self) -> usize { + V2_HEADER_SIZE + self.num_objects as usize * self.hash_len + } + + #[inline] + fn offset_pack_offset_v2(&self) -> usize { + self.offset_crc32_v2() + self.num_objects as usize * N32_SIZE + } + + #[inline] + fn offset_pack_offset64_v2(&self) -> usize { + self.offset_pack_offset_v2() + self.num_objects as usize * N32_SIZE + } + + #[inline] + fn pack_offset_from_offset_v2(&self, offset: &[u8], pack64_offset: usize) -> data::Offset { + debug_assert_eq!(self.version, index::Version::V2); + let ofs32 = crate::read_u32(offset); + if (ofs32 & N32_HIGH_BIT) == N32_HIGH_BIT { + let from = pack64_offset + (ofs32 ^ N32_HIGH_BIT) as usize * N64_SIZE; + crate::read_u64(&self.data[from..][..N64_SIZE]) + } else { + u64::from(ofs32) + } + } +} + +pub(crate) fn lookup_prefix<'a>( + prefix: gix_hash::Prefix, + candidates: Option<&mut Range>, + fan: &[u32; FAN_LEN], + oid_at_index: &dyn Fn(EntryIndex) -> &'a gix_hash::oid, + num_objects: u32, +) -> Option { + let first_byte = prefix.as_oid().first_byte() as usize; + let mut upper_bound = fan[first_byte]; + let mut lower_bound = if first_byte != 0 { fan[first_byte - 1] } else { 0 }; + + // Bisect using indices + while lower_bound < upper_bound { + let mid = u32::midpoint(lower_bound, upper_bound); + let mid_sha = oid_at_index(mid); + + use std::cmp::Ordering::*; + match prefix.cmp_oid(mid_sha) { + Less => upper_bound = mid, + Equal => match candidates { + Some(candidates) => { + let first_past_entry = ((0..mid).rev()) + .take_while(|prev| prefix.cmp_oid(oid_at_index(*prev)) == Equal) + .last(); + + let last_future_entry = ((mid + 1)..num_objects) + .take_while(|next| prefix.cmp_oid(oid_at_index(*next)) == Equal) + .last(); + + *candidates = match (first_past_entry, last_future_entry) { + (Some(first), Some(last)) => first..last + 1, + (Some(first), None) => first..mid + 1, + (None, Some(last)) => mid..last + 1, + (None, None) => mid..mid + 1, + }; + + return if candidates.len() > 1 { + Some(Err(())) + } else { + Some(Ok(mid)) + }; + } + None => { + let next = mid + 1; + if next < num_objects && prefix.cmp_oid(oid_at_index(next)) == Equal { + return Some(Err(())); + } + if mid != 0 && prefix.cmp_oid(oid_at_index(mid - 1)) == Equal { + return Some(Err(())); + } + return Some(Ok(mid)); + } + }, + Greater => lower_bound = mid + 1, + } + } + + if let Some(candidates) = candidates { + *candidates = 0..0; + } + None +} + +pub(crate) fn lookup<'a>( + id: &gix_hash::oid, + fan: &[u32; FAN_LEN], + oid_at_index: &dyn Fn(EntryIndex) -> &'a gix_hash::oid, +) -> Option { + let first_byte = id.first_byte() as usize; + let mut upper_bound = fan[first_byte]; + let mut lower_bound = if first_byte != 0 { fan[first_byte - 1] } else { 0 }; + + while lower_bound < upper_bound { + let mid = u32::midpoint(lower_bound, upper_bound); + let mid_sha = oid_at_index(mid); + + use std::cmp::Ordering::*; + match id.cmp(mid_sha) { + Less => upper_bound = mid, + Equal => return Some(mid), + Greater => lower_bound = mid + 1, + } + } + None +} diff --git a/knot2/third_party/gix-pack/src/index/encode.rs b/knot2/third_party/gix-pack/src/index/encode.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/index/encode.rs @@ -0,0 +1,156 @@ +use std::cmp::Ordering; + +pub(crate) const LARGE_OFFSET_THRESHOLD: u64 = 0x7fff_ffff; +pub(crate) const HIGH_BIT: u32 = 0x8000_0000; + +pub(crate) fn fanout(iter: &mut dyn ExactSizeIterator) -> [u32; 256] { + let mut fan_out = [0u32; 256]; + let entries_len = iter.len() as u32; + let mut iter = iter.enumerate(); + let mut idx_and_entry = iter.next(); + let mut upper_bound = 0; + + for (offset_be, byte) in fan_out.iter_mut().zip(0u8..=255) { + *offset_be = match idx_and_entry.as_ref() { + Some((_idx, first_byte)) => match first_byte.cmp(&byte) { + Ordering::Less => unreachable!("ids should be ordered, and we make sure to keep ahead with them"), + Ordering::Greater => upper_bound, + Ordering::Equal => { + if byte == 255 { + entries_len + } else { + idx_and_entry = iter.find(|(_, first_byte)| *first_byte != byte); + upper_bound = idx_and_entry.as_ref().map_or(entries_len, |(idx, _)| *idx as u32); + upper_bound + } + } + }, + None => entries_len, + }; + } + + fan_out +} + +#[cfg(feature = "streaming-input")] +mod function { + use std::io; + + use gix_features::progress::{self, DynNestedProgress}; + + use super::{HIGH_BIT, LARGE_OFFSET_THRESHOLD, fanout}; + use crate::index::V2_SIGNATURE; + + struct Count { + bytes: u64, + inner: W, + } + + impl Count { + fn new(inner: W) -> Self { + Count { bytes: 0, inner } + } + } + + impl io::Write for Count + where + W: io::Write, + { + fn write(&mut self, buf: &[u8]) -> io::Result { + let written = self.inner.write(buf)?; + self.bytes += written as u64; + Ok(written) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } + } + + pub(crate) fn write_to( + out: &mut dyn io::Write, + entries_sorted_by_oid: Vec>, + pack_hash: &gix_hash::ObjectId, + kind: crate::index::Version, + object_hash: gix_hash::Kind, + progress: &mut dyn DynNestedProgress, + ) -> Result { + use io::Write; + assert_eq!(kind, crate::index::Version::V2, "Can only write V2 packs right now"); + assert!( + entries_sorted_by_oid.len() <= u32::MAX as usize, + "a pack cannot have more than u32::MAX objects" + ); + + // Write header + let mut out = Count::new(std::io::BufWriter::with_capacity( + 8 * 4096, + gix_hash::io::Write::new(out, object_hash), + )); + out.write_all(V2_SIGNATURE)?; + out.write_all(&(kind as u32).to_be_bytes())?; + + progress.init(Some(4), progress::steps()); + let start = std::time::Instant::now(); + let _info = progress.add_child_with_id("writing fan-out table".into(), gix_features::progress::UNKNOWN); + let fan_out = fanout(&mut entries_sorted_by_oid.iter().map(|e| e.data.id.first_byte())); + + for value in fan_out.iter() { + out.write_all(&value.to_be_bytes())?; + } + + progress.inc(); + let _info = progress.add_child_with_id("writing ids".into(), gix_features::progress::UNKNOWN); + for entry in &entries_sorted_by_oid { + out.write_all(entry.data.id.as_slice())?; + } + + progress.inc(); + let _info = progress.add_child_with_id("writing crc32".into(), gix_features::progress::UNKNOWN); + for entry in &entries_sorted_by_oid { + out.write_all(&entry.data.crc32.to_be_bytes())?; + } + + progress.inc(); + let _info = progress.add_child_with_id("writing offsets".into(), gix_features::progress::UNKNOWN); + { + let mut offsets64 = Vec::::new(); + for entry in &entries_sorted_by_oid { + let offset: u32 = if entry.offset > LARGE_OFFSET_THRESHOLD { + assert!( + offsets64.len() < LARGE_OFFSET_THRESHOLD as usize, + "Encoding breakdown - way too many 64bit offsets" + ); + offsets64.push(entry.offset); + ((offsets64.len() - 1) as u32) | HIGH_BIT + } else { + entry.offset as u32 + }; + out.write_all(&offset.to_be_bytes())?; + } + for value in offsets64 { + out.write_all(&value.to_be_bytes())?; + } + } + + out.write_all(pack_hash.as_slice())?; + + let bytes_written_without_trailer = out.bytes; + let out = out.inner.into_inner().map_err(io::Error::from)?; + let index_hash = out.hash.try_finalize()?; + out.inner.write_all(index_hash.as_slice())?; + out.inner.flush()?; + + progress.inc(); + progress.show_throughput_with( + start, + (bytes_written_without_trailer + object_hash.len_in_bytes() as u64) as usize, + progress::bytes().expect("unit always set"), + progress::MessageLevel::Success, + ); + + Ok(index_hash) + } +} +#[cfg(feature = "streaming-input")] +pub(crate) use function::write_to; diff --git a/knot2/third_party/gix-pack/src/index/init.rs b/knot2/third_party/gix-pack/src/index/init.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/index/init.rs @@ -0,0 +1,197 @@ +use std::{ + mem::size_of, + path::{Path, PathBuf}, +}; + +use crate::index::{self, FAN_LEN, V2_SIGNATURE, Version}; + +/// Returned by [`index::File::at()`]. +#[derive(thiserror::Error, Debug)] +#[allow(missing_docs)] +pub enum Error { + #[error("Could not open pack index file at '{path}'")] + Io { + source: std::io::Error, + path: std::path::PathBuf, + }, + #[error("{message}")] + Corrupt { message: String }, + #[error("Unsupported index version: {version})")] + UnsupportedVersion { version: u32 }, +} + +const N32_SIZE: usize = size_of::(); + +/// Instantiation +impl index::File { + /// Open the pack index file at the given `path`. + /// + /// The `object_hash` is a way to read (and write) the same file format with different hashes, as the hash kind + /// isn't stored within the file format itself. + pub fn at(path: impl AsRef, object_hash: gix_hash::Kind) -> Result { + Self::at_inner(path.as_ref(), object_hash) + } + + fn at_inner(path: &Path, object_hash: gix_hash::Kind) -> Result { + let data = crate::mmap::read_only(path).map_err(|source| Error::Io { + source, + path: path.to_owned(), + })?; + Self::from_data(data, path.to_owned(), object_hash) + } +} + +impl index::File +where + T: crate::FileData, +{ + /// Instantiate an index file from `data` as assumed to be read or memory-mapped from `path`. + pub fn from_data(data: T, path: PathBuf, object_hash: gix_hash::Kind) -> Result { + let idx_len = data.len(); + let hash_len = object_hash.len_in_bytes(); + + let footer_size = hash_len * 2; + if idx_len < FAN_LEN * N32_SIZE + footer_size { + return Err(Error::Corrupt { + message: format!("Pack index of size {idx_len} is too small for even an empty index"), + }); + } + let (kind, fan, num_objects) = { + let (kind, d) = { + let (sig, d) = data.split_at(V2_SIGNATURE.len()); + if sig == V2_SIGNATURE { + (Version::V2, d) + } else { + (Version::V1, &data[..]) + } + }; + let d = { + if let Version::V2 = kind { + let (vd, dr) = d.split_at(N32_SIZE); + let version = crate::read_u32(vd); + if version != Version::V2 as u32 { + return Err(Error::UnsupportedVersion { version }); + } + dr + } else { + d + } + }; + let (fan, bytes_read) = read_fan(d); + let (_, _d) = d.split_at(bytes_read); + let num_objects = fan[FAN_LEN - 1]; + + (kind, fan, num_objects) + }; + validate_fan(&fan)?; + validate_size(&data, kind, num_objects, hash_len)?; + Ok(Self { + data, + path, + version: kind, + num_objects, + fan, + hash_len, + object_hash, + }) + } +} + +fn read_fan(d: &[u8]) -> ([u32; FAN_LEN], usize) { + assert!(d.len() >= FAN_LEN * N32_SIZE); + + let mut fan = [0; FAN_LEN]; + for (c, f) in d.chunks_exact(N32_SIZE).zip(fan.iter_mut()) { + *f = crate::read_u32(c); + } + (fan, FAN_LEN * N32_SIZE) +} + +fn validate_fan(fan: &[u32; FAN_LEN]) -> Result<(), Error> { + if !crate::fan_is_monotonically_increasing(fan) { + return Err(Error::Corrupt { + message: "Pack index fan-out table must be monotonically increasing".into(), + }); + } + Ok(()) +} + +fn validate_size(data: &[u8], kind: Version, num_objects: u32, hash_len: usize) -> Result<(), Error> { + let num_objects = num_objects as usize; + let footer_size = hash_len * 2; + let expected_size = match kind { + Version::V1 => FAN_LEN + .checked_mul(N32_SIZE) + .and_then(|size| size.checked_add(num_objects.checked_mul(N32_SIZE + hash_len)?)) + .and_then(|size| size.checked_add(footer_size)) + .ok_or_else(|| Error::Corrupt { + message: "Pack index size overflowed while validating version 1 layout".into(), + })?, + Version::V2 => { + let v2_header_size = V2_SIGNATURE.len() + N32_SIZE + FAN_LEN * N32_SIZE; + let oid_bytes = num_objects.checked_mul(hash_len).ok_or_else(|| Error::Corrupt { + message: "Pack index size overflowed while validating object ids".into(), + })?; + let table_bytes = num_objects.checked_mul(N32_SIZE).ok_or_else(|| Error::Corrupt { + message: "Pack index size overflowed while validating 32-bit tables".into(), + })?; + let offset32_start = v2_header_size + .checked_add(oid_bytes) + .and_then(|size| size.checked_add(table_bytes)) + .ok_or_else(|| Error::Corrupt { + message: "Pack index size overflowed while locating 32-bit offsets".into(), + })?; + let offset32_end = offset32_start.checked_add(table_bytes).ok_or_else(|| Error::Corrupt { + message: "Pack index size overflowed while locating 32-bit offsets".into(), + })?; + if offset32_end > data.len() { + return Err(Error::Corrupt { + message: format!( + "Pack index of size {} is too small for {} objects in version 2", + data.len(), + num_objects + ), + }); + } + let (large_offsets, max_large_offset_index) = data[offset32_start..offset32_end] + .chunks_exact(N32_SIZE) + .filter_map(|offset| { + let offset = crate::read_u32(offset); + (offset & (1 << 31) != 0).then_some((offset ^ (1 << 31)) as usize) + }) + .fold((0usize, 0usize), |(count, max_index), index| { + (count + 1, max_index.max(index)) + }); + v2_header_size + .checked_add(oid_bytes) + .and_then(|size| size.checked_add(table_bytes)) + .and_then(|size| size.checked_add(table_bytes)) + .and_then(|size| size.checked_add(large_offsets.checked_mul(size_of::())?)) + .and_then(|size| size.checked_add(footer_size)) + .ok_or_else(|| Error::Corrupt { + message: "Pack index size overflowed while validating version 2 layout".into(), + }) + .and_then(|expected_size| { + if large_offsets > 0 && max_large_offset_index >= large_offsets { + return Err(Error::Corrupt { + message: format!( + "Pack index references large offset {max_large_offset_index}, but only {large_offsets} large offsets are present" + ), + }); + } + Ok(expected_size) + })? + } + }; + if data.len() != expected_size { + // Aborting here is needed for protection against malformed inputs, or the offset access done later can panic + // as it's done without explicit error handling. + return Err(Error::Corrupt { + message: format!( + "Pack index size is incorrect, expected {expected_size} bytes for {num_objects} objects in version {kind:?}, but got {} bytes", + data.len() + ), + }); + } + Ok(()) +} diff --git a/knot2/third_party/gix-pack/src/index/mod.rs b/knot2/third_party/gix-pack/src/index/mod.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/index/mod.rs @@ -0,0 +1,163 @@ +//! an index into the pack file + +/// From itertools +/// Create an iterator running multiple iterators in lockstep. +/// +/// The `izip!` iterator yields elements until any subiterator +/// returns `None`. +/// +/// This is a version of the standard ``.zip()`` that's supporting more than +/// two iterators. The iterator element type is a tuple with one element +/// from each of the input iterators. Just like ``.zip()``, the iteration stops +/// when the shortest of the inputs reaches its end. +/// +/// **Note:** The result of this macro is in the general case an iterator +/// composed of repeated `.zip()` and a `.map()`; it has an anonymous type. +/// The special cases of one and two arguments produce the equivalent of +/// `$a.into_iter()` and `$a.into_iter().zip($b)` respectively. +/// +/// Prefer this macro `izip!()` over [`multizip`] for the performance benefits +/// of using the standard library `.zip()`. +/// +/// [`multizip`]: fn.multizip.html +/// +/// ```ignore +/// # use itertools::izip; +/// # +/// # fn main() { +/// +/// // iterate over three sequences side-by-side +/// let mut results = [0, 0, 0, 0]; +/// let inputs = [3, 7, 9, 6]; +/// +/// for (r, index, input) in izip!(&mut results, 0..10, &inputs) { +/// *r = index * 10 + input; +/// } +/// +/// assert_eq!(results, [0 + 3, 10 + 7, 29, 36]); +/// # } +/// ``` +/// +/// (The above is vendored from [itertools](https://github.com/rust-itertools/itertools), +/// including the original doctest, though it has been marked `ignore` here.) +macro_rules! izip { + // @closure creates a tuple-flattening closure for .map() call. usage: + // @closure partial_pattern => partial_tuple , rest , of , iterators + // eg. izip!( @closure ((a, b), c) => (a, b, c) , dd , ee ) + ( @closure $p:pat => $tup:expr ) => { + |$p| $tup + }; + + // The "b" identifier is a different identifier on each recursion level thanks to hygiene. + ( @closure $p:pat => ( $($tup:tt)* ) , $_iter:expr $( , $tail:expr )* ) => { + izip!(@closure ($p, b) => ( $($tup)*, b ) $( , $tail )*) + }; + + // unary + ($first:expr $(,)*) => { + std::iter::IntoIterator::into_iter($first) + }; + + // binary + ($first:expr, $second:expr $(,)*) => { + izip!($first) + .zip($second) + }; + + // n-ary where n > 2 + ( $first:expr $( , $rest:expr )* $(,)* ) => { + izip!($first) + $( + .zip($rest) + )* + .map( + izip!(@closure a => (a) $( , $rest )*) + ) + }; +} + +use crate::MMap; + +/// The version of an index file +#[derive(Default, PartialEq, Eq, Ord, PartialOrd, Debug, Hash, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[allow(missing_docs)] +pub enum Version { + V1 = 1, + #[default] + V2 = 2, +} + +impl Version { + /// The kind of hash to produce to be compatible to this kind of index + pub fn hash(&self) -> gix_hash::Kind { + #[cfg(not(feature = "sha1"))] + unreachable!("pack index versions V1 and V2 require SHA1 support"); + #[cfg(feature = "sha1")] + match self { + Version::V1 | Version::V2 => gix_hash::Kind::Sha1, + } + } +} + +/// A way to indicate if a lookup, despite successful, was ambiguous or yielded exactly +/// one result in the particular index. +pub type PrefixLookupResult = Result; + +/// The type for referring to indices of an entry within the index file. +pub type EntryIndex = u32; + +const FAN_LEN: usize = 256; + +/// A representation of a pack index file +pub struct File { + data: T, + path: std::path::PathBuf, + version: Version, + num_objects: u32, + fan: [u32; FAN_LEN], + hash_len: usize, + object_hash: gix_hash::Kind, +} + +/// Basic file information +impl File +where + T: crate::FileData, +{ + /// The version of the pack index + pub fn version(&self) -> Version { + self.version + } + /// The path of the opened index file + pub fn path(&self) -> &std::path::Path { + &self.path + } + /// The amount of objects stored in the pack and index, as one past the highest entry index. + pub fn num_objects(&self) -> EntryIndex { + self.num_objects + } + /// The kind of hash we assume + pub fn object_hash(&self) -> gix_hash::Kind { + self.object_hash + } +} + +const V2_SIGNATURE: &[u8] = b"\xfftOc"; +/// +pub mod init; + +pub(crate) mod access; +pub use access::Entry; + +pub(crate) mod encode; +/// +pub mod traverse; +mod util; +/// +pub mod verify; +/// +#[cfg(feature = "streaming-input")] +pub mod write; +#[cfg(feature = "streaming-input")] +pub use write::function::write_data_iter_to_stream; diff --git a/knot2/third_party/gix-pack/src/index/util.rs b/knot2/third_party/gix-pack/src/index/util.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/index/util.rs @@ -0,0 +1,23 @@ +use std::time::Instant; + +use gix_features::progress::{self, Progress}; + +use crate::exact_vec; + +pub(crate) fn index_entries_sorted_by_offset_ascending( + idx: &crate::index::File, + progress: &mut dyn Progress, +) -> Vec { + progress.init(Some(idx.num_objects as usize), progress::count("entries")); + let start = Instant::now(); + + let mut v = exact_vec(idx.num_objects as usize); + for entry in idx.iter() { + v.push(entry); + progress.inc(); + } + v.sort_by_key(|e| e.pack_offset); + + progress.show_throughput(start); + v +} diff --git a/knot2/third_party/gix-pack/src/index/verify.rs b/knot2/third_party/gix-pack/src/index/verify.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/index/verify.rs @@ -0,0 +1,273 @@ +use std::sync::atomic::AtomicBool; + +use gix_features::progress::{DynNestedProgress, Progress}; +use gix_object::WriteTo; + +use crate::index; + +/// +pub mod integrity { + use std::marker::PhantomData; + + use gix_object::bstr::BString; + + /// Returned by [`index::File::verify_integrity()`][crate::index::File::verify_integrity()]. + #[derive(thiserror::Error, Debug)] + #[allow(missing_docs)] + pub enum Error { + #[error("Reserialization of an object failed")] + Io(#[from] std::io::Error), + #[error("The fan at index {index} is out of order as it's larger then the following value.")] + Fan { index: usize }, + #[error("{kind} object {id} could not be decoded")] + ObjectDecode { + source: gix_object::decode::Error, + kind: gix_object::Kind, + id: gix_hash::ObjectId, + }, + #[error("{kind} object {id} wasn't re-encoded without change, wanted\n{expected}\n\nGOT\n\n{actual}")] + ObjectEncodeMismatch { + kind: gix_object::Kind, + id: gix_hash::ObjectId, + expected: BString, + actual: BString, + }, + } + + /// Returned by [`index::File::verify_integrity()`][crate::index::File::verify_integrity()]. + pub struct Outcome { + /// The computed checksum of the index which matched the stored one. + pub actual_index_checksum: gix_hash::ObjectId, + /// The packs traversal outcome, if one was provided + pub pack_traverse_statistics: Option, + } + + /// Additional options to define how the integrity should be verified. + #[derive(Clone)] + pub struct Options { + /// The thoroughness of the verification + pub verify_mode: crate::index::verify::Mode, + /// The way to traverse packs + pub traversal: crate::index::traverse::Algorithm, + /// The amount of threads to use of `Some(N)`, with `None|Some(0)` using all available cores are used. + pub thread_limit: Option, + /// A function to create a pack cache + pub make_pack_lookup_cache: F, + } + + impl Default for Options crate::cache::Never> { + fn default() -> Self { + Options { + verify_mode: Default::default(), + traversal: Default::default(), + thread_limit: None, + make_pack_lookup_cache: || crate::cache::Never, + } + } + } + + /// The progress ids used in [`index::File::verify_integrity()`][crate::index::File::verify_integrity()]. + /// + /// Use this information to selectively extract the progress of interest in case the parent application has custom visualization. + #[derive(Debug, Copy, Clone)] + pub enum ProgressId { + /// The amount of bytes read to verify the index checksum. + ChecksumBytes, + /// A root progress for traversal which isn't actually used directly, but here to link to the respective `ProgressId` types. + Traverse(PhantomData), + } + + impl From for gix_features::progress::Id { + fn from(v: ProgressId) -> Self { + match v { + ProgressId::ChecksumBytes => *b"PTHI", + ProgressId::Traverse(_) => gix_features::progress::UNKNOWN, + } + } + } +} + +/// +pub mod checksum { + /// Returned by [`index::File::verify_checksum()`][crate::index::File::verify_checksum()]. + pub type Error = crate::verify::checksum::Error; +} + +/// Various ways in which a pack and index can be verified +#[derive(Default, Debug, Eq, PartialEq, Hash, Clone, Copy)] +pub enum Mode { + /// Validate the object hash and CRC32 + HashCrc32, + /// Validate hash and CRC32, and decode each non-Blob object. + /// Each object should be valid, i.e. be decodable. + HashCrc32Decode, + /// Validate hash and CRC32, and decode and encode each non-Blob object. + /// Each object should yield exactly the same hash when re-encoded. + #[default] + HashCrc32DecodeEncode, +} + +/// Information to allow verifying the integrity of an index with the help of its corresponding pack. +pub struct PackContext<'a, F> { + /// The pack data file itself. + pub data: &'a crate::data::File, + /// The options further configuring the pack traversal and verification + pub options: integrity::Options, +} + +/// Verify and validate the content of the index file +impl index::File +where + T: crate::FileData + Sync, +{ + /// Returns the trailing hash stored at the end of this index file. + /// + /// It's a hash over all bytes of the index. + pub fn index_checksum(&self) -> gix_hash::ObjectId { + gix_hash::ObjectId::from_bytes_or_panic(&self.data[self.data.len() - self.hash_len..]) + } + + /// Returns the hash of the pack data file that this index file corresponds to. + /// + /// It should [`crate::data::File::checksum()`] of the corresponding pack data file. + pub fn pack_checksum(&self) -> gix_hash::ObjectId { + let from = self.data.len() - self.hash_len * 2; + gix_hash::ObjectId::from_bytes_or_panic(&self.data[from..][..self.hash_len]) + } + + /// Validate that our [`index_checksum()`][index::File::index_checksum()] matches the actual contents + /// of this index file, and return it if it does. + pub fn verify_checksum( + &self, + progress: &mut dyn Progress, + should_interrupt: &AtomicBool, + ) -> Result { + crate::verify::checksum_on_disk_or_mmap( + self.path(), + &self.data, + self.index_checksum(), + self.object_hash, + progress, + should_interrupt, + ) + } + + /// The most thorough validation of integrity of both index file and the corresponding pack data file, if provided. + /// Returns the checksum of the index file, the traversal outcome and the given progress if the integrity check is successful. + /// + /// If `pack` is provided, it is expected (and validated to be) the pack belonging to this index. + /// It will be used to validate internal integrity of the pack before checking each objects integrity + /// is indeed as advertised via its SHA1 as stored in this index, as well as the CRC32 hash. + /// The last member of the Option is a function returning an implementation of [`crate::cache::DecodeEntry`] to be used if + /// the [`index::traverse::Algorithm`] is `Lookup`. + /// To set this to `None`, use `None::<(_, _, _, fn() -> crate::cache::Never)>`. + /// + /// The `thread_limit` optionally specifies the amount of threads to be used for the [pack traversal][index::File::traverse()]. + /// `make_cache` is only used in case a `pack` is specified, use existing implementations in the [`crate::cache`] module. + /// + /// # Tradeoffs + /// + /// The given `progress` is inevitably consumed if there is an error, which is a tradeoff chosen to easily allow using `?` in the + /// error case. + pub fn verify_integrity( + &self, + pack: Option>, + progress: &mut dyn DynNestedProgress, + should_interrupt: &AtomicBool, + ) -> Result> + where + C: crate::cache::DecodeEntry, + F: Fn() -> C + Send + Clone, + { + if let Some(first_invalid) = crate::verify::fan(&self.fan) { + return Err(index::traverse::Error::Processor(integrity::Error::Fan { + index: first_invalid, + })); + } + + match pack { + Some(PackContext { + data: pack, + options: + integrity::Options { + verify_mode, + traversal, + thread_limit, + make_pack_lookup_cache, + }, + }) => self + .traverse( + pack, + progress, + should_interrupt, + { + let mut encode_buf = Vec::with_capacity(2048); + move |kind, data, index_entry, progress| { + Self::verify_entry(verify_mode, &mut encode_buf, kind, data, index_entry, progress) + } + }, + index::traverse::Options { + traversal, + thread_limit, + check: index::traverse::SafetyCheck::All, + make_pack_lookup_cache, + }, + ) + .map(|o| integrity::Outcome { + actual_index_checksum: o.actual_index_checksum, + pack_traverse_statistics: Some(o.statistics), + }), + None => self + .verify_checksum( + &mut progress + .add_child_with_id("Sha1 of index".into(), integrity::ProgressId::ChecksumBytes.into()), + should_interrupt, + ) + .map_err(index::traverse::Error::IndexVerify) + .map(|id| integrity::Outcome { + actual_index_checksum: id, + pack_traverse_statistics: None, + }), + } + } + + #[allow(clippy::too_many_arguments)] + fn verify_entry( + verify_mode: Mode, + encode_buf: &mut Vec, + object_kind: gix_object::Kind, + buf: &[u8], + index_entry: &index::Entry, + _progress: &dyn gix_features::progress::Progress, + ) -> Result<(), integrity::Error> { + if let Mode::HashCrc32Decode | Mode::HashCrc32DecodeEncode = verify_mode { + use gix_object::Kind::*; + match object_kind { + Tree | Commit | Tag => { + let object = + gix_object::ObjectRef::from_bytes(buf, object_kind, index_entry.oid.kind()).map_err(|err| { + integrity::Error::ObjectDecode { + source: err, + kind: object_kind, + id: index_entry.oid, + } + })?; + if let Mode::HashCrc32DecodeEncode = verify_mode { + encode_buf.clear(); + object.write_to(&mut *encode_buf)?; + if encode_buf.as_slice() != buf { + return Err(integrity::Error::ObjectEncodeMismatch { + kind: object_kind, + id: index_entry.oid, + expected: buf.into(), + actual: encode_buf.clone().into(), + }); + } + } + } + Blob => {} + } + } + Ok(()) + } +} diff --git a/knot2/third_party/gix-pack/src/multi_index/access.rs b/knot2/third_party/gix-pack/src/multi_index/access.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/multi_index/access.rs @@ -0,0 +1,149 @@ +use std::{ + ops::Range, + path::{Path, PathBuf}, +}; + +use crate::{ + data, + index::PrefixLookupResult, + multi_index::{EntryIndex, File, PackIndex, Version}, +}; + +/// Represents an entry within a multi index file, effectively mapping object [`IDs`][gix_hash::ObjectId] to pack data +/// files and the offset within. +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Entry { + /// The ID of the object. + pub oid: gix_hash::ObjectId, + /// The offset to the object's header in the pack data file. + pub pack_offset: data::Offset, + /// The index of the pack matching our [`File::index_names()`] slice. + pub pack_index: PackIndex, +} + +/// Access methods +impl File +where + T: crate::FileData, +{ + /// Returns the version of the multi-index file. + pub fn version(&self) -> Version { + self.version + } + /// Returns the path from which the multi-index file was loaded. + /// + /// Note that it might have changed in the mean time, or might have been removed as well. + pub fn path(&self) -> &Path { + &self.path + } + /// Returns the amount of indices stored in this multi-index file. It's the same as [File::index_names().len()][File::index_names()], + /// and returned as one past the highest known index. + pub fn num_indices(&self) -> PackIndex { + self.num_indices + } + /// Returns the total amount of objects available for lookup, and returned as one past the highest known entry index + pub fn num_objects(&self) -> EntryIndex { + self.num_objects + } + /// Returns the kind of hash function used for object ids available in this index. + pub fn object_hash(&self) -> gix_hash::Kind { + self.object_hash + } + /// Returns the checksum over the entire content of the file (excluding the checksum itself). + /// + /// It can be used to validate it didn't change after creation. + pub fn checksum(&self) -> gix_hash::ObjectId { + gix_hash::ObjectId::from_bytes_or_panic(&self.data[self.data.len() - self.hash_len..]) + } + /// Return all names of index files (`*.idx`) whose objects we contain. + /// + /// The corresponding pack can be found by replacing the `.idx` extension with `.pack`. + pub fn index_names(&self) -> &[PathBuf] { + &self.index_names + } +} + +impl File +where + T: crate::FileData, +{ + /// Return the object id at the given `index`, which ranges from 0 to [File::num_objects()]. + pub fn oid_at_index(&self, index: EntryIndex) -> &gix_hash::oid { + debug_assert!(index < self.num_objects, "index out of bounds"); + let index: usize = index as usize; + let start = self.lookup_ofs + index * self.hash_len; + gix_hash::oid::from_bytes_unchecked(&self.data[start..][..self.hash_len]) + } + + /// Given a `prefix`, find an object that matches it uniquely within this index and return `Some(Ok(entry_index))`. + /// If there is more than one object matching the object `Some(Err(())` is returned. + /// + /// Finally, if no object matches the index, the return value is `None`. + /// + /// Pass `candidates` to obtain the set of entry-indices matching `prefix`, with the same return value as + /// one would have received if it remained `None`. It will be empty if no object matched the `prefix`. + /// + // NOTE: pretty much the same things as in `index::File::lookup`, change things there + // as well. + pub fn lookup_prefix( + &self, + prefix: gix_hash::Prefix, + candidates: Option<&mut Range>, + ) -> Option { + crate::index::access::lookup_prefix( + prefix, + candidates, + &self.fan, + &|idx| self.oid_at_index(idx), + self.num_objects, + ) + } + + /// Find the index ranging from 0 to [File::num_objects()] that belongs to data associated with `id`, or `None` if it wasn't found. + /// + /// Use this index for finding additional information via [`File::pack_id_and_pack_offset_at_index()`]. + pub fn lookup(&self, id: impl AsRef) -> Option { + crate::index::access::lookup(id.as_ref(), &self.fan, &|idx| self.oid_at_index(idx)) + } + + /// Given the `index` ranging from 0 to [File::num_objects()], return the pack index and its absolute offset into the pack. + /// + /// The pack-index refers to an entry in the [`index_names`][File::index_names()] list, from which the pack can be derived. + pub fn pack_id_and_pack_offset_at_index(&self, index: EntryIndex) -> (PackIndex, data::Offset) { + const OFFSET_ENTRY_SIZE: usize = 4 + 4; + let index = index as usize; + let start = self.offsets_ofs + index * OFFSET_ENTRY_SIZE; + + const HIGH_BIT: u32 = 1 << 31; + + let pack_index = crate::read_u32(&self.data[start..][..4]); + let offset = &self.data[start + 4..][..4]; + let ofs32 = crate::read_u32(offset); + let pack_offset = if (ofs32 & HIGH_BIT) == HIGH_BIT { + // We determine if large offsets are actually larger than 4GB and if not, we don't use the high-bit to signal anything + // but allow the presence of the large-offset chunk to signal what's happening. + if let Some(offsets_64) = self.large_offsets_ofs { + let from = offsets_64 + (ofs32 ^ HIGH_BIT) as usize * 8; + crate::read_u64(&self.data[from..][..8]) + } else { + u64::from(ofs32) + } + } else { + u64::from(ofs32) + }; + (pack_index, pack_offset) + } + + /// Return an iterator over all entries within this file. + pub fn iter(&self) -> impl Iterator + '_ { + (0..self.num_objects).map(move |idx| { + let (pack_index, pack_offset) = self.pack_id_and_pack_offset_at_index(idx); + Entry { + oid: self.oid_at_index(idx).to_owned(), + pack_offset, + pack_index, + } + }) + } +} diff --git a/knot2/third_party/gix-pack/src/multi_index/chunk.rs b/knot2/third_party/gix-pack/src/multi_index/chunk.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/multi_index/chunk.rs @@ -0,0 +1,312 @@ +/// Information for the chunk about index names +pub mod index_names { + use std::path::{Path, PathBuf}; + + use gix_object::bstr::{BString, ByteSlice}; + + /// The ID used for the index-names chunk. + pub const ID: gix_chunk::Id = *b"PNAM"; + + /// + pub mod decode { + use std::collections::TryReserveError; + + use gix_object::bstr::BString; + + /// The error returned by [`from_bytes()`][super::from_bytes()]. + #[derive(Debug, thiserror::Error)] + #[allow(missing_docs)] + pub enum Error { + #[error("The pack names were not ordered alphabetically.")] + NotOrderedAlphabetically, + #[error("Each pack path name must be terminated with a null byte")] + MissingNullByte, + #[error("Entry too large to fit in memory")] + OutOfMemory, + #[error("Couldn't turn path '{path}' into OS path due to encoding issues")] + PathEncoding { path: BString }, + #[error("non-padding bytes found after all paths were read.")] + UnknownTrailerBytes, + } + + impl From for Error { + #[cold] + fn from(_: TryReserveError) -> Self { + Self::OutOfMemory + } + } + } + + /// Parse null-separated index names from the given `chunk` of bytes. + /// + /// `chunk` + ///: Contains `num_packs` null-terminated index names, optionally followed by padding bytes which are typically `\0`. + /// + /// `num_packs` + ///: The number of index names expected to be present in `chunk`. + /// + /// `alloc_limit_bytes` + ///: Limits allocations caused by attacker-controlled on-disk multi-index data. + /// It is used to reject reserving the output `Vec` if its capacity estimate exceeds the limit, + /// and to reject any single path entry whose byte length exceeds the limit before turning it into a `PathBuf`. + /// Use `None` to disable this limit. + pub fn from_bytes( + mut chunk: &[u8], + num_packs: u32, + alloc_limit_bytes: Option, + ) -> Result, decode::Error> { + let mut out = Vec::new(); + let num_packs = usize::try_from(num_packs).map_err(|_| decode::Error::OutOfMemory)?; + let vec_allocation = num_packs + .checked_mul(std::mem::size_of::()) + .ok_or(decode::Error::OutOfMemory)?; + if alloc_limit_bytes.is_some_and(|limit| vec_allocation > limit) { + return Err(decode::Error::OutOfMemory); + } + out.try_reserve(num_packs)?; + + for _ in 0..num_packs { + let null_byte_pos = chunk.find_byte(b'\0').ok_or(decode::Error::MissingNullByte)?; + + let path = &chunk[..null_byte_pos]; + if alloc_limit_bytes.is_some_and(|limit| path.len() > limit) { + return Err(decode::Error::OutOfMemory); + } + let path = gix_path::try_from_byte_slice(path) + .map_err(|_| decode::Error::PathEncoding { + path: BString::from(path), + })? + .to_owned(); + + if let Some(previous) = out.last() { + if previous >= &path { + return Err(decode::Error::NotOrderedAlphabetically); + } + } + out.push(path); + + chunk = &chunk[null_byte_pos + 1..]; + } + + if !chunk.is_empty() && !chunk.iter().all(|b| *b == 0) { + return Err(decode::Error::UnknownTrailerBytes); + } + // NOTE: git writes garbage into this chunk, usually extra \0 bytes, which we simply ignore. If we were strict + // about it we couldn't read this chunk data at all. + Ok(out) + } + + /// Calculate the size on disk for our chunk with the given index paths. Note that these are expected to have been processed already + /// to actually be file names. + pub fn storage_size(paths: impl IntoIterator>) -> u64 { + let mut count = 0u64; + for path in paths { + let path = path.as_ref(); + let ascii_path = path.to_str().expect("UTF-8 compatible paths"); + assert!( + ascii_path.is_ascii(), + "must use ascii bytes for correct size computation" + ); + count += (ascii_path.len() + 1/* null byte */) as u64; + } + + let needed_alignment = CHUNK_ALIGNMENT - (count % CHUNK_ALIGNMENT); + if needed_alignment < CHUNK_ALIGNMENT { + count += needed_alignment; + } + count + } + + /// Write all `paths` in order to `out`, including padding. + pub fn write( + paths: impl IntoIterator>, + out: &mut dyn std::io::Write, + ) -> std::io::Result<()> { + let mut written_bytes = 0; + for path in paths { + let path = path.as_ref().to_str().expect("UTF-8 path"); + out.write_all(path.as_bytes())?; + out.write_all(&[0])?; + written_bytes += path.len() as u64 + 1; + } + + let needed_alignment = CHUNK_ALIGNMENT - (written_bytes % CHUNK_ALIGNMENT); + if needed_alignment < CHUNK_ALIGNMENT { + let padding = [0u8; CHUNK_ALIGNMENT as usize]; + out.write_all(&padding[..needed_alignment as usize])?; + } + Ok(()) + } + + const CHUNK_ALIGNMENT: u64 = 4; +} + +/// Information for the chunk with the fanout table +pub mod fanout { + use crate::multi_index; + + /// The size of the fanout table + pub const SIZE: usize = 4 * 256; + + /// The id uniquely identifying the fanout table. + pub const ID: gix_chunk::Id = *b"OIDF"; + + /// Decode the fanout table contained in `chunk`, or return `None` if it didn't have the expected size. + pub fn from_bytes(chunk: &[u8]) -> Option<[u32; 256]> { + if chunk.len() != SIZE { + return None; + } + let mut out = [0; 256]; + for (c, f) in chunk.chunks_exact(4).zip(out.iter_mut()) { + *f = u32::from_be_bytes(c.try_into().unwrap()); + } + out.into() + } + + /// Write the fanout for the given entries, which must be sorted by oid + pub(crate) fn write( + sorted_entries: &[multi_index::write::Entry], + out: &mut dyn std::io::Write, + ) -> std::io::Result<()> { + let fanout = crate::index::encode::fanout(&mut sorted_entries.iter().map(|e| e.id.first_byte())); + + for value in fanout.iter() { + out.write_all(&value.to_be_bytes())?; + } + Ok(()) + } +} + +/// Information about the oid lookup table. +pub mod lookup { + use std::ops::Range; + + use crate::multi_index; + + /// The id uniquely identifying the oid lookup table. + pub const ID: gix_chunk::Id = *b"OIDL"; + + /// Return the number of bytes needed to store the data on disk for the given amount of `entries` + pub fn storage_size(entries: usize, object_hash: gix_hash::Kind) -> u64 { + (entries * object_hash.len_in_bytes()) as u64 + } + + pub(crate) fn write( + sorted_entries: &[multi_index::write::Entry], + out: &mut dyn std::io::Write, + ) -> std::io::Result<()> { + for entry in sorted_entries { + out.write_all(entry.id.as_slice())?; + } + Ok(()) + } + + /// Return true if the size of the `offset` range seems to match for a `hash` of the given kind and the amount of objects. + pub fn is_valid(offset: &Range, hash: gix_hash::Kind, num_objects: u32) -> bool { + (offset.end - offset.start) == (num_objects as usize).saturating_mul(hash.len_in_bytes()) + } +} + +/// Information about the offsets table. +pub mod offsets { + use std::ops::Range; + + use crate::multi_index; + + /// The id uniquely identifying the offsets table. + pub const ID: gix_chunk::Id = *b"OOFF"; + + /// Return the amount of bytes needed to offset data for `entries`. + pub fn storage_size(entries: usize) -> u64 { + (entries * (4 /*pack-id*/ + 4/* pack offset */)) as u64 + } + + pub(crate) fn write( + sorted_entries: &[multi_index::write::Entry], + large_offsets_needed: bool, + out: &mut dyn std::io::Write, + ) -> std::io::Result<()> { + use crate::index::encode::{HIGH_BIT, LARGE_OFFSET_THRESHOLD}; + let mut num_large_offsets = 0u32; + + for entry in sorted_entries { + out.write_all(&entry.pack_index.to_be_bytes())?; + + let offset: u32 = if large_offsets_needed { + if entry.pack_offset > LARGE_OFFSET_THRESHOLD { + let res = num_large_offsets | HIGH_BIT; + num_large_offsets += 1; + res + } else { + entry.pack_offset as u32 + } + } else { + entry + .pack_offset + .try_into() + .expect("without large offsets, pack-offset fits u32") + }; + out.write_all(&offset.to_be_bytes())?; + } + Ok(()) + } + + /// Returns true if the `offset` range seems to match the size required for the untrusted `num_objects`. + pub fn is_valid(offset: &Range, num_objects: u32) -> bool { + let entry_size = 4 /* pack-id */ + 4 /* pack-offset */; + (offset.end - offset.start) == (num_objects as usize).saturating_mul(entry_size) + } +} + +/// Information about the large offsets table. +pub mod large_offsets { + use std::ops::Range; + + use crate::{index::encode::LARGE_OFFSET_THRESHOLD, multi_index}; + + /// The id uniquely identifying the large offsets table (with 64 bit offsets) + pub const ID: gix_chunk::Id = *b"LOFF"; + + /// Returns Some(num-large-offset) if there are offsets larger than u32. + pub(crate) fn num_large_offsets(entries: &[multi_index::write::Entry]) -> Option { + let mut num_large_offsets = 0; + let mut needs_large_offsets = false; + for entry in entries { + if entry.pack_offset > LARGE_OFFSET_THRESHOLD { + num_large_offsets += 1; + } + if entry.pack_offset > crate::data::Offset::from(u32::MAX) { + needs_large_offsets = true; + } + } + + needs_large_offsets.then_some(num_large_offsets) + } + /// Returns true if the `offsets` range seems to be properly aligned for the data we expect. + pub fn is_valid(offset: &Range) -> bool { + (offset.end - offset.start) % 8 == 0 + } + + pub(crate) fn write( + sorted_entries: &[multi_index::write::Entry], + mut num_large_offsets: usize, + out: &mut dyn std::io::Write, + ) -> std::io::Result<()> { + for offset in sorted_entries + .iter() + .filter_map(|e| (e.pack_offset > LARGE_OFFSET_THRESHOLD).then_some(e.pack_offset)) + { + out.write_all(&offset.to_be_bytes())?; + num_large_offsets = num_large_offsets + .checked_sub(1) + .expect("BUG: wrote more offsets the previously found"); + } + assert_eq!(num_large_offsets, 0, "BUG: wrote less offsets than initially counted"); + Ok(()) + } + + /// Return the number of bytes needed to store the given amount of `large_offsets` + pub(crate) fn storage_size(large_offsets: usize) -> u64 { + 8 * large_offsets as u64 + } +} diff --git a/knot2/third_party/gix-pack/src/multi_index/init.rs b/knot2/third_party/gix-pack/src/multi_index/init.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/multi_index/init.rs @@ -0,0 +1,178 @@ +use std::path::{Path, PathBuf}; + +use crate::multi_index::{File, Version, chunk}; + +mod error { + use crate::multi_index::chunk; + + /// The error returned by [File::at()][super::File::at()]. + #[derive(Debug, thiserror::Error)] + #[allow(missing_docs)] + pub enum Error { + #[error("Could not open multi-index file at '{path}'")] + Io { + source: std::io::Error, + path: std::path::PathBuf, + }, + #[error("{message}")] + Corrupt { message: &'static str }, + #[error("Unsupported multi-index version: {version})")] + UnsupportedVersion { version: u8 }, + #[error("Unsupported hash kind: {kind})")] + UnsupportedObjectHash { kind: u8 }, + #[error(transparent)] + ChunkFileQuery(#[from] gix_error::Message), + #[error(transparent)] + ChunkFileDecode(#[from] gix_error::ValidationError), + #[error("The multi-pack fan doesn't have the correct size of 256 * 4 bytes")] + MultiPackFanSize, + #[error(transparent)] + PackNames(#[from] chunk::index_names::decode::Error), + #[error("multi-index chunk {:?} has invalid size: {message}", String::from_utf8_lossy(.id))] + InvalidChunkSize { id: gix_chunk::Id, message: &'static str }, + } +} + +pub use error::Error; + +/// Initialization +impl File { + /// Open the multi-index file at the given `path`. + /// + /// `alloc_limit_bytes` bounds each allocation caused by user-controlled on-disk data, useful for untrusted input. + /// Use `None` to disable the limit. + pub fn at(path: impl AsRef, alloc_limit_bytes: Option) -> Result { + Self::at_inner(path.as_ref(), alloc_limit_bytes) + } + + fn at_inner(path: &Path, alloc_limit_bytes: Option) -> Result { + let data = crate::mmap::read_only(path).map_err(|source| Error::Io { + source, + path: path.to_owned(), + })?; + Self::from_data(data, path.to_owned(), alloc_limit_bytes) + } +} + +impl File +where + T: crate::FileData, +{ + /// Instantiate a multi-index file from `data` as assumed to be read or memory-mapped from `path`. + /// + /// `alloc_limit_bytes` bounds each allocation caused by untrusted on-disk multi-index data. + /// Use `None` to disable the limit. + /// + /// It is used to reject reserving the output `Vec` if its capacity estimate exceeds the limit, + /// and to reject any single path entry whose byte length exceeds the limit before turning it into a `PathBuf`. + pub fn from_data(data: T, path: PathBuf, alloc_limit_bytes: Option) -> Result { + const TRAILER_LEN: usize = gix_hash::Kind::shortest().len_in_bytes(); /* trailing hash */ + if data.len() + < Self::HEADER_LEN + + gix_chunk::file::Index::size_for_entries(4 /*index names, fan, offsets, oids*/) + + chunk::fanout::SIZE + + TRAILER_LEN + { + return Err(Error::Corrupt { + message: "multi-index file is truncated and too short", + }); + } + + let (version, object_hash, num_chunks, num_indices) = { + let (signature, data) = data.split_at(4); + if signature != Self::SIGNATURE { + return Err(Error::Corrupt { + message: "Invalid signature", + }); + } + let (version, data) = data.split_at(1); + let version = match version[0] { + 1 => Version::V1, + version => return Err(Error::UnsupportedVersion { version }), + }; + + let (object_hash, data) = data.split_at(1); + let object_hash = gix_hash::Kind::try_from(object_hash[0]) + .map_err(|unknown| Error::UnsupportedObjectHash { kind: unknown })?; + let (num_chunks, data) = data.split_at(1); + let num_chunks = num_chunks[0]; + + let (_num_base_files, data) = data.split_at(1); // TODO: handle base files once it's clear what this does + + let (num_indices, _) = data.split_at(4); + let num_indices = crate::read_u32(num_indices); + + (version, object_hash, num_chunks, num_indices) + }; + + let chunks = gix_chunk::file::Index::from_bytes(&data, Self::HEADER_LEN, u32::from(num_chunks))?; + + let index_names = chunks.data_by_id(&data, chunk::index_names::ID)?; + let index_names = chunk::index_names::from_bytes(index_names, num_indices, alloc_limit_bytes)?; + + let fan = chunks.data_by_id(&data, chunk::fanout::ID)?; + let fan = chunk::fanout::from_bytes(fan).ok_or(Error::MultiPackFanSize)?; + let num_objects = fan[255]; + validate_fan(&fan)?; + + let lookup = chunks.validated_usize_offset_by_id(chunk::lookup::ID, |offset| { + chunk::lookup::is_valid(&offset, object_hash, num_objects) + .then_some(offset) + .ok_or(Error::InvalidChunkSize { + id: chunk::lookup::ID, + message: "The chunk with alphabetically ordered object ids doesn't have the correct size", + }) + })??; + let offsets = chunks.validated_usize_offset_by_id(chunk::offsets::ID, |offset| { + chunk::offsets::is_valid(&offset, num_objects) + .then_some(offset) + .ok_or(Error::InvalidChunkSize { + id: chunk::offsets::ID, + message: "The chunk with offsets into the pack doesn't have the correct size", + }) + })??; + let large_offsets = chunks + .validated_usize_offset_by_id(chunk::large_offsets::ID, |offset| { + chunk::large_offsets::is_valid(&offset) + .then_some(offset) + .ok_or(Error::InvalidChunkSize { + id: chunk::large_offsets::ID, + message: "The chunk with large offsets into the pack doesn't have the correct size", + }) + }) + .ok() + .transpose()?; + + let checksum_offset = chunks.highest_offset() as usize; + let trailer = &data[checksum_offset..]; + if trailer.len() != object_hash.len_in_bytes() { + return Err(Error::Corrupt { + message: "Trailing checksum didn't have the expected size or there were unknown bytes after the checksum.", + }); + } + + Ok(File { + data, + path, + version, + hash_len: object_hash.len_in_bytes(), + object_hash, + fan, + index_names, + lookup_ofs: lookup.start, + offsets_ofs: offsets.start, + large_offsets_ofs: large_offsets.map(|r| r.start), + num_objects, + num_indices, + }) + } +} + +fn validate_fan(fan: &[u32; 256]) -> Result<(), Error> { + if !crate::fan_is_monotonically_increasing(fan) { + return Err(Error::Corrupt { + message: "multi-index fan-out table must be monotonically increasing", + }); + } + Ok(()) +} diff --git a/knot2/third_party/gix-pack/src/multi_index/mod.rs b/knot2/third_party/gix-pack/src/multi_index/mod.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/multi_index/mod.rs @@ -0,0 +1,53 @@ +use std::path::PathBuf; + +use crate::MMap; + +/// Known multi-index file versions +#[derive(Default, PartialEq, Eq, Ord, PartialOrd, Debug, Hash, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[allow(missing_docs)] +pub enum Version { + #[default] + V1 = 1, +} + +/// An index into our [`File::index_names()`] array yielding the name of the index and by implication, its pack file. +pub type PackIndex = u32; + +/// The type for referring to indices of an entry within the index file. +pub type EntryIndex = u32; + +/// A representation of an index file for multiple packs at the same time, typically stored in a file +/// named 'multi-pack-index'. +pub struct File { + data: T, + path: std::path::PathBuf, + version: Version, + hash_len: usize, + object_hash: gix_hash::Kind, + /// The amount of pack files contained within + num_indices: u32, + num_objects: u32, + + fan: [u32; 256], + index_names: Vec, + lookup_ofs: usize, + offsets_ofs: usize, + large_offsets_ofs: Option, +} + +/// +pub mod write; +pub use write::function::write_from_index_paths; + +/// +mod access; + +/// +pub mod verify; + +/// +pub mod chunk; + +/// +pub mod init; diff --git a/knot2/third_party/gix-pack/src/multi_index/verify.rs b/knot2/third_party/gix-pack/src/multi_index/verify.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/multi_index/verify.rs @@ -0,0 +1,330 @@ +use std::{cmp::Ordering, sync::atomic::AtomicBool, time::Instant}; + +use gix_features::progress::{Count, DynNestedProgress, Progress}; + +use crate::{exact_vec, index, multi_index::File}; + +/// +pub mod integrity { + use crate::multi_index::EntryIndex; + + /// Returned by [`multi_index::File::verify_integrity()`][crate::multi_index::File::verify_integrity()]. + #[derive(thiserror::Error, Debug)] + #[allow(missing_docs)] + pub enum Error { + #[error("Object {id} should be at pack-offset {expected_pack_offset} but was found at {actual_pack_offset}")] + PackOffsetMismatch { + id: gix_hash::ObjectId, + expected_pack_offset: u64, + actual_pack_offset: u64, + }, + #[error(transparent)] + MultiIndexChecksum(#[from] crate::multi_index::verify::checksum::Error), + #[error(transparent)] + IndexIntegrity(#[from] crate::index::verify::integrity::Error), + #[error(transparent)] + BundleInit(#[from] crate::bundle::init::Error), + #[error("Counted {actual} objects, but expected {expected} as per multi-index")] + UnexpectedObjectCount { actual: usize, expected: usize }, + #[error("{id} wasn't found in the index referenced in the multi-pack index")] + OidNotFound { id: gix_hash::ObjectId }, + #[error("The object id at multi-index entry {index} wasn't in order")] + OutOfOrder { index: EntryIndex }, + #[error("The fan at index {index} is out of order as it's larger then the following value.")] + Fan { index: usize }, + #[error("The multi-index claims to have no objects")] + Empty, + #[error("The multi-index path '{path}' has no parent directory")] + InvalidPath { path: std::path::PathBuf }, + #[error("Interrupted")] + Interrupted, + } + + /// Returned by [`multi_index::File::verify_integrity()`][crate::multi_index::File::verify_integrity()]. + pub struct Outcome { + /// The computed checksum of the multi-index which matched the stored one. + pub actual_index_checksum: gix_hash::ObjectId, + /// The for each entry in [`index_names()`][super::File::index_names()] provide the corresponding pack traversal outcome. + pub pack_traverse_statistics: Vec, + } + + /// The progress ids used in [`multi_index::File::verify_integrity()`][crate::multi_index::File::verify_integrity()]. + /// + /// Use this information to selectively extract the progress of interest in case the parent application has custom visualization. + #[derive(Debug, Copy, Clone)] + pub enum ProgressId { + /// The amount of bytes read to verify the multi-index checksum. + ChecksumBytes, + /// The amount of objects whose offset has been checked. + ObjectOffsets, + } + + impl From for gix_features::progress::Id { + fn from(v: ProgressId) -> Self { + match v { + ProgressId::ChecksumBytes => *b"MVCK", + ProgressId::ObjectOffsets => *b"MVOF", + } + } + } +} + +/// +pub mod checksum { + /// Returned by [`multi_index::File::verify_checksum()`][crate::multi_index::File::verify_checksum()]. + pub type Error = crate::verify::checksum::Error; +} + +impl File +where + T: crate::FileData, +{ + /// Validate that our [`checksum()`][File::checksum()] matches the actual contents + /// of this index file, and return it if it does. + pub fn verify_checksum( + &self, + progress: &mut dyn Progress, + should_interrupt: &AtomicBool, + ) -> Result { + crate::verify::checksum_on_disk_or_mmap( + self.path(), + &self.data, + self.checksum(), + self.object_hash, + progress, + should_interrupt, + ) + } + + /// Similar to [`verify_integrity()`][File::verify_integrity()] but without any deep inspection of objects. + /// + /// Instead we only validate the contents of the multi-index itself. + pub fn verify_integrity_fast( + &self, + progress: &mut dyn DynNestedProgress, + should_interrupt: &AtomicBool, + ) -> Result { + self.verify_integrity_inner( + progress, + should_interrupt, + false, + index::verify::integrity::Options::default(), + ) + .map_err(|err| match err { + index::traverse::Error::Processor(err) => err, + _ => unreachable!("BUG: no other error type is possible"), + }) + .map(|o| o.actual_index_checksum) + } + + /// Similar to [`crate::Bundle::verify_integrity()`] but checks all contained indices and their packs. + /// + /// Note that it's considered a failure if an index doesn't have a corresponding pack. + pub fn verify_integrity( + &self, + progress: &mut dyn DynNestedProgress, + should_interrupt: &AtomicBool, + options: index::verify::integrity::Options, + ) -> Result> + where + C: crate::cache::DecodeEntry, + F: Fn() -> C + Send + Clone, + { + self.verify_integrity_inner(progress, should_interrupt, true, options) + } + + fn verify_integrity_inner( + &self, + progress: &mut dyn DynNestedProgress, + should_interrupt: &AtomicBool, + deep_check: bool, + options: index::verify::integrity::Options, + ) -> Result> + where + C: crate::cache::DecodeEntry, + F: Fn() -> C + Send + Clone, + { + let parent = self.path.parent().ok_or_else(|| { + index::traverse::Error::Processor(integrity::Error::InvalidPath { + path: self.path.clone(), + }) + })?; + + let actual_index_checksum = self + .verify_checksum( + &mut progress.add_child_with_id( + format!("{}: checksum", self.path.display()), + integrity::ProgressId::ChecksumBytes.into(), + ), + should_interrupt, + ) + .map_err(integrity::Error::from) + .map_err(index::traverse::Error::Processor)?; + + if let Some(first_invalid) = crate::verify::fan(&self.fan) { + return Err(index::traverse::Error::Processor(integrity::Error::Fan { + index: first_invalid, + })); + } + + if self.num_objects == 0 { + return Err(index::traverse::Error::Processor(integrity::Error::Empty)); + } + + let mut pack_traverse_statistics = Vec::new(); + + let operation_start = Instant::now(); + let mut total_objects_checked = 0; + let mut pack_ids_and_offsets = exact_vec(self.num_objects as usize); + { + let order_start = Instant::now(); + let mut progress = progress.add_child_with_id("checking oid order".into(), gix_features::progress::UNKNOWN); + progress.init( + Some(self.num_objects as usize), + gix_features::progress::count("objects"), + ); + + for entry_index in 0..(self.num_objects - 1) { + let lhs = self.oid_at_index(entry_index); + let rhs = self.oid_at_index(entry_index + 1); + + if rhs.cmp(lhs) != Ordering::Greater { + return Err(index::traverse::Error::Processor(integrity::Error::OutOfOrder { + index: entry_index, + })); + } + let (pack_id, _) = self.pack_id_and_pack_offset_at_index(entry_index); + pack_ids_and_offsets.push((pack_id, entry_index)); + progress.inc(); + } + { + let entry_index = self.num_objects - 1; + let (pack_id, _) = self.pack_id_and_pack_offset_at_index(entry_index); + pack_ids_and_offsets.push((pack_id, entry_index)); + } + // sort by pack-id to allow handling all indices matching a pack while its open. + pack_ids_and_offsets.sort_by_key(|l| l.0); + progress.show_throughput(order_start); + }; + + progress.init( + Some(self.num_indices as usize), + gix_features::progress::count("indices"), + ); + + let mut pack_ids_slice = pack_ids_and_offsets.as_slice(); + + for (pack_id, index_file_name) in self.index_names.iter().enumerate() { + progress.set_name(index_file_name.display().to_string()); + progress.inc(); + + let mut bundle = None; + let index; + let index_path = parent.join(index_file_name); + let index = if deep_check { + bundle = crate::Bundle::at(index_path, self.object_hash) + .map_err(integrity::Error::from) + .map_err(index::traverse::Error::Processor)? + .into(); + bundle.as_ref().map(|b| &b.index).expect("just set") + } else { + index = Some( + index::File::at(index_path, self.object_hash) + .map_err(|err| integrity::Error::BundleInit(crate::bundle::init::Error::Index(err))) + .map_err(index::traverse::Error::Processor)?, + ); + index.as_ref().expect("just set") + }; + + let slice_end = pack_ids_slice.partition_point(|e| e.0 == pack_id as crate::data::Id); + let multi_index_entries_to_check = &pack_ids_slice[..slice_end]; + { + let offset_start = Instant::now(); + let mut offsets_progress = progress.add_child_with_id( + "verify object offsets".into(), + integrity::ProgressId::ObjectOffsets.into(), + ); + offsets_progress.init( + Some(pack_ids_and_offsets.len()), + gix_features::progress::count("objects"), + ); + pack_ids_slice = &pack_ids_slice[slice_end..]; + + for entry_id in multi_index_entries_to_check.iter().map(|e| e.1) { + let oid = self.oid_at_index(entry_id); + let (_, expected_pack_offset) = self.pack_id_and_pack_offset_at_index(entry_id); + let entry_in_bundle_index = index.lookup(oid).ok_or_else(|| { + index::traverse::Error::Processor(integrity::Error::OidNotFound { id: oid.to_owned() }) + })?; + let actual_pack_offset = index.pack_offset_at_index(entry_in_bundle_index); + if actual_pack_offset != expected_pack_offset { + return Err(index::traverse::Error::Processor( + integrity::Error::PackOffsetMismatch { + id: oid.to_owned(), + expected_pack_offset, + actual_pack_offset, + }, + )); + } + offsets_progress.inc(); + } + + if should_interrupt.load(std::sync::atomic::Ordering::Relaxed) { + return Err(index::traverse::Error::Processor(integrity::Error::Interrupted)); + } + offsets_progress.show_throughput(offset_start); + } + + total_objects_checked += multi_index_entries_to_check.len(); + + if let Some(bundle) = bundle { + progress.set_name(format!("Validating {}", index_file_name.display())); + let crate::bundle::verify::integrity::Outcome { + actual_index_checksum: _, + pack_traverse_outcome, + } = bundle + .verify_integrity(progress, should_interrupt, options.clone()) + .map_err(|err| { + use index::traverse::Error::*; + match err { + Processor(err) => Processor(integrity::Error::IndexIntegrity(err)), + IndexVerify(err) => IndexVerify(err), + Tree(err) => Tree(err), + TreeTraversal(err) => TreeTraversal(err), + PackVerify(err) => PackVerify(err), + PackDecode { id, offset, source } => PackDecode { id, offset, source }, + PackMismatch(err) => PackMismatch(err), + EntryType(err) => EntryType(err), + PackObjectVerify { offset, source } => PackObjectVerify { offset, source }, + Crc32Mismatch { + expected, + actual, + offset, + kind, + } => Crc32Mismatch { + expected, + actual, + offset, + kind, + }, + Interrupted => Interrupted, + } + })?; + pack_traverse_statistics.push(pack_traverse_outcome); + } + } + + assert_eq!( + self.num_objects as usize, total_objects_checked, + "BUG: our slicing should allow to visit all objects" + ); + + progress.set_name("Validating multi-pack".into()); + progress.show_throughput(operation_start); + + Ok(integrity::Outcome { + actual_index_checksum, + pack_traverse_statistics, + }) + } +} diff --git a/knot2/third_party/gix-pack/src/multi_index/write.rs b/knot2/third_party/gix-pack/src/multi_index/write.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/multi_index/write.rs @@ -0,0 +1,255 @@ +use std::time::SystemTime; + +use crate::multi_index; + +mod error { + /// The error returned by [`crate::multi_index::write_from_index_paths()`]. + #[derive(Debug, thiserror::Error)] + #[allow(missing_docs)] + pub enum Error { + #[error(transparent)] + Io(#[from] gix_hash::io::Error), + #[error("Interrupted")] + Interrupted, + #[error(transparent)] + OpenIndex(#[from] crate::index::init::Error), + } +} +pub use error::Error; + +/// An entry suitable for sorting and writing +pub(crate) struct Entry { + pub(crate) id: gix_hash::ObjectId, + pub(crate) pack_index: u32, + pub(crate) pack_offset: crate::data::Offset, + /// Used for sorting in case of duplicates + index_mtime: SystemTime, +} + +/// Options for use in [`multi_index::write_from_index_paths()`]. +pub struct Options { + /// The kind of hash to use for objects and to expect in the input files. + pub object_hash: gix_hash::Kind, +} + +/// The result of [`multi_index::write_from_index_paths()`]. +pub struct Outcome { + /// The calculated multi-index checksum of the file at `multi_index_path`. + pub multi_index_checksum: gix_hash::ObjectId, +} + +/// The progress ids used in [`crate::multi_index::write_from_index_paths()`]. +/// +/// Use this information to selectively extract the progress of interest in case the parent application has custom visualization. +#[derive(Debug, Copy, Clone)] +pub enum ProgressId { + /// Counts each path in the input set whose entries we enumerate and write into the multi-index + FromPathsCollectingEntries, + /// The amount of bytes written as part of the multi-index. + BytesWritten, +} + +impl From for gix_features::progress::Id { + fn from(v: ProgressId) -> Self { + match v { + ProgressId::FromPathsCollectingEntries => *b"MPCE", + ProgressId::BytesWritten => *b"MPBW", + } + } +} + +impl multi_index::File { + pub(crate) const SIGNATURE: &'static [u8] = b"MIDX"; + pub(crate) const HEADER_LEN: usize = 4 /*signature*/ + + 1 /*version*/ + + 1 /*object id version*/ + + 1 /*num chunks */ + + 1 /*num base files */ + + 4 /*num pack files*/; +} + +pub(super) mod function { + use std::{ + path::PathBuf, + sync::atomic::{AtomicBool, Ordering}, + time::{Instant, SystemTime}, + }; + + use gix_features::progress::{Count, DynNestedProgress, Progress}; + + use crate::{MMap, multi_index}; + + use super::{Entry, Error, Options, Outcome, ProgressId}; + + /// Create a new multi-index file for writing to `out` from the pack index files at `index_paths`. + /// + /// Progress is sent to `progress` and interruptions checked via `should_interrupt`. + pub fn write_from_index_paths( + mut index_paths: Vec, + out: &mut dyn std::io::Write, + progress: &mut dyn DynNestedProgress, + should_interrupt: &AtomicBool, + Options { object_hash }: Options, + ) -> Result { + let out = gix_hash::io::Write::new(out, object_hash); + let (index_paths_sorted, index_filenames_sorted) = { + index_paths.sort(); + let file_names = index_paths + .iter() + .map(|p| PathBuf::from(p.file_name().expect("file name present"))) + .collect::>(); + (index_paths, file_names) + }; + + let entries = { + let mut entries = Vec::new(); + let start = Instant::now(); + let mut progress = progress.add_child_with_id( + "Collecting entries".into(), + ProgressId::FromPathsCollectingEntries.into(), + ); + progress.init(Some(index_paths_sorted.len()), gix_features::progress::count("indices")); + + // This could be parallelized… but it's probably not worth it unless you have 500mio objects. + for (index_id, index) in index_paths_sorted.iter().enumerate() { + let mtime = index + .metadata() + .and_then(|m| m.modified()) + .unwrap_or(SystemTime::UNIX_EPOCH); + let index = crate::index::File::at(index, object_hash)?; + + entries.reserve(index.num_objects() as usize); + entries.extend(index.iter().map(|e| Entry { + id: e.oid, + pack_index: index_id as u32, + pack_offset: e.pack_offset, + index_mtime: mtime, + })); + progress.inc(); + if should_interrupt.load(Ordering::Relaxed) { + return Err(Error::Interrupted); + } + } + progress.show_throughput(start); + + let start = Instant::now(); + progress.set_name("Deduplicate".into()); + progress.init(Some(entries.len()), gix_features::progress::count("entries")); + entries.sort_by(|l, r| { + l.id.cmp(&r.id) + .then_with(|| l.index_mtime.cmp(&r.index_mtime).reverse()) + .then_with(|| l.pack_index.cmp(&r.pack_index)) + }); + entries.dedup_by_key(|e| e.id); + progress.inc_by(entries.len()); + progress.show_throughput(start); + if should_interrupt.load(Ordering::Relaxed) { + return Err(Error::Interrupted); + } + entries + }; + + let mut cf = gix_chunk::file::Index::for_writing(); + cf.plan_chunk( + multi_index::chunk::index_names::ID, + multi_index::chunk::index_names::storage_size(&index_filenames_sorted), + ); + cf.plan_chunk(multi_index::chunk::fanout::ID, multi_index::chunk::fanout::SIZE as u64); + cf.plan_chunk( + multi_index::chunk::lookup::ID, + multi_index::chunk::lookup::storage_size(entries.len(), object_hash), + ); + cf.plan_chunk( + multi_index::chunk::offsets::ID, + multi_index::chunk::offsets::storage_size(entries.len()), + ); + + let num_large_offsets = multi_index::chunk::large_offsets::num_large_offsets(&entries); + if let Some(num_large_offsets) = num_large_offsets { + cf.plan_chunk( + multi_index::chunk::large_offsets::ID, + multi_index::chunk::large_offsets::storage_size(num_large_offsets), + ); + } + + let mut write_progress = + progress.add_child_with_id("Writing multi-index".into(), ProgressId::BytesWritten.into()); + let write_start = Instant::now(); + write_progress.init( + Some(cf.planned_storage_size() as usize + multi_index::File::::HEADER_LEN), + gix_features::progress::bytes(), + ); + let mut out = gix_features::progress::Write { + inner: out, + progress: write_progress, + }; + + let bytes_written = multi_index::File::::write_header( + &mut out, + cf.num_chunks().try_into().expect("BUG: wrote more than 256 chunks"), + index_paths_sorted.len() as u32, + object_hash, + ) + .map_err(gix_hash::io::Error::from)?; + + { + progress.set_name("Writing chunks".into()); + progress.init(Some(cf.num_chunks()), gix_features::progress::count("chunks")); + + let mut chunk_write = cf + .into_write(&mut out, bytes_written) + .map_err(gix_hash::io::Error::from)?; + while let Some(chunk_to_write) = chunk_write.next_chunk() { + match chunk_to_write { + multi_index::chunk::index_names::ID => { + multi_index::chunk::index_names::write(&index_filenames_sorted, &mut chunk_write) + } + multi_index::chunk::fanout::ID => multi_index::chunk::fanout::write(&entries, &mut chunk_write), + multi_index::chunk::lookup::ID => multi_index::chunk::lookup::write(&entries, &mut chunk_write), + multi_index::chunk::offsets::ID => { + multi_index::chunk::offsets::write(&entries, num_large_offsets.is_some(), &mut chunk_write) + } + multi_index::chunk::large_offsets::ID => multi_index::chunk::large_offsets::write( + &entries, + num_large_offsets.expect("available if planned"), + &mut chunk_write, + ), + unknown => unreachable!("BUG: forgot to implement chunk {:?}", std::str::from_utf8(&unknown)), + } + .map_err(gix_hash::io::Error::from)?; + progress.inc(); + if should_interrupt.load(Ordering::Relaxed) { + return Err(Error::Interrupted); + } + } + } + + // write trailing checksum + let multi_index_checksum = out.inner.hash.try_finalize().map_err(gix_hash::io::Error::from)?; + out.inner + .inner + .write_all(multi_index_checksum.as_slice()) + .map_err(gix_hash::io::Error::from)?; + out.progress.show_throughput(write_start); + + Ok(Outcome { multi_index_checksum }) + } +} + +impl multi_index::File { + fn write_header( + out: &mut dyn std::io::Write, + num_chunks: u8, + num_indices: u32, + object_hash: gix_hash::Kind, + ) -> std::io::Result { + out.write_all(Self::SIGNATURE)?; + out.write_all(&[crate::multi_index::Version::V1 as u8])?; + out.write_all(&[object_hash as u8])?; + out.write_all(&[num_chunks])?; + out.write_all(&[0])?; /* unused number of base files */ + out.write_all(&num_indices.to_be_bytes())?; + + Ok(Self::HEADER_LEN) + } +} diff --git a/knot2/third_party/gix-pack/src/bundle/write/error.rs b/knot2/third_party/gix-pack/src/bundle/write/error.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/bundle/write/error.rs @@ -0,0 +1,17 @@ +use std::io; + +use gix_tempfile::handle::Writable; + +/// The error returned by [`Bundle::write_to_directory()`][crate::Bundle::write_to_directory()] +#[derive(thiserror::Error, Debug)] +#[allow(missing_docs)] +pub enum Error { + #[error("An IO error occurred when reading the pack or creating a temporary file")] + Io(#[from] io::Error), + #[error(transparent)] + PackIter(#[from] crate::data::input::Error), + #[error("Could not move a temporary file into its desired place")] + Persist(#[from] gix_tempfile::handle::persist::Error), + #[error(transparent)] + IndexWrite(#[from] crate::index::write::Error), +} diff --git a/knot2/third_party/gix-pack/src/bundle/write/mod.rs b/knot2/third_party/gix-pack/src/bundle/write/mod.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/bundle/write/mod.rs @@ -0,0 +1,389 @@ +use std::{ + io, + io::Write, + marker::PhantomData, + path::{Path, PathBuf}, + sync::{Arc, atomic::AtomicBool}, +}; + +use gix_features::{interrupt, progress, progress::Progress}; +use gix_tempfile::{AutoRemove, ContainingDirectory}; + +use crate::data; + +mod error; +pub use error::Error; +use gix_features::progress::prodash::DynNestedProgress; + +mod types; +use types::{LockWriter, PassThrough}; +pub use types::{Options, Outcome}; + +use crate::bundle::write::types::SharedTempFile; + +/// The progress ids used in [`write_to_directory()`][crate::Bundle::write_to_directory()]. +/// +/// Use this information to selectively extract the progress of interest in case the parent application has custom visualization. +#[derive(Debug, Copy, Clone)] +pub enum ProgressId { + /// The amount of bytes read from the input pack data file. + ReadPackBytes, + /// A root progress counting logical steps towards an index file on disk. + /// + /// Underneath will be more progress information related to actually producing the index. + IndexingSteps(PhantomData), +} + +impl From for gix_features::progress::Id { + fn from(v: ProgressId) -> Self { + match v { + ProgressId::ReadPackBytes => *b"BWRB", + ProgressId::IndexingSteps(_) => *b"BWCI", + } + } +} + +impl crate::Bundle { + /// Given a `pack` data stream, write it along with a generated index into the `directory` if `Some` or discard all output if `None`. + /// + /// In the latter case, the functionality provided here is more a kind of pack data stream validation. + /// + /// * `progress` provides detailed progress information which can be discarded with [`gix_features::progress::Discard`]. + /// * `should_interrupt` is checked regularly and when true, the whole operation will stop. + /// * `thin_pack_base_object_lookup` If set, we expect to see a thin-pack with objects that reference their base object by object id which is + /// expected to exist in the object database the bundle is contained within. + /// `options` further configure how the task is performed. + /// + /// # Note + /// + /// * the resulting pack may be empty, that is, contains zero objects in some situations. This is a valid reply by a server and should + /// be accounted for. + /// - Empty packs always have the same name and not handling this case will result in at most one superfluous pack. + pub fn write_to_directory( + pack: &mut dyn io::BufRead, + directory: Option<&Path>, + progress: &mut dyn DynNestedProgress, + should_interrupt: &AtomicBool, + thin_pack_base_object_lookup: Option, + options: Options, + ) -> Result { + let _span = gix_features::trace::coarse!("gix_pack::Bundle::write_to_directory()"); + let mut read_progress = progress.add_child_with_id("read pack".into(), ProgressId::ReadPackBytes.into()); + read_progress.init(None, progress::bytes()); + let pack = progress::Read { + inner: pack, + progress: progress::ThroughputOnDrop::new(read_progress), + }; + + let object_hash = options.object_hash; + let data_file = Arc::new(parking_lot::Mutex::new(io::BufWriter::with_capacity( + 64 * 1024, + match directory.as_ref() { + Some(directory) => gix_tempfile::new(directory, ContainingDirectory::Exists, AutoRemove::Tempfile)?, + None => gix_tempfile::new(std::env::temp_dir(), ContainingDirectory::Exists, AutoRemove::Tempfile)?, + }, + ))); + let (pack_entries_iter, pack_version): ( + Box>>, + _, + ) = match thin_pack_base_object_lookup { + Some(thin_pack_lookup) => { + let pack = interrupt::Read { + inner: pack, + should_interrupt, + }; + let buffered_pack = io::BufReader::new(pack); + let pack_entries_iter = data::input::LookupRefDeltaObjectsIter::new( + data::input::BytesToEntriesIter::new_from_header( + buffered_pack, + options.iteration_mode, + data::input::EntryDataMode::KeepAndCrc32, + object_hash, + )?, + thin_pack_lookup, + ); + let pack_version = pack_entries_iter.inner.version(); + let pack_entries_iter = data::input::EntriesToBytesIter::new( + pack_entries_iter, + LockWriter { + writer: data_file.clone(), + }, + pack_version, + object_hash, + ); + (Box::new(pack_entries_iter), pack_version) + } + None => { + let pack = PassThrough { + reader: interrupt::Read { + inner: pack, + should_interrupt, + }, + writer: Some(data_file.clone()), + }; + // This buf-reader is required to assure we call 'read()' in order to fill the (extra) buffer. Otherwise all the counting + // we do with the wrapped pack reader doesn't work as it does not expect anyone to call BufRead functions directly. + // However, this is exactly what's happening in the ZipReader implementation that is eventually used. + // The performance impact of this is probably negligible, compared to all the other work that is done anyway :D. + let buffered_pack = io::BufReader::new(pack); + let pack_entries_iter = data::input::BytesToEntriesIter::new_from_header( + buffered_pack, + options.iteration_mode, + data::input::EntryDataMode::Crc32, + object_hash, + )?; + let pack_version = pack_entries_iter.version(); + (Box::new(pack_entries_iter), pack_version) + } + }; + let WriteOutcome { + outcome, + data_path, + index_path, + keep_path, + } = crate::Bundle::inner_write( + directory, + progress, + options, + data_file, + pack_entries_iter, + should_interrupt, + pack_version, + )?; + + Ok(Outcome { + index: outcome, + object_hash, + pack_version, + data_path, + index_path, + keep_path, + }) + } + + /// Equivalent to [`write_to_directory()`][crate::Bundle::write_to_directory()] but offloads reading of the pack into its own thread, hence the `Send + 'static'` bounds. + /// + /// # Note + /// + /// As it sends portions of the input to a thread it requires the 'static lifetime for the interrupt flags. This can only + /// be satisfied by a static `AtomicBool` which is only suitable for programs that only run one of these operations at a time + /// or don't mind that all of them abort when the flag is set. + pub fn write_to_directory_eagerly( + pack: Box, + pack_size: Option, + directory: Option>, + progress: &mut dyn DynNestedProgress, + should_interrupt: &'static AtomicBool, + thin_pack_base_object_lookup: Option, + options: Options, + ) -> Result { + let _span = gix_features::trace::coarse!("gix_pack::Bundle::write_to_directory_eagerly()"); + let mut read_progress = progress.add_child_with_id("read pack".into(), ProgressId::ReadPackBytes.into()); /* Bundle Write Read pack Bytes*/ + read_progress.init(pack_size.map(|s| s as usize), progress::bytes()); + let pack = progress::Read { + inner: pack, + progress: progress::ThroughputOnDrop::new(read_progress), + }; + + let data_file = Arc::new(parking_lot::Mutex::new(io::BufWriter::new(match directory.as_ref() { + Some(directory) => gix_tempfile::new(directory, ContainingDirectory::Exists, AutoRemove::Tempfile)?, + None => gix_tempfile::new(std::env::temp_dir(), ContainingDirectory::Exists, AutoRemove::Tempfile)?, + }))); + let object_hash = options.object_hash; + let eight_pages = 4096 * 8; + let (pack_entries_iter, pack_version): ( + Box> + Send + 'static>, + _, + ) = match thin_pack_base_object_lookup { + Some(thin_pack_lookup) => { + let pack = interrupt::Read { + inner: pack, + should_interrupt, + }; + let buffered_pack = io::BufReader::with_capacity(eight_pages, pack); + let pack_entries_iter = data::input::LookupRefDeltaObjectsIter::new( + data::input::BytesToEntriesIter::new_from_header( + buffered_pack, + options.iteration_mode, + data::input::EntryDataMode::KeepAndCrc32, + object_hash, + )?, + thin_pack_lookup, + ); + let pack_kind = pack_entries_iter.inner.version(); + (Box::new(pack_entries_iter), pack_kind) + } + None => { + let pack = PassThrough { + reader: interrupt::Read { + inner: pack, + should_interrupt, + }, + writer: Some(data_file.clone()), + }; + let buffered_pack = io::BufReader::with_capacity(eight_pages, pack); + let pack_entries_iter = data::input::BytesToEntriesIter::new_from_header( + buffered_pack, + options.iteration_mode, + data::input::EntryDataMode::Crc32, + object_hash, + )?; + let pack_kind = pack_entries_iter.version(); + (Box::new(pack_entries_iter), pack_kind) + } + }; + let num_objects = pack_entries_iter.size_hint().0; + let pack_entries_iter = + gix_features::parallel::EagerIterIf::new(move || num_objects > 25_000, pack_entries_iter, 5_000, 5); + + let WriteOutcome { + outcome, + data_path, + index_path, + keep_path, + } = crate::Bundle::inner_write( + directory, + progress, + options, + data_file, + Box::new(pack_entries_iter), + should_interrupt, + pack_version, + )?; + + Ok(Outcome { + index: outcome, + object_hash, + pack_version, + data_path, + index_path, + keep_path, + }) + } + + fn inner_write<'a>( + directory: Option>, + progress: &mut dyn DynNestedProgress, + Options { + thread_limit, + iteration_mode: _, + index_version: index_kind, + object_hash, + }: Options, + data_file: SharedTempFile, + mut pack_entries_iter: Box> + 'a>, + should_interrupt: &AtomicBool, + pack_version: data::Version, + ) -> Result { + let mut indexing_progress = progress.add_child_with_id( + "create index file".into(), + ProgressId::IndexingSteps(Default::default()).into(), + ); + Ok(match directory { + Some(directory) => { + let directory = directory.as_ref(); + let mut index_file = gix_tempfile::new(directory, ContainingDirectory::Exists, AutoRemove::Tempfile)?; + + let outcome = crate::index::write_data_iter_to_stream( + index_kind, + { + let data_file = Arc::clone(&data_file); + move || new_pack_file_resolver(data_file, object_hash) + }, + &mut pack_entries_iter, + thread_limit, + &mut indexing_progress, + &mut index_file, + should_interrupt, + object_hash, + pack_version, + )?; + drop(pack_entries_iter); + + if outcome.num_objects == 0 { + WriteOutcome { + outcome, + data_path: None, + index_path: None, + keep_path: None, + } + } else { + let data_path = directory.join(format!("pack-{}.pack", outcome.data_hash.to_hex())); + let index_path = data_path.with_extension("idx"); + let keep_path = if data_path.is_file() { + // avoid trying to overwrite existing files, we know they have the same content + // and this is likely to fail on Windows as negotiation opened the pack. + None + } else { + let keep_path = data_path.with_extension("keep"); + + std::fs::write(&keep_path, b"")?; + Arc::try_unwrap(data_file) + .expect("only one handle left after pack was consumed") + .into_inner() + .into_inner() + .map_err(|err| Error::from(err.into_error()))? + .persist(&data_path)?; + Some(keep_path) + }; + if !index_path.is_file() { + index_file + .persist(&index_path) + .inspect_err(|_err| { + gix_features::trace::warn!("pack file at \"{}\" is retained despite failing to move the index file into place. You can use plumbing to make it usable.",data_path.display()); + })?; + } + WriteOutcome { + outcome, + data_path: Some(data_path), + index_path: Some(index_path), + keep_path, + } + } + } + None => WriteOutcome { + outcome: crate::index::write_data_iter_to_stream( + index_kind, + move || new_pack_file_resolver(data_file, object_hash), + &mut pack_entries_iter, + thread_limit, + &mut indexing_progress, + &mut io::sink(), + should_interrupt, + object_hash, + pack_version, + )?, + data_path: None, + index_path: None, + keep_path: None, + }, + }) + } +} + +fn resolve_entry(range: data::EntryRange, pack: &crate::data::File, buf: &mut Vec) -> bool { + pack.read_into(range, buf) +} + +#[allow(clippy::type_complexity)] // cannot typedef impl Fn +fn new_pack_file_resolver( + data_file: SharedTempFile, + object_hash: gix_hash::Kind, +) -> io::Result<( + impl Fn(data::EntryRange, &crate::data::File, &mut Vec) -> bool + Send + Clone, + crate::data::File, +)> { + let mut guard = data_file.lock(); + guard.flush()?; + let path = guard.get_mut().with_mut(|f| f.path().to_owned())?; + let pack = crate::data::File::at(&path, object_hash) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err.to_string()))?; + Ok((resolve_entry, pack)) +} + +struct WriteOutcome { + outcome: crate::index::write::Outcome, + data_path: Option, + index_path: Option, + keep_path: Option, +} diff --git a/knot2/third_party/gix-pack/src/bundle/write/types.rs b/knot2/third_party/gix-pack/src/bundle/write/types.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/bundle/write/types.rs @@ -0,0 +1,122 @@ +use std::{hash::Hash, io, io::SeekFrom, path::PathBuf, sync::Arc}; + +use gix_tempfile::handle::Writable; + +/// Configuration for [`write_to_directory`][crate::Bundle::write_to_directory()] or +/// [`write_to_directory_eagerly`][crate::Bundle::write_to_directory_eagerly()] +#[derive(Debug, Clone)] +pub struct Options { + /// The amount of threads to use at most when resolving the pack. If `None`, all logical cores are used. + pub thread_limit: Option, + /// Determine how much processing to spend on protecting against corruption or recovering from errors. + pub iteration_mode: crate::data::input::Mode, + /// The version of pack index to write, should be [`crate::index::Version::default()`] + pub index_version: crate::index::Version, + /// The kind of hash to use when writing the bundle. + pub object_hash: gix_hash::Kind, +} + +impl Default for Options { + /// Options which favor speed and correctness and write the most commonly supported index file. + fn default() -> Self { + Options { + thread_limit: None, + iteration_mode: crate::data::input::Mode::Verify, + index_version: Default::default(), + object_hash: Default::default(), + } + } +} + +/// Returned by [`write_to_directory`][crate::Bundle::write_to_directory()] or +/// [`write_to_directory_eagerly`][crate::Bundle::write_to_directory_eagerly()] +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Outcome { + /// The successful result of the index write operation. + pub index: crate::index::write::Outcome, + /// The version of the pack. + pub pack_version: crate::data::Version, + /// The kind of hash stored within the pack and indices. + pub object_hash: gix_hash::Kind, + + /// The path to the pack index file. + pub index_path: Option, + /// The path to the pack data file. + pub data_path: Option, + /// The path to the `.keep` file to prevent collection of the newly written pack until refs are pointing to it. + /// It might be `None` if the file at `data_path` already existed, indicating that we have received a pack that + /// was already present locally. + /// + /// The file is created right before moving the pack data and index data into place (i.e. `data_path` and `index_path`) + /// and is expected to be removed by the caller when ready. + pub keep_path: Option, +} + +impl Outcome { + /// Instantiate a bundle from the newly written index and data file that are represented by this `Outcome` + pub fn to_bundle(&self) -> Option> { + self.index_path + .as_ref() + .map(|path| crate::Bundle::at(path, self.object_hash)) + } +} + +pub(crate) type SharedTempFile = Arc>>>; + +pub(crate) struct PassThrough { + pub reader: R, + pub writer: Option, +} + +impl io::Read for PassThrough +where + R: io::Read, +{ + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let bytes_read = self.reader.read(buf)?; + if let Some(writer) = self.writer.as_mut() { + use std::io::Write; + writer.lock().write_all(&buf[..bytes_read])?; + } + Ok(bytes_read) + } +} +impl io::BufRead for PassThrough +where + R: io::BufRead, +{ + fn fill_buf(&mut self) -> io::Result<&[u8]> { + self.reader.fill_buf() + } + + fn consume(&mut self, amt: usize) { + self.reader.consume(amt); + } +} + +pub(crate) struct LockWriter { + pub writer: SharedTempFile, +} + +impl io::Write for LockWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.writer.lock().write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.writer.lock().flush() + } +} + +impl io::Read for LockWriter { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.writer.lock().get_mut().read(buf) + } +} + +impl io::Seek for LockWriter { + fn seek(&mut self, pos: SeekFrom) -> io::Result { + self.writer.lock().seek(pos) + } +} diff --git a/knot2/third_party/gix-pack/src/cache/delta/from_offsets.rs b/knot2/third_party/gix-pack/src/cache/delta/from_offsets.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/cache/delta/from_offsets.rs @@ -0,0 +1,161 @@ +use std::{ + fs, io, + io::{BufRead, Read, Seek, SeekFrom}, + sync::atomic::{AtomicBool, Ordering}, + time::Instant, +}; + +use gix_features::progress::{self, Progress}; + +use crate::{cache::delta::Tree, data}; + +/// Returned by [`Tree::from_offsets_in_pack()`] +#[derive(thiserror::Error, Debug)] +#[allow(missing_docs)] +pub enum Error { + #[error("{message}")] + Io { source: io::Error, message: &'static str }, + #[error(transparent)] + Header(#[from] crate::data::header::decode::Error), + #[error("Could find object with id {id} in this pack. Thin packs are not supported")] + UnresolvedRefDelta { id: gix_hash::ObjectId }, + #[error(transparent)] + Tree(#[from] crate::cache::delta::Error), + #[error("Interrupted")] + Interrupted, +} + +const PACK_HEADER_LEN: usize = 12; + +/// Generate tree from certain input +impl Tree { + /// Create a new `Tree` from any data sorted by offset, ascending as returned by the `data_sorted_by_offsets` iterator. + /// * `get_pack_offset(item: &T) -> data::Offset` is a function returning the pack offset of the given item, which can be used + /// for obtaining the objects entry within the pack. + /// * `pack_path` is the path to the pack file itself and from which to read the entry data, which is a pack file matching the offsets + /// returned by `get_pack_offset(…)`. + /// * `progress` is used to track progress when creating the tree. + /// * `resolve_in_pack_id(gix_hash::oid) -> Option` takes an object ID and tries to resolve it to an object within this pack if + /// possible. Failing to do so aborts the operation, and this function is not expected to be called in usual packs. It's a theoretical + /// possibility though as old packs might have referred to their objects using the 20 bytes hash, instead of their encoded offset from the base. + /// + /// Note that the sort order is ascending. The given pack file path must match the provided offsets. + pub fn from_offsets_in_pack( + pack_path: &std::path::Path, + data_sorted_by_offsets: impl Iterator, + get_pack_offset: &dyn Fn(&T) -> data::Offset, + resolve_in_pack_id: &dyn Fn(&gix_hash::oid) -> Option, + progress: &mut dyn Progress, + should_interrupt: &AtomicBool, + object_hash: gix_hash::Kind, + ) -> Result { + let mut r = io::BufReader::with_capacity( + 8192 * 8, // this value directly corresponds to performance, 8k (default) is about 4x slower than 64k + fs::File::open(pack_path).map_err(|err| Error::Io { + source: err, + message: "open pack path", + })?, + ); + + let anticipated_num_objects = data_sorted_by_offsets + .size_hint() + .1 + .inspect(|&num_objects| { + progress.init(Some(num_objects), progress::count("objects")); + }) + .unwrap_or_default(); + let mut tree = Tree::with_capacity(anticipated_num_objects)?; + + { + // safety check - assure ourselves it's a pack we can handle + let mut buf = [0u8; PACK_HEADER_LEN]; + r.read_exact(&mut buf).map_err(|err| Error::Io { + source: err, + message: "reading header buffer with at least 12 bytes failed - pack file truncated?", + })?; + crate::data::header::decode(&buf)?; + } + + let then = Instant::now(); + + let mut previous_cursor_position = None::; + + let hash_len = object_hash.len_in_bytes(); + for (idx, data) in data_sorted_by_offsets.enumerate() { + let pack_offset = get_pack_offset(&data); + if let Some(previous_offset) = previous_cursor_position { + Self::advance_cursor_to_pack_offset(&mut r, pack_offset, previous_offset)?; + } + let entry = crate::data::Entry::from_read(&mut r, pack_offset, hash_len).map_err(|err| Error::Io { + source: err, + message: "EOF while parsing header", + })?; + previous_cursor_position = Some(pack_offset + entry.header_size() as u64); + + use crate::data::entry::Header::*; + match entry.header { + Tree | Blob | Commit | Tag => { + tree.add_root(pack_offset, data)?; + } + RefDelta { base_id } => { + resolve_in_pack_id(base_id.as_ref()) + .ok_or(Error::UnresolvedRefDelta { id: base_id }) + .and_then(|base_pack_offset| { + tree.add_child(base_pack_offset, pack_offset, data).map_err(Into::into) + })?; + } + OfsDelta { base_distance } => { + let base_pack_offset = pack_offset + .checked_sub(base_distance) + .expect("in bound distance for deltas"); + tree.add_child(base_pack_offset, pack_offset, data)?; + } + } + progress.inc(); + if idx % 10_000 == 0 && should_interrupt.load(Ordering::SeqCst) { + return Err(Error::Interrupted); + } + } + + progress.show_throughput(then); + Ok(tree) + } + + fn advance_cursor_to_pack_offset( + r: &mut io::BufReader, + pack_offset: u64, + previous_offset: u64, + ) -> Result<(), Error> { + let bytes_to_skip: u64 = pack_offset + .checked_sub(previous_offset) + .expect("continuously ascending pack offsets"); + if bytes_to_skip == 0 { + return Ok(()); + } + let buf = r.fill_buf().map_err(|err| Error::Io { + source: err, + message: "skip bytes", + })?; + if buf.is_empty() { + // This means we have reached the end of file and can't make progress anymore, before we have satisfied our need + // for more + return Err(Error::Io { + source: io::Error::new( + io::ErrorKind::UnexpectedEof, + "ran out of bytes before reading desired amount of bytes", + ), + message: "index file is damaged or corrupt", + }); + } + if bytes_to_skip <= u64::try_from(buf.len()).expect("sensible buffer size") { + // SAFETY: bytes_to_skip <= buf.len() <= usize::MAX + r.consume(bytes_to_skip as usize); + } else { + r.seek(SeekFrom::Start(pack_offset)).map_err(|err| Error::Io { + source: err, + message: "seek to next entry", + })?; + } + Ok(()) + } +} diff --git a/knot2/third_party/gix-pack/src/cache/delta/mod.rs b/knot2/third_party/gix-pack/src/cache/delta/mod.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/cache/delta/mod.rs @@ -0,0 +1,25 @@ +/// Returned when using various methods on a [`Tree`] +#[derive(thiserror::Error, Debug)] +#[allow(missing_docs)] +pub enum Error { + #[error( + "Pack offsets must only increment. The previous pack offset was {last_pack_offset}, the current one is {pack_offset}" + )] + InvariantIncreasingPackOffset { + /// The last seen pack offset + last_pack_offset: crate::data::Offset, + /// The invariant violating offset + pack_offset: crate::data::Offset, + }, +} + +/// +pub mod traverse; + +/// +pub mod from_offsets; + +/// Tree datastructure +mod tree; + +pub use tree::{Item, Tree}; diff --git a/knot2/third_party/gix-pack/src/cache/delta/tree.rs b/knot2/third_party/gix-pack/src/cache/delta/tree.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/cache/delta/tree.rs @@ -0,0 +1,228 @@ +use super::{Error, traverse}; + +#[allow(missing_docs)] +pub struct Item { + pub offset: crate::data::Offset, + pub next_offset: crate::data::Offset, + pub data: T, +} + +impl Item { + pub(crate) fn new(offset: crate::data::Offset, next_offset: crate::data::Offset, data: T) -> Self { + Item { + offset, + next_offset, + data, + } + } +} + +enum Parent { + Root, + Base(u32), + Pending(crate::data::Offset), +} + +#[allow(missing_docs)] +pub struct Tree { + offsets: Vec, + data: Vec, + parent: Vec, +} + +#[allow(missing_docs)] +impl Tree { + pub fn with_capacity(num_objects: usize) -> Result { + Ok(Tree { + offsets: Vec::with_capacity(num_objects), + data: Vec::with_capacity(num_objects), + parent: Vec::with_capacity(num_objects), + }) + } + + pub(super) fn num_items(&self) -> usize { + self.offsets.len() + } + + fn assert_incrementing(&self, offset: crate::data::Offset) -> Result<(), Error> { + match self.offsets.last() { + Some(&last) if offset <= last => Err(Error::InvariantIncreasingPackOffset { + last_pack_offset: last, + pack_offset: offset, + }), + _ => Ok(()), + } + } + + pub fn add_root(&mut self, offset: crate::data::Offset, data: T) -> Result<(), Error> { + self.assert_incrementing(offset)?; + self.offsets.push(offset); + self.data.push(data); + self.parent.push(Parent::Root); + Ok(()) + } + + pub fn add_child( + &mut self, + base_offset: crate::data::Offset, + offset: crate::data::Offset, + data: T, + ) -> Result<(), Error> { + self.assert_incrementing(offset)?; + let parent = match self.offsets.binary_search(&base_offset) { + Ok(index) => Parent::Base(index as u32), + Err(_) => Parent::Pending(base_offset), + }; + self.offsets.push(offset); + self.data.push(data); + self.parent.push(parent); + Ok(()) + } + + pub(super) fn into_forest( + self, + pack_entries_end: crate::data::Offset, + ) -> Result<(Forest, Vec), traverse::Error> { + let Tree { + offsets, + data, + mut parent, + } = self; + let num_nodes = offsets.len(); + + let mut child_start = vec![0u32; num_nodes + 1]; + let mut roots: Vec = Vec::new(); + for index in 0..num_nodes { + if let Parent::Pending(base_offset) = parent[index] { + let base = offsets.binary_search(&base_offset).map_err(|_| { + traverse::Error::OutOfPackRefDelta { + base_pack_offset: base_offset, + } + })?; + parent[index] = Parent::Base(base as u32); + } + match parent[index] { + Parent::Root => roots.push(index as u32), + Parent::Base(base) => child_start[base as usize + 1] += 1, + Parent::Pending(_) => unreachable!("pending parents were resolved above"), + } + } + for index in 0..num_nodes { + child_start[index + 1] += child_start[index]; + } + let mut cursor = child_start.clone(); + let mut child_ids = vec![0u32; child_start[num_nodes] as usize]; + for index in 0..num_nodes { + if let Parent::Base(base) = parent[index] { + let slot = cursor[base as usize]; + child_ids[slot as usize] = index as u32; + cursor[base as usize] = slot + 1; + } + } + + Ok(( + Forest { + offsets, + data, + child_start, + child_ids, + pack_entries_end, + }, + roots, + )) + } +} + +pub(super) struct Forest { + offsets: Vec, + pub(super) data: Vec, + child_start: Vec, + child_ids: Vec, + pack_entries_end: crate::data::Offset, +} + +impl Forest { + pub(super) fn offset(&self, id: u32) -> crate::data::Offset { + self.offsets[id as usize] + } + + pub(super) fn next_offset(&self, id: u32) -> crate::data::Offset { + self.offsets + .get(id as usize + 1) + .copied() + .unwrap_or(self.pack_entries_end) + } + + pub(super) fn entry_slice(&self, id: u32) -> crate::data::EntryRange { + self.offset(id)..self.next_offset(id) + } + + pub(super) fn children(&self, id: u32) -> &[u32] { + let start = self.child_start[id as usize] as usize; + let end = self.child_start[id as usize + 1] as usize; + &self.child_ids[start..end] + } + + pub(super) fn item(&self, id: u32, data: T) -> Item { + Item::new(self.offset(id), self.next_offset(id), data) + } +} + +#[cfg(test)] +mod tests { + mod from_offsets_in_pack { + use std::sync::atomic::AtomicBool; + + use crate as pack; + + const SMALL_PACK_INDEX: &str = + "objects/pack/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.idx"; + const SMALL_PACK: &str = "objects/pack/pack-a2bf8e71d8c18879e499335762dd95119d93d9f1.pack"; + + const INDEX_V1: &str = "objects/pack/pack-c0438c19fb16422b6bbcce24387b3264416d485b.idx"; + const PACK_FOR_INDEX_V1: &str = + "objects/pack/pack-c0438c19fb16422b6bbcce24387b3264416d485b.pack"; + + use gix_testtools::fixture_path; + + #[test] + fn v1() -> Result<(), Box> { + tree(INDEX_V1, PACK_FOR_INDEX_V1) + } + + #[test] + fn v2() -> Result<(), Box> { + tree(SMALL_PACK_INDEX, SMALL_PACK) + } + + fn tree(index_path: &str, pack_path: &str) -> Result<(), Box> { + let idx = pack::index::File::at(fixture_path(index_path), gix_hash::Kind::Sha1)?; + crate::cache::delta::Tree::from_offsets_in_pack( + &fixture_path(pack_path), + idx.sorted_offsets().into_iter(), + &|ofs| *ofs, + &|id| idx.lookup(id).map(|index| idx.pack_offset_at_index(index)), + &mut gix_features::progress::Discard, + &AtomicBool::new(false), + gix_hash::Kind::Sha1, + )?; + Ok(()) + } + } + + mod size { + use gix_testtools::size_ok; + + use super::super::Item; + + #[test] + fn size_of_pack_tree_item() { + let actual = std::mem::size_of::<[Item<()>; 7_500_000]>(); + let expected = 120_000_000; + assert!( + size_ok(actual, expected), + "we don't want these to grow unnoticed: {actual} <~ {expected}" + ); + } + } +} diff --git a/knot2/third_party/gix-pack/src/data/entry/decode.rs b/knot2/third_party/gix-pack/src/data/entry/decode.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/entry/decode.rs @@ -0,0 +1,212 @@ +use std::io; + +use gix_features::decode::leb64_from_read; + +use super::{BLOB, COMMIT, OFS_DELTA, REF_DELTA, TAG, TREE}; +use crate::data; + +/// The error returned by [data::Entry::from_bytes()]. +#[derive(Debug, thiserror::Error)] +#[allow(missing_docs)] +pub enum Error { + #[error("Object type {type_id} is unsupported")] + UnsupportedType { type_id: u8 }, + #[error("Pack entry is truncated: {message}")] + Corrupt { message: &'static str }, + #[error("Pack entry header value overflowed while decoding")] + Overflow, +} + +/// Decoding +impl data::Entry { + /// Decode an entry from the given entry data `d`, providing the `pack_offset` to allow tracking the start of the entry data section. + /// + /// # Panics + /// + /// If we cannot understand the header, garbage data is likely to trigger this. + pub fn from_bytes(d: &[u8], pack_offset: data::Offset, hash_len: usize) -> Result { + let (type_id, size, mut consumed) = parse_header_info(d)?; + + use crate::data::entry::Header::*; + let object = match type_id { + OFS_DELTA => { + let (distance, leb_bytes) = parse_leb64(&d[consumed..])?; + let delta = OfsDelta { + base_distance: distance, + }; + consumed += leb_bytes; + delta + } + REF_DELTA => { + let delta = RefDelta { + base_id: gix_hash::ObjectId::from_bytes_or_panic(d.get(consumed..consumed + hash_len).ok_or( + Error::Corrupt { + message: "ref-delta base object id", + }, + )?), + }; + consumed += hash_len; + delta + } + BLOB => Blob, + TREE => Tree, + COMMIT => Commit, + TAG => Tag, + other => return Err(Error::UnsupportedType { type_id: other }), + }; + Ok(data::Entry { + header: object, + decompressed_size: size, + data_offset: pack_offset + consumed as u64, + }) + } + + /// Instantiate an `Entry` from the reader `r`, providing the `pack_offset` to allow tracking the start of the entry data section. + pub fn from_read(r: &mut dyn io::Read, pack_offset: data::Offset, hash_len: usize) -> io::Result { + let (type_id, size, mut consumed) = streaming_parse_header_info(r)?; + + use crate::data::entry::Header::*; + let object = match type_id { + OFS_DELTA => { + let (distance, leb_bytes) = leb64_from_read(&mut *r)?; + let delta = OfsDelta { + base_distance: distance, + }; + consumed += leb_bytes; + delta + } + REF_DELTA => { + let mut buf = gix_hash::Kind::buf(); + let hash = &mut buf[..hash_len]; + r.read_exact(hash)?; + #[allow(clippy::redundant_slicing)] + let delta = RefDelta { + base_id: gix_hash::ObjectId::from_bytes_or_panic(&hash[..]), + }; + consumed += hash_len; + delta + } + BLOB => Blob, + TREE => Tree, + COMMIT => Commit, + TAG => Tag, + other => return Err(io::Error::other(format!("Object type {other} is unsupported"))), + }; + Ok(data::Entry { + header: object, + decompressed_size: size, + data_offset: pack_offset + consumed as u64, + }) + } +} + +#[inline] +fn streaming_parse_header_info(read: &mut dyn io::Read) -> Result<(u8, u64, usize), io::Error> { + let mut byte = [0u8; 1]; + read.read_exact(&mut byte)?; + let mut c = byte[0]; + let mut i = 1; + let type_id = (c >> 4) & 0b0000_0111; + let mut size = u64::from(c) & 0b0000_1111; + let mut shift = 4u32; + while c & 0b1000_0000 != 0 { + read.read_exact(&mut byte)?; + c = byte[0]; + i += 1; + let component = u64::from(c & 0b0111_1111) + .checked_shl(shift) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "pack entry header overflowed"))?; + size = size + .checked_add(component) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "pack entry header overflowed"))?; + shift += 7; + } + if i != encoded_pack_entry_header_size(size) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "pack entry header uses a non-canonical size encoding", + )); + } + Ok((type_id, size, i)) +} + +/// Parses the header of a pack-entry, yielding object type id, decompressed object size, and consumed bytes +#[inline] +fn parse_header_info(data: &[u8]) -> Result<(u8, u64, usize), Error> { + let mut c = *data.first().ok_or(Error::Corrupt { + message: "need a pack entry header, got empty input", + })?; + let mut i = 1; + let type_id = (c >> 4) & 0b0000_0111; + let mut size = u64::from(c) & 0b0000_1111; + let mut shift = 4u32; + while c & 0b1000_0000 != 0 { + c = *data.get(i).ok_or(Error::Corrupt { + message: "pack entry header continuation byte", + })?; + i += 1; + let component = u64::from(c & 0b0111_1111).checked_shl(shift).ok_or(Error::Overflow)?; + size = size.checked_add(component).ok_or(Error::Overflow)?; + shift += 7; + } + if i != encoded_pack_entry_header_size(size) { + return Err(Error::Corrupt { + message: "pack entry header uses a non-canonical size encoding", + }); + } + Ok((type_id, size, i)) +} + +fn parse_leb64(data: &[u8]) -> Result<(u64, usize), Error> { + let mut i = 0; + let mut c = *data.first().ok_or(Error::Corrupt { + message: "an ofs-delta base distance", + })?; + i += 1; + let mut value = u64::from(c) & 0x7f; + while c & 0x80 != 0 { + c = *data.get(i).ok_or(Error::Corrupt { + message: "an ofs-delta base distance continuation byte", + })?; + i += 1; + value = value + .checked_add(1) + .and_then(|value| value.checked_shl(7)) + .and_then(|value| value.checked_add(u64::from(c) & 0x7f)) + .ok_or(Error::Overflow)?; + } + Ok((value, i)) +} + +/// Return the canonical byte length of a pack-entry size header for `size`. +/// +/// We use this to reject overlong size encodings during parsing. +/// That matters for our delta resolution implementation, which later reconstructs an entry's +/// pack offset from `data_offset - header_size()`. If we accepted non-canonical encodings here, +/// `header_size()` would compute the canonical length while `data_offset` would reflect the +/// actually consumed bytes, breaking that invariant and allowing malformed delta entries to point +/// back to themselves or otherwise walk the wrong base objects. +fn encoded_pack_entry_header_size(mut size: u64) -> usize { + let mut bytes = 1; + size >>= 4; + while size != 0 { + bytes += 1; + size >>= 7; + } + bytes +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_non_canonical_pack_entry_header_encoding() { + assert!(matches!( + data::Entry::from_bytes(&[0xed, 0x00], 0, gix_hash::Kind::Sha1.len_in_bytes()), + Err(Error::Corrupt { + message: "pack entry header uses a non-canonical size encoding" + }) + )); + } +} diff --git a/knot2/third_party/gix-pack/src/data/entry/header.rs b/knot2/third_party/gix-pack/src/data/entry/header.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/entry/header.rs @@ -0,0 +1,150 @@ +use std::io; + +use super::{BLOB, COMMIT, OFS_DELTA, REF_DELTA, TAG, TREE}; +use crate::data; + +/// The header portion of a pack data entry, identifying the kind of stored object. +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[allow(missing_docs)] +pub enum Header { + /// The object is a commit + Commit, + /// The object is a tree + Tree, + /// The object is a blob + Blob, + /// The object is a tag + Tag, + /// Describes a delta-object which needs to be applied to a base. The base object is identified by the `base_id` field + /// which is found within the parent repository. + /// Most commonly used for **thin-packs** when receiving pack files from the server to refer to objects that are not + /// part of the pack but expected to be present in the receivers repository. + /// + /// # Note + /// This could also be an object within this pack if the LSB encoded offset would be larger than 20 bytes, which is unlikely to + /// happen. + /// + /// **The naming** is exactly the same as the canonical implementation uses, namely **REF_DELTA**. + RefDelta { base_id: gix_hash::ObjectId }, + /// Describes a delta-object present in this pack which acts as base for this object. + /// The base object is measured as a distance from this objects + /// pack offset, so that `base_pack_offset = this_objects_pack_offset - base_distance` + /// + /// # Note + /// + /// **The naming** is exactly the same as the canonical implementation uses, namely **OFS_DELTA**. + OfsDelta { base_distance: u64 }, +} + +impl Header { + /// Subtract `distance` from `pack_offset` safely without the chance for overflow or no-ops if `distance` is 0. + pub fn verified_base_pack_offset(pack_offset: data::Offset, distance: u64) -> Option { + if distance == 0 { + return None; + } + pack_offset.checked_sub(distance) + } + /// Convert the header's object kind into [`gix_object::Kind`] if possible + pub fn as_kind(&self) -> Option { + use gix_object::Kind::*; + Some(match self { + Header::Tree => Tree, + Header::Blob => Blob, + Header::Commit => Commit, + Header::Tag => Tag, + Header::RefDelta { .. } | Header::OfsDelta { .. } => return None, + }) + } + /// Convert this header's object kind into the packs internal representation + pub fn as_type_id(&self) -> u8 { + use Header::*; + match self { + Blob => BLOB, + Tree => TREE, + Commit => COMMIT, + Tag => TAG, + OfsDelta { .. } => OFS_DELTA, + RefDelta { .. } => REF_DELTA, + } + } + /// Return's true if this is a delta object, i.e. not a full object. + pub fn is_delta(&self) -> bool { + matches!(self, Header::OfsDelta { .. } | Header::RefDelta { .. }) + } + /// Return's true if this is a base object, i.e. not a delta object. + pub fn is_base(&self) -> bool { + !self.is_delta() + } +} + +impl Header { + /// Encode this header along the given `decompressed_size_in_bytes` into the `out` write stream for use within a data pack. + /// + /// Returns the amount of bytes written to `out`. + /// `decompressed_size_in_bytes` is the full size in bytes of the object that this header represents + pub fn write_to(&self, decompressed_size_in_bytes: u64, out: &mut dyn io::Write) -> io::Result { + let mut size = decompressed_size_in_bytes; + let mut written = 1; + let mut c: u8 = (self.as_type_id() << 4) | (size as u8 & 0b0000_1111); + size >>= 4; + while size != 0 { + out.write_all(&[c | 0b1000_0000])?; + written += 1; + c = size as u8 & 0b0111_1111; + size >>= 7; + } + out.write_all(&[c])?; + + use Header::*; + match self { + RefDelta { base_id: oid } => { + out.write_all(oid.as_slice())?; + written += oid.as_slice().len(); + } + OfsDelta { base_distance } => { + let mut buf = [0u8; 10]; + let buf = leb64_encode(*base_distance, &mut buf); + out.write_all(buf)?; + written += buf.len(); + } + Blob | Tree | Commit | Tag => {} + } + Ok(written) + } + + /// The size of the header in bytes when serialized + pub fn size(&self, decompressed_size: u64) -> usize { + self.write_to(decompressed_size, &mut io::sink()) + .expect("io::sink() to never fail") + } +} + +#[inline] +fn leb64_encode(mut n: u64, buf: &mut [u8; 10]) -> &[u8] { + let mut bytes_written = 1; + buf[buf.len() - 1] = n as u8 & 0b0111_1111; + for out in buf.iter_mut().rev().skip(1) { + n >>= 7; + if n == 0 { + break; + } + n -= 1; + *out = 0b1000_0000 | (n as u8 & 0b0111_1111); + bytes_written += 1; + } + debug_assert_eq!(n, 0, "BUG: buffer must be large enough to hold a 64 bit integer"); + &buf[buf.len() - bytes_written..] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn leb64_encode_max_int() { + let mut buf = [0u8; 10]; + let buf = leb64_encode(u64::MAX, &mut buf); + assert_eq!(buf.len(), 10, "10 bytes should be used when 64bits are encoded"); + } +} diff --git a/knot2/third_party/gix-pack/src/data/entry/mod.rs b/knot2/third_party/gix-pack/src/data/entry/mod.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/entry/mod.rs @@ -0,0 +1,65 @@ +use crate::data::Entry; + +const _TYPE_EXT1: u8 = 0; +const COMMIT: u8 = 1; +const TREE: u8 = 2; +const BLOB: u8 = 3; +const TAG: u8 = 4; +const _TYPE_EXT2: u8 = 5; +const OFS_DELTA: u8 = 6; +const REF_DELTA: u8 = 7; + +/// A way to uniquely identify the location of an entry within a pack bundle +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Location { + /// The id of the pack containing the object. It's unique within its frame of reference which is the owning object database. + pub pack_id: u32, + /// The size of the entry of disk so that the range of bytes of the entry is `pack_offset..pack_offset + entry_size`. + pub entry_size: usize, + /// The start of the entry in the pack identified by `pack_id`. + pub pack_offset: data::Offset, +} + +impl Location { + /// Compute a range suitable for lookup in pack data using the [`entry_slice()`][crate::data::File::entry_slice()] method. + pub fn entry_range(&self, pack_offset: data::Offset) -> crate::data::EntryRange { + pack_offset..pack_offset + self.entry_size as u64 + } +} + +/// Access +impl Entry { + /// Compute the pack offset to the base entry of the object represented by this entry, or + /// return `None` if the distance would underflow or is invalid. + pub fn checked_base_pack_offset(&self, distance: u64) -> Option { + let pack_offset = self.data_offset - self.header_size() as u64; + Header::verified_base_pack_offset(pack_offset, distance) + } + + /// Compute the pack offset to the base entry of the object represented by this entry. + /// + /// # Panics + /// + /// Panics if the `distance` will cause an underflow or is invalid. + pub fn base_pack_offset(&self, distance: u64) -> data::Offset { + self.checked_base_pack_offset(distance) + .expect("in-bound distance of deltas") + } + /// The pack offset at which this entry starts + pub fn pack_offset(&self) -> data::Offset { + self.data_offset - self.header_size() as u64 + } + /// The amount of bytes used to describe this entry in the pack. The header starts at [`Self::pack_offset()`] + pub fn header_size(&self) -> usize { + self.header.size(self.decompressed_size) + } +} + +/// +pub mod decode; + +mod header; +pub use header::Header; + +use crate::data; diff --git a/knot2/third_party/gix-pack/src/data/file/init.rs b/knot2/third_party/gix-pack/src/data/file/init.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/file/init.rs @@ -0,0 +1,73 @@ +use std::path::Path; + +use crate::data; + +/// Instantiation +impl data::File { + /// Try opening a data file at the given `path`. + /// + /// The `object_hash` is a way to read (and write) the same file format with different hashes, as the hash kind + /// isn't stored within the file format itself. + /// + /// This constructor leaves allocation limiting disabled, allowing allocations of any size dictated by pack data. + /// Call [`File::with_alloc_limit_bytes()`][crate::data::File::with_alloc_limit_bytes()] before decoding entries from untrusted input. + pub fn at(path: impl AsRef, object_hash: gix_hash::Kind) -> Result { + Self::at_inner(path.as_ref(), object_hash) + } + + fn at_inner(path: &Path, object_hash: gix_hash::Kind) -> Result { + use std::os::unix::fs::FileExt; + + use crate::data::header::N32_SIZE; + let hash_len = object_hash.len_in_bytes(); + let file = std::fs::File::open(path).map_err(|e| data::header::decode::Error::Io { + source: e, + path: path.to_owned(), + })?; + let pack_len = file + .metadata() + .map_err(|e| data::header::decode::Error::Io { + source: e, + path: path.to_owned(), + })? + .len(); + let pack_len = usize::try_from(pack_len).map_err(|_| { + data::header::decode::Error::Corrupt(format!("Pack data of size {pack_len} is too large for this machine")) + })?; + if pack_len < N32_SIZE * 3 + hash_len { + return Err(data::header::decode::Error::Corrupt(format!( + "Pack data of size {pack_len} is too small for even an empty pack with shortest hash" + ))); + } + let mut header = [0u8; 12]; + file.read_exact_at(&mut header, 0).map_err(|e| data::header::decode::Error::Io { + source: e, + path: path.to_owned(), + })?; + let (version, num_objects) = data::header::decode(&header)?; + let id = gix_features::hash::crc32(path.as_os_str().to_string_lossy().as_bytes()); + Ok(Self { + file, + len: pack_len, + path: path.to_owned(), + id, + version, + num_objects, + hash_len, + object_hash, + alloc_limit_bytes: None, + }) + } + + /// Configure the maximum size of a single allocation caused by user-controlled on-disk pack data. + /// + /// Use `None` to disable the limit, which is also the default. + /// + /// This is currently enforced when decoding pack entries and resolving delta chains. + /// Callers that allocate from pack metadata directly should consult [`File::alloc_limit_bytes()`][crate::data::File::alloc_limit_bytes()] + /// and apply the same limit themselves. + pub fn with_alloc_limit_bytes(mut self, alloc_limit_bytes: Option) -> Self { + self.alloc_limit_bytes = alloc_limit_bytes; + self + } +} diff --git a/knot2/third_party/gix-pack/src/data/file/mod.rs b/knot2/third_party/gix-pack/src/data/file/mod.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/file/mod.rs @@ -0,0 +1,9 @@ +mod init; +/// +pub mod verify; + +/// +pub mod decode; + +/// The bytes used as header in a pack data file. +pub type Header = [u8; 12]; diff --git a/knot2/third_party/gix-pack/src/data/file/verify.rs b/knot2/third_party/gix-pack/src/data/file/verify.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/file/verify.rs @@ -0,0 +1,60 @@ +use std::sync::atomic::AtomicBool; + +use gix_features::progress::Progress; + +use crate::data::File; + +/// +pub mod checksum { + /// Returned by [`data::File::verify_checksum()`][crate::data::File::verify_checksum()]. + pub type Error = crate::verify::checksum::Error; +} + +/// Checksums and verify checksums +impl File { + /// The checksum in the trailer of this pack data file + pub fn checksum(&self) -> gix_hash::ObjectId { + let trailer = self + .read_span((self.data_len() - self.object_hash.len_in_bytes()) as u64..self.data_len() as u64) + .expect("pack trailer is within the pack data"); + gix_hash::ObjectId::from_bytes_or_panic(&trailer) + } + + /// Verifies that the checksum of the packfile over all bytes preceding it indeed matches the actual checksum, + /// returning the actual checksum equivalent to the return value of [`checksum()`][File::checksum()] if there + /// is no mismatch. + /// + /// Note that if no `progress` is desired, one can pass [`gix_features::progress::Discard`]. + /// + /// Have a look at [`index::File::verify_integrity(…)`][crate::index::File::verify_integrity()] for an + /// even more thorough integrity check. + pub fn verify_checksum( + &self, + progress: &mut dyn Progress, + should_interrupt: &AtomicBool, + ) -> Result { + let expected = self.checksum(); + let body_len = (self.data_len() - self.hash_len) as u64; + let actual = match gix_hash::bytes_of_file( + self.path(), + body_len, + self.object_hash, + progress, + should_interrupt, + ) { + Ok(id) => id, + Err(gix_hash::io::Error::Io(err)) if err.kind() == std::io::ErrorKind::Interrupted => { + return Err(checksum::Error::Interrupted); + } + Err(gix_hash::io::Error::Io(_)) => { + let data = self.materialized()?; + let mut hasher = gix_hash::hasher(self.object_hash); + hasher.update(&data[..body_len as usize]); + hasher.try_finalize()? + } + Err(gix_hash::io::Error::Hasher(err)) => return Err(checksum::Error::Hasher(err)), + }; + actual.verify(&expected)?; + Ok(actual) + } +} diff --git a/knot2/third_party/gix-pack/src/data/input/bytes_to_entries.rs b/knot2/third_party/gix-pack/src/data/input/bytes_to_entries.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/input/bytes_to_entries.rs @@ -0,0 +1,325 @@ +use std::{fs, io}; + +use gix_features::zlib::Decompress; +use gix_hash::{Hasher, ObjectId}; + +use crate::data::input; + +/// An iterator over [`Entries`][input::Entry] in a byte stream. +/// +/// The iterator used as part of [`Bundle::write_to_directory(…)`][crate::Bundle::write_to_directory()]. +pub struct BytesToEntriesIter
{ + read: BR, + decompressor: Decompress, + offset: u64, + had_error: bool, + version: crate::data::Version, + objects_left: u32, + hash: Option, + mode: input::Mode, + compressed: input::EntryDataMode, + compressed_buf: Option>, + hash_len: usize, + object_hash: gix_hash::Kind, +} + +/// Access +impl
BytesToEntriesIter
{ + /// The pack version currently being iterated + pub fn version(&self) -> crate::data::Version { + self.version + } + + /// The kind of iteration + pub fn mode(&self) -> input::Mode { + self.mode + } +} + +/// Initialization +impl
BytesToEntriesIter
+where + BR: io::BufRead, +{ + /// Obtain an iterator from a `read` stream to a pack data file and configure it using `mode` and `compressed`. + /// `object_hash` specifies which hash is used for objects in ref-delta entries. + /// + /// Note that `read` is expected at the beginning of a valid pack data file with a header, entries and a trailer. + pub fn new_from_header( + mut read: BR, + mode: input::Mode, + compressed: input::EntryDataMode, + object_hash: gix_hash::Kind, + ) -> Result, input::Error> { + let mut header_data = [0u8; 12]; + read.read_exact(&mut header_data).map_err(gix_hash::io::Error::from)?; + + let (version, num_objects) = crate::data::header::decode(&header_data)?; + match version { + crate::data::Version::V2 => {} + crate::data::Version::V3 => { + return Err(crate::data::header::decode::Error::UnsupportedVersion(3).into()); + } + } + Ok(BytesToEntriesIter { + read, + decompressor: Decompress::new(), + compressed, + offset: 12, + had_error: false, + version, + objects_left: num_objects, + hash: (mode != input::Mode::AsIs).then(|| { + let mut hash = gix_hash::hasher(object_hash); + hash.update(&header_data); + hash + }), + mode, + compressed_buf: None, + hash_len: object_hash.len_in_bytes(), + object_hash, + }) + } + + fn next_inner(&mut self) -> Result { + self.objects_left -= 1; // even an error counts as objects + + // Read header + let entry = match self.hash.as_mut() { + Some(hash) => { + let mut read = read_and_pass_to( + &mut self.read, + HashWrite { + inner: io::sink(), + hash, + }, + ); + crate::data::Entry::from_read(&mut read, self.offset, self.hash_len) + } + None => crate::data::Entry::from_read(&mut self.read, self.offset, self.hash_len), + } + .map_err(gix_hash::io::Error::from)?; + + // Decompress object to learn its compressed bytes + let compressed_buf = self.compressed_buf.take().unwrap_or_else(|| Vec::with_capacity(4096)); + self.decompressor.reset(); + let mut decompressed_reader = DecompressRead { + inner: read_and_pass_to( + &mut self.read, + if self.compressed.keep() { + Vec::with_capacity(entry.decompressed_size.min(65_536) as usize) + } else { + compressed_buf + }, + ), + decompressor: &mut self.decompressor, + }; + + let bytes_copied = io::copy(&mut decompressed_reader, &mut io::sink()).map_err(gix_hash::io::Error::from)?; + if bytes_copied != entry.decompressed_size { + return Err(input::Error::IncompletePack { + actual: bytes_copied, + expected: entry.decompressed_size, + }); + } + + let pack_offset = self.offset; + let compressed_size = decompressed_reader.decompressor.total_in(); + self.offset += entry.header_size() as u64 + compressed_size; + + let mut compressed = decompressed_reader.inner.write; + debug_assert_eq!( + compressed_size, + compressed.len() as u64, + "we must track exactly the same amount of bytes as read by the decompressor" + ); + if let Some(hash) = self.hash.as_mut() { + hash.update(&compressed); + } + + let crc32 = if self.compressed.crc32() { + let mut header_buf = [0u8; 12 + gix_hash::Kind::longest().len_in_bytes()]; + let header_len = entry + .header + .write_to(bytes_copied, &mut header_buf.as_mut()) + .map_err(gix_hash::io::Error::from)?; + let state = gix_features::hash::crc32_update(0, &header_buf[..header_len]); + Some(gix_features::hash::crc32_update(state, &compressed)) + } else { + None + }; + + let compressed = if self.compressed.keep() { + Some(compressed) + } else { + compressed.clear(); + self.compressed_buf = Some(compressed); + None + }; + + // Last objects gets trailer (which is potentially verified) + let trailer = self.try_read_trailer()?; + Ok(input::Entry { + header: entry.header, + header_size: entry.header_size() as u16, + compressed, + compressed_size, + crc32, + pack_offset, + decompressed_size: bytes_copied, + trailer, + }) + } + + fn try_read_trailer(&mut self) -> Result, input::Error> { + Ok(if self.objects_left == 0 { + let mut id = gix_hash::ObjectId::null(self.object_hash); + if let Err(err) = self.read.read_exact(id.as_mut_slice()) { + if self.mode != input::Mode::Restore { + return Err(input::Error::Io(err.into())); + } + } + + if let Some(hash) = self.hash.take() { + let actual_id = hash.try_finalize().map_err(gix_hash::io::Error::from)?; + if self.mode == input::Mode::Restore { + id = actual_id; + } else { + actual_id.verify(&id)?; + } + } + Some(id) + } else if self.mode == input::Mode::Restore { + let hash = self.hash.clone().expect("in restore mode a hash is set"); + Some(hash.try_finalize().map_err(gix_hash::io::Error::from)?) + } else { + None + }) + } +} + +fn read_and_pass_to(read: &mut R, to: W) -> PassThrough<&mut R, W> { + PassThrough { read, write: to } +} + +impl Iterator for BytesToEntriesIter +where + R: io::BufRead, +{ + type Item = Result; + + fn next(&mut self) -> Option { + if self.had_error || self.objects_left == 0 { + return None; + } + let result = self.next_inner(); + self.had_error = result.is_err(); + if self.had_error { + self.objects_left = 0; + } + if self.mode == input::Mode::Restore && self.had_error { + None + } else { + Some(result) + } + } + + fn size_hint(&self) -> (usize, Option) { + (self.objects_left as usize, Some(self.objects_left as usize)) + } +} + +impl std::iter::ExactSizeIterator for BytesToEntriesIter where R: io::BufRead {} + +struct PassThrough { + read: R, + write: W, +} + +impl io::BufRead for PassThrough +where + Self: io::Read, + R: io::BufRead, + W: io::Write, +{ + fn fill_buf(&mut self) -> io::Result<&[u8]> { + self.read.fill_buf() + } + + fn consume(&mut self, amt: usize) { + let buf = self + .read + .fill_buf() + .expect("never fail as we called fill-buf before and this does nothing"); + self.write + .write_all(&buf[..amt]) + .expect("a write to never fail - should be a memory buffer"); + self.read.consume(amt); + } +} + +impl io::Read for PassThrough +where + W: io::Write, + R: io::Read, +{ + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let bytes_read = self.read.read(buf)?; + self.write.write_all(&buf[..bytes_read])?; + Ok(bytes_read) + } +} + +impl crate::data::File { + /// Returns an iterator over [`Entries`][crate::data::input::Entry], without making use of the memory mapping. + pub fn streaming_iter(&self) -> Result, input::Error> { + let reader = + io::BufReader::with_capacity(4096 * 8, fs::File::open(&self.path).map_err(gix_hash::io::Error::from)?); + BytesToEntriesIter::new_from_header( + reader, + input::Mode::Verify, + input::EntryDataMode::KeepAndCrc32, + self.object_hash, + ) + } +} + +/// The boxed variant is faster for what we do (moving the decompressor in and out a lot) +pub struct DecompressRead<'a, R> { + /// The reader from which bytes should be decompressed. + pub inner: R, + /// The decompressor doing all the work. + pub decompressor: &'a mut Decompress, +} + +impl io::Read for DecompressRead<'_, R> +where + R: io::BufRead, +{ + fn read(&mut self, into: &mut [u8]) -> io::Result { + gix_features::zlib::stream::inflate::read(&mut self.inner, self.decompressor, into) + } +} + +/// A utility to automatically generate a hash while writing into an inner writer. +pub struct HashWrite<'a, T> { + /// The hash implementation. + pub hash: &'a mut Hasher, + /// The inner writer. + pub inner: T, +} + +impl std::io::Write for HashWrite<'_, T> +where + T: std::io::Write, +{ + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let written = self.inner.write(buf)?; + self.hash.update(&buf[..written]); + Ok(written) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.inner.flush() + } +} diff --git a/knot2/third_party/gix-pack/src/data/input/entries_to_bytes.rs b/knot2/third_party/gix-pack/src/data/input/entries_to_bytes.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/input/entries_to_bytes.rs @@ -0,0 +1,151 @@ +use std::iter::Peekable; + +use crate::data::input; + +/// An implementation of [`Iterator`] to write [encoded entries][input::Entry] to an inner implementation each time +/// `next()` is called. +/// +/// It is able to deal with an unknown amount of objects as it will rewrite the pack header once the entries iterator +/// is depleted and compute the hash in one go by re-reading the whole file. +pub struct EntriesToBytesIter { + /// An iterator for input [`input::Entry`] instances + pub input: Peekable, + /// A way of writing encoded bytes. + output: W, + /// Our trailing hash when done writing all input entries + trailer: Option, + /// The amount of objects in the iteration and the version of the packfile to be written. + /// Will be `None` to signal the header was written already. + data_version: crate::data::Version, + /// The amount of entries seen so far + num_entries: u32, + /// If we are done, no additional writes will occur + is_done: bool, + /// The kind of hash to use for the digest + object_hash: gix_hash::Kind, +} + +impl EntriesToBytesIter +where + I: Iterator>, + W: std::io::Read + std::io::Write + std::io::Seek, +{ + /// Create a new instance reading [entries][input::Entry] from an `input` iterator and write pack data bytes to + /// `output` writer, resembling a pack of `version`. The amount of entries will be dynamically determined and + /// the pack is completed once the last entry was written. + /// `object_hash` is the kind of hash to use for the pack checksum and maybe other places, depending on the version. + /// + /// # Panics + /// + /// Only [Version::V2](crate::data::Version::V2) is allowed for `version. + pub fn new(input: I, output: W, version: crate::data::Version, object_hash: gix_hash::Kind) -> Self { + assert!( + matches!(version, crate::data::Version::V2), + "currently only pack version 2 can be written", + ); + EntriesToBytesIter { + input: input.peekable(), + output, + object_hash, + num_entries: 0, + trailer: None, + data_version: version, + is_done: false, + } + } + + /// Returns the trailing hash over all ~ entries once done. + /// It's `None` if we are not yet done writing. + pub fn digest(&self) -> Option { + self.trailer + } + + fn next_inner(&mut self, entry: input::Entry) -> Result { + if self.num_entries == 0 { + let header_bytes = crate::data::header::encode(self.data_version, 0); + self.output.write_all(&header_bytes[..])?; + } + self.num_entries += 1; + entry.header.write_to(entry.decompressed_size, &mut self.output)?; + self.output.write_all( + entry + .compressed + .as_deref() + .expect("caller must configure generator to keep compressed bytes"), + )?; + Ok(entry) + } + + fn write_header_and_digest(&mut self, last_entry: Option<&mut input::Entry>) -> Result<(), gix_hash::io::Error> { + let header_bytes = crate::data::header::encode(self.data_version, self.num_entries); + let num_bytes_written = if last_entry.is_some() { + self.output.stream_position()? + } else { + header_bytes.len() as u64 + }; + self.output.rewind()?; + self.output.write_all(&header_bytes[..])?; + self.output.flush()?; + + self.output.rewind()?; + let interrupt_never = std::sync::atomic::AtomicBool::new(false); + let digest = gix_hash::bytes( + &mut self.output, + num_bytes_written, + self.object_hash, + &mut gix_features::progress::Discard, + &interrupt_never, + )?; + self.output.write_all(digest.as_slice())?; + self.output.flush()?; + + self.is_done = true; + if let Some(last_entry) = last_entry { + last_entry.trailer = Some(digest); + } + self.trailer = Some(digest); + Ok(()) + } +} + +impl Iterator for EntriesToBytesIter +where + I: Iterator>, + W: std::io::Read + std::io::Write + std::io::Seek, +{ + /// The amount of bytes written to `out` if `Ok` or the error `E` received from the input. + type Item = Result; + + fn next(&mut self) -> Option { + if self.is_done { + return None; + } + + match self.input.next() { + Some(res) => Some(match res { + Ok(entry) => self + .next_inner(entry) + .and_then(|mut entry| { + if self.input.peek().is_none() { + self.write_header_and_digest(Some(&mut entry)).map(|_| entry) + } else { + Ok(entry) + } + }) + .map_err(input::Error::from), + Err(err) => { + self.is_done = true; + Err(err) + } + }), + None => match self.write_header_and_digest(None) { + Ok(_) => None, + Err(err) => Some(Err(err.into())), + }, + } + } + + fn size_hint(&self) -> (usize, Option) { + self.input.size_hint() + } +} diff --git a/knot2/third_party/gix-pack/src/data/input/entry.rs b/knot2/third_party/gix-pack/src/data/input/entry.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/input/entry.rs @@ -0,0 +1,65 @@ +use std::io::Write; + +use crate::data::{entry::Header, input}; + +impl input::Entry { + /// Create a new input entry from a given data `obj` set to be placed at the given `pack_offset`. + /// + /// This method is useful when arbitrary base entries are created + pub fn from_data_obj(obj: &gix_object::Data<'_>, pack_offset: u64) -> Result { + let header = to_header(obj.kind); + let compressed = compress_data(obj)?; + let compressed_size = compressed.len() as u64; + let mut entry = input::Entry { + header, + header_size: header.size(obj.data.len() as u64) as u16, + pack_offset, + compressed: Some(compressed), + compressed_size, + crc32: None, + decompressed_size: obj.data.len() as u64, + trailer: None, + }; + entry.crc32 = Some(entry.compute_crc32()); + Ok(entry) + } + /// The amount of bytes this entry may consume in a pack data file + pub fn bytes_in_pack(&self) -> u64 { + u64::from(self.header_size) + self.compressed_size + } + + /// Update our CRC value by recalculating it from our header and compressed data. + pub fn compute_crc32(&self) -> u32 { + let mut header_buf = [0u8; 12 + gix_hash::Kind::longest().len_in_bytes()]; + let header_len = self + .header + .write_to(self.decompressed_size, &mut header_buf.as_mut()) + .expect("write to memory will not fail"); + let state = gix_features::hash::crc32_update(0, &header_buf[..header_len]); + gix_features::hash::crc32_update(state, self.compressed.as_ref().expect("we always set it")) + } +} + +fn to_header(kind: gix_object::Kind) -> Header { + use gix_object::Kind::*; + match kind { + Tree => Header::Tree, + Blob => Header::Blob, + Commit => Header::Commit, + Tag => Header::Tag, + } +} + +fn compress_data(obj: &gix_object::Data<'_>) -> Result, input::Error> { + let mut out = gix_features::zlib::stream::deflate::Write::new(Vec::new()); + if let Err(err) = std::io::copy(&mut &*obj.data, &mut out) { + match err.kind() { + std::io::ErrorKind::Other => return Err(input::Error::Io(err.into())), + err => { + unreachable!("Should never see other errors than zlib, but got {:?}", err) + } + } + } + out.flush().expect("zlib flush should never fail"); + Ok(out.into_inner()) +} diff --git a/knot2/third_party/gix-pack/src/data/input/lookup_ref_delta_objects.rs b/knot2/third_party/gix-pack/src/data/input/lookup_ref_delta_objects.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/input/lookup_ref_delta_objects.rs @@ -0,0 +1,203 @@ +use gix_hash::ObjectId; + +use crate::data::{entry::Header, input}; + +/// An iterator to resolve thin packs on the fly. +pub struct LookupRefDeltaObjectsIter { + /// The inner iterator whose entries we will resolve. + pub inner: I, + lookup: Find, + /// The cached delta to provide next time we are called, it's the delta to go with the base we just resolved in its place. + next_delta: Option, + /// Fuse to stop iteration after first missing object. + error: bool, + /// The overall pack-offset we accumulated thus far. Each inserted entry offsets all following + /// objects by its length. We need to determine exactly where the object was inserted to see if its affected at all. + inserted_entry_length_at_offset: Vec, + /// The sum of all entries added so far, as a cache to avoid recomputation + inserted_entries_length_in_bytes: i64, + buf: Vec, +} + +impl LookupRefDeltaObjectsIter +where + I: Iterator>, + Find: gix_object::Find, +{ + /// Create a new instance wrapping `iter` and using `lookup` as function to retrieve objects that will serve as bases + /// for ref deltas seen while traversing `iter`. + pub fn new(iter: I, lookup: Find) -> Self { + LookupRefDeltaObjectsIter { + inner: iter, + lookup, + error: false, + inserted_entry_length_at_offset: Vec::new(), + inserted_entries_length_in_bytes: 0, + next_delta: None, + buf: Vec::new(), + } + } + + fn shifted_pack_offset(&self, pack_offset: u64) -> u64 { + let new_ofs = pack_offset as i64 + self.inserted_entries_length_in_bytes; + new_ofs.try_into().expect("offset value is never becomes negative") + } + + /// positive `size_change` values mean an object grew or was more commonly, was inserted. Negative values + /// mean the object shrunk, usually because there header changed from ref-deltas to ofs deltas. + fn track_change(&mut self, shifted_pack_offset: u64, pack_offset: u64, size_change: i64, oid: Option) { + if size_change == 0 { + return; + } + self.inserted_entry_length_at_offset.push(Change { + shifted_pack_offset, + pack_offset, + size_change_in_bytes: size_change, + oid: oid.unwrap_or_else(|| + // NOTE: this value acts as sentinel and the actual hash kind doesn't matter. + gix_hash::Kind::shortest().null()), + }); + self.inserted_entries_length_in_bytes += size_change; + } + + fn shift_entry_and_point_to_base_by_offset(&mut self, entry: &mut input::Entry, base_distance: u64) { + let pack_offset = entry.pack_offset; + entry.pack_offset = self.shifted_pack_offset(pack_offset); + entry.header = Header::OfsDelta { base_distance }; + let previous_header_size = entry.header_size; + entry.header_size = entry.header.size(entry.decompressed_size) as u16; + + let change = i64::from(entry.header_size) - i64::from(previous_header_size); + entry.crc32 = Some(entry.compute_crc32()); + self.track_change(entry.pack_offset, pack_offset, change, None); + } +} + +impl Iterator for LookupRefDeltaObjectsIter +where + I: Iterator>, + Find: gix_object::Find, +{ + type Item = Result; + + fn next(&mut self) -> Option { + if self.error { + return None; + } + if let Some(delta) = self.next_delta.take() { + return Some(Ok(delta)); + } + match self.inner.next() { + Some(Ok(mut entry)) => match entry.header { + Header::RefDelta { base_id } => { + match self.inserted_entry_length_at_offset.iter().rfind(|e| e.oid == base_id) { + None => { + let base_entry = match self.lookup.try_find(&base_id, &mut self.buf).ok()? { + Some(obj) => { + let current_pack_offset = entry.pack_offset; + let mut entry = match input::Entry::from_data_obj(&obj, 0) { + Ok(e) => e, + Err(err) => return Some(Err(err)), + }; + entry.pack_offset = self.shifted_pack_offset(current_pack_offset); + self.track_change( + entry.pack_offset, + current_pack_offset, + entry.bytes_in_pack() as i64, + Some(base_id), + ); + entry + } + None => { + self.error = true; + return Some(Err(input::Error::NotFound { object_id: base_id })); + } + }; + + { + self.shift_entry_and_point_to_base_by_offset(&mut entry, base_entry.bytes_in_pack()); + self.next_delta = Some(entry); + } + Some(Ok(base_entry)) + } + Some(base_entry) => { + let base_distance = + self.shifted_pack_offset(entry.pack_offset) - base_entry.shifted_pack_offset; + self.shift_entry_and_point_to_base_by_offset(&mut entry, base_distance); + Some(Ok(entry)) + } + } + } + _ => { + if self.inserted_entries_length_in_bytes != 0 { + if let Header::OfsDelta { base_distance } = entry.header { + // We have to find the new distance based on the previous distance to the base, using the absolute + // pack offset computed from it as stored in `base_pack_offset`. + let base_pack_offset = entry + .pack_offset + .checked_sub(base_distance) + .expect("distance to be in range of pack"); + match self + .inserted_entry_length_at_offset + .binary_search_by_key(&base_pack_offset, |c| c.pack_offset) + { + Ok(index) => { + let index = { + let maybe_index_of_actual_entry = index + 1; + self.inserted_entry_length_at_offset + .get(maybe_index_of_actual_entry) + .and_then(|c| { + (c.pack_offset == base_pack_offset) + .then_some(maybe_index_of_actual_entry) + }) + .unwrap_or(index) + }; + let new_distance = self + .shifted_pack_offset(entry.pack_offset) + .checked_sub(self.inserted_entry_length_at_offset[index].shifted_pack_offset) + .expect("a base that is behind us in the pack"); + self.shift_entry_and_point_to_base_by_offset(&mut entry, new_distance); + } + Err(index) => { + let change_since_offset = self.inserted_entry_length_at_offset[index..] + .iter() + .map(|c| c.size_change_in_bytes) + .sum::(); + let new_distance: u64 = { + (base_distance as i64 + change_since_offset) + .try_into() + .expect("it still points behind us") + }; + self.shift_entry_and_point_to_base_by_offset(&mut entry, new_distance); + } + } + } else { + // Offset this entry by all changes (positive or negative) that we saw thus far. + entry.pack_offset = self.shifted_pack_offset(entry.pack_offset); + } + } + Some(Ok(entry)) + } + }, + other => other, + } + } + + fn size_hint(&self) -> (usize, Option) { + let (min, max) = self.inner.size_hint(); + max.map_or_else(|| (min * 2, None), |max| (min, Some(max * 2))) + } +} + +#[derive(Debug)] +struct Change { + /// The original pack offset as mentioned in the entry we saw. This is used to find this as base object if deltas refer to it by + /// old offset. + pack_offset: u64, + /// The new pack offset that is the shifted location of the pack entry in the pack. + shifted_pack_offset: u64, + /// The size change of the entry header, negative values denote shrinking, positive denote growing. + size_change_in_bytes: i64, + /// The object id of the entry responsible for the change, or null if it's an entry just for tracking an insertion. + oid: ObjectId, +} diff --git a/knot2/third_party/gix-pack/src/data/input/mod.rs b/knot2/third_party/gix-pack/src/data/input/mod.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/input/mod.rs @@ -0,0 +1,41 @@ +/// An item of the iteration produced by [`BytesToEntriesIter`] +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Entry { + /// The header of a pack entry + pub header: crate::data::entry::Header, + /// The amount of bytes used to encode the `header`. `pack_offset + header_size` is the beginning of + /// the compressed data in the pack. + pub header_size: u16, + /// The first byte of the entry at which the `header` can be read. + pub pack_offset: u64, + /// The bytes consumed while producing `decompressed` + /// These do not contain the header, which makes it possible to easily replace a RefDelta with offset deltas + /// when resolving thin packs. + /// Depends on `CompressionMode` when the iterator is initialized. + pub compressed: Option>, + /// The amount of bytes the compressed portion of the entry takes, i.e. the portion behind the header. + pub compressed_size: u64, + /// The CRC32 over the complete entry, that is encoded header and compressed object data. + /// Depends on `CompressionMode` when the iterator is initialized + pub crc32: Option, + /// The amount of decompressed bytes of the entry. + pub decompressed_size: u64, + /// Set for the last object in the iteration, providing the hash over all bytes of the iteration + /// for use as trailer in a pack or to verify it matches the trailer. + pub trailer: Option, +} + +mod entry; + +mod types; +pub use types::{EntryDataMode, Error, Mode}; + +mod bytes_to_entries; +pub use bytes_to_entries::BytesToEntriesIter; + +mod lookup_ref_delta_objects; +pub use lookup_ref_delta_objects::LookupRefDeltaObjectsIter; + +mod entries_to_bytes; +pub use entries_to_bytes::EntriesToBytesIter; diff --git a/knot2/third_party/gix-pack/src/data/input/types.rs b/knot2/third_party/gix-pack/src/data/input/types.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/input/types.rs @@ -0,0 +1,68 @@ +/// Returned by [`BytesToEntriesIter::new_from_header()`][crate::data::input::BytesToEntriesIter::new_from_header()] and as part +/// of `Item` of [`BytesToEntriesIter`][crate::data::input::BytesToEntriesIter]. +#[derive(thiserror::Error, Debug)] +#[allow(missing_docs)] +pub enum Error { + #[error("An IO operation failed while streaming an entry")] + Io(#[from] gix_hash::io::Error), + #[error(transparent)] + PackParse(#[from] crate::data::header::decode::Error), + #[error("Failed to verify pack checksum in trailer")] + Verify(#[from] gix_hash::verify::Error), + #[error("pack is incomplete: it was decompressed into {actual} bytes but {expected} bytes where expected.")] + IncompletePack { actual: u64, expected: u64 }, + #[error("The object {object_id} could not be decoded or wasn't found")] + NotFound { object_id: gix_hash::ObjectId }, +} + +/// Iteration Mode +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum Mode { + /// Provide the trailer as read from the pack + AsIs, + /// Generate an own hash and trigger an error on the last iterated object + /// if it does not match the hash provided with the pack. + /// + /// This way the one iterating the data cannot miss corruption as long as + /// the iteration is continued through to the end. + Verify, + /// Generate an own hash and if there was an error or the objects are depleted early + /// due to partial packs, return the last valid entry and with our own hash thus far. + /// Note that the existing pack hash, if present, will be ignored. + /// As we won't know which objects fails, every object will have the hash obtained thus far. + /// This also means that algorithms must know about this possibility, or else might wrongfully + /// assume the pack is finished. + Restore, +} + +/// Define what to do with the compressed bytes portion of a pack [`Entry`][super::Entry] +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum EntryDataMode { + /// Do nothing with the compressed bytes we read + Ignore, + /// Only create a CRC32 of the entry, otherwise similar to `Ignore` + Crc32, + /// Keep them and pass them along in a newly allocated buffer + Keep, + /// As above, but also compute a CRC32 + KeepAndCrc32, +} + +impl EntryDataMode { + /// Returns true if a crc32 should be computed + pub fn crc32(&self) -> bool { + match self { + EntryDataMode::KeepAndCrc32 | EntryDataMode::Crc32 => true, + EntryDataMode::Keep | EntryDataMode::Ignore => false, + } + } + /// Returns true if compressed bytes should be kept + pub fn keep(&self) -> bool { + match self { + EntryDataMode::Keep | EntryDataMode::KeepAndCrc32 => true, + EntryDataMode::Ignore | EntryDataMode::Crc32 => false, + } + } +} diff --git a/knot2/third_party/gix-pack/src/data/output/bytes.rs b/knot2/third_party/gix-pack/src/data/output/bytes.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/output/bytes.rs @@ -0,0 +1,167 @@ +use std::io::Write; + +use crate::{data::output, exact_vec}; + +/// The error returned by `next()` in the [`FromEntriesIter`] iterator. +#[allow(missing_docs)] +#[derive(Debug, thiserror::Error)] +pub enum Error +where + E: std::error::Error + 'static, +{ + #[error(transparent)] + Io(#[from] gix_hash::io::Error), + #[error(transparent)] + Input(E), +} + +/// An implementation of [`Iterator`] to write [encoded entries][output::Entry] to an inner implementation each time +/// `next()` is called. +pub struct FromEntriesIter { + /// An iterator for input [`output::Entry`] instances + pub input: I, + /// A way of writing encoded bytes. + output: gix_hash::io::Write, + /// Our trailing hash when done writing all input entries + trailer: Option, + /// The amount of objects in the iteration and the version of the packfile to be written. + /// Will be `None` to signal the header was written already. + header_info: Option<(crate::data::Version, u32)>, + /// The pack data version with which pack entries should be written. + entry_version: crate::data::Version, + /// The amount of written bytes thus far + written: u64, + /// Required to quickly find offsets by object IDs, as future objects may refer to those in the past to become a delta offset base. + /// It stores the pack offsets at which objects begin. + /// Additionally we store if an object was invalid, and if so we will not write it nor will we allow delta objects to it. + pack_offsets_and_validity: Vec<(u64, bool)>, + /// If we are done, no additional writes will occur + is_done: bool, +} + +impl FromEntriesIter +where + I: Iterator, E>>, + W: std::io::Write, + E: std::error::Error + 'static, +{ + /// Create a new instance reading [entries][output::Entry] from an `input` iterator and write pack data bytes to + /// `output` writer, resembling a pack of `version` with exactly `num_entries` amount of objects contained in it. + /// `object_hash` is the kind of hash to use for the pack checksum and maybe other places, depending on the version. + /// + /// The input chunks are expected to be sorted already. You can use the [`InOrderIter`][gix_features::parallel::InOrderIter] to assure + /// this happens on the fly holding entire chunks in memory as long as needed for them to be dispensed in order. + /// + /// # Panics + /// + /// Not all combinations of `object_hash` and `version` are supported currently triggering assertion errors. + pub fn new( + input: I, + output: W, + num_entries: u32, + version: crate::data::Version, + object_hash: gix_hash::Kind, + ) -> Self { + assert!( + matches!(version, crate::data::Version::V2), + "currently only pack version 2 can be written", + ); + FromEntriesIter { + input, + output: gix_hash::io::Write::new(output, object_hash), + trailer: None, + entry_version: version, + pack_offsets_and_validity: exact_vec(num_entries as usize), + written: 0, + header_info: Some((version, num_entries)), + is_done: false, + } + } + + /// Consume this instance and return the `output` implementation. + /// + /// _Note_ that the `input` iterator can be moved out of this instance beforehand. + pub fn into_write(self) -> W { + self.output.inner + } + + /// Returns the trailing hash over all written entries once done. + /// It's `None` if we are not yet done writing. + pub fn digest(&self) -> Option { + self.trailer + } + + fn next_inner(&mut self) -> Result> { + let previous_written = self.written; + if let Some((version, num_entries)) = self.header_info.take() { + let header_bytes = crate::data::header::encode(version, num_entries); + self.output + .write_all(&header_bytes[..]) + .map_err(gix_hash::io::Error::from)?; + self.written += header_bytes.len() as u64; + } + match self.input.next() { + Some(entries) => { + for entry in entries.map_err(Error::Input)? { + if entry.is_invalid() { + self.pack_offsets_and_validity.push((0, false)); + continue; + } + self.pack_offsets_and_validity.push((self.written, true)); + let header = entry.to_entry_header(self.entry_version, |index| { + let (base_offset, is_valid_object) = self.pack_offsets_and_validity[index]; + if !is_valid_object { + unreachable!("if you see this the object database is correct as a delta refers to a non-existing object") + } + self.written - base_offset + }); + self.written += header + .write_to(entry.decompressed_size as u64, &mut self.output) + .map_err(gix_hash::io::Error::from)? as u64; + self.written += std::io::copy(&mut &*entry.compressed_data, &mut self.output) + .map_err(gix_hash::io::Error::from)?; + } + } + None => { + let digest = self + .output + .hash + .clone() + .try_finalize() + .map_err(gix_hash::io::Error::from)?; + self.output + .inner + .write_all(digest.as_slice()) + .map_err(gix_hash::io::Error::from)?; + self.written += digest.as_slice().len() as u64; + self.output.inner.flush().map_err(gix_hash::io::Error::from)?; + self.is_done = true; + self.trailer = Some(digest); + } + } + Ok(self.written - previous_written) + } +} + +impl Iterator for FromEntriesIter +where + I: Iterator, E>>, + W: std::io::Write, + E: std::error::Error + 'static, +{ + /// The amount of bytes written to `out` if `Ok` or the error `E` received from the input. + type Item = Result>; + + fn next(&mut self) -> Option { + if self.is_done { + return None; + } + Some(match self.next_inner() { + Err(err) => { + self.is_done = true; + Err(err) + } + Ok(written) => Ok(written), + }) + } +} diff --git a/knot2/third_party/gix-pack/src/data/output/mod.rs b/knot2/third_party/gix-pack/src/data/output/mod.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/output/mod.rs @@ -0,0 +1,41 @@ +use gix_hash::ObjectId; + +/// +pub mod count; + +/// An item representing a future Entry in the leanest way possible. +/// +/// One can expect to have one of these in memory when building big objects, so smaller is better here. +/// They should contain everything of importance to generate a pack as fast as possible. +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Count { + /// The hash of the object to write + pub id: ObjectId, + /// A way to locate a pack entry in the object database, only available if the object is in a pack. + pub entry_pack_location: count::PackLocation, +} + +/// An entry to be written to a file. +/// +/// Some of these will be in-flight and in memory while waiting to be written. Memory requirements depend on the amount of compressed +/// data they hold. +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Entry { + /// The hash of the object to write + pub id: ObjectId, + /// The kind of entry represented by `data`. It's used alongside with it to complete the pack entry + /// at rest or in transit. + pub kind: entry::Kind, + /// The size in bytes needed once `data` gets decompressed + pub decompressed_size: usize, + /// The compressed data right behind the header + pub compressed_data: Vec, +} + +/// +pub mod entry; + +/// +pub mod bytes; diff --git a/knot2/third_party/gix-pack/src/index/traverse/error.rs b/knot2/third_party/gix-pack/src/index/traverse/error.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/index/traverse/error.rs @@ -0,0 +1,44 @@ +use crate::index; + +/// Returned by [`index::File::traverse_with_index()`] and [`index::File::traverse_with_lookup`] +#[derive(thiserror::Error, Debug)] +#[allow(missing_docs)] +pub enum Error { + #[error("One of the traversal processors failed")] + Processor(#[source] E), + #[error("Failed to verify index file checksum")] + IndexVerify(#[source] index::verify::checksum::Error), + #[error("The pack delta tree index could not be built")] + Tree(#[from] crate::cache::delta::from_offsets::Error), + #[error("The tree traversal failed")] + TreeTraversal(#[from] crate::cache::delta::traverse::Error), + #[error(transparent)] + EntryType(#[from] crate::data::entry::decode::Error), + #[error("Object {id} at offset {offset} could not be decoded")] + PackDecode { + id: gix_hash::ObjectId, + offset: u64, + source: crate::data::decode::Error, + }, + #[error("The packfiles checksum didn't match the index file checksum")] + PackMismatch(#[source] gix_hash::verify::Error), + #[error("Failed to verify pack file checksum")] + PackVerify(#[source] crate::verify::checksum::Error), + #[error("Error verifying object at offset {offset} against checksum in the index file")] + PackObjectVerify { + offset: u64, + #[source] + source: gix_object::data::verify::Error, + }, + #[error( + "The CRC32 of {kind} object at offset {offset} didn't match the checksum in the index file: expected {expected}, got {actual}" + )] + Crc32Mismatch { + expected: u32, + actual: u32, + offset: u64, + kind: gix_object::Kind, + }, + #[error("Interrupted")] + Interrupted, +} diff --git a/knot2/third_party/gix-pack/src/index/traverse/mod.rs b/knot2/third_party/gix-pack/src/index/traverse/mod.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/index/traverse/mod.rs @@ -0,0 +1,232 @@ +use std::sync::atomic::AtomicBool; + +use gix_features::{parallel, progress::Progress, zlib}; + +use crate::index; + +mod reduce; +/// +pub mod with_index; +/// +pub mod with_lookup; +use reduce::Reducer; + +mod error; +pub use error::Error; +use gix_features::progress::DynNestedProgress; + +mod types; +pub use types::{Algorithm, ProgressId, SafetyCheck, Statistics}; + +/// Traversal options for [`index::File::traverse()`]. +#[derive(Debug, Clone)] +pub struct Options { + /// The algorithm to employ. + pub traversal: Algorithm, + /// If `Some`, only use the given number of threads. Otherwise, the number of threads to use will be selected based on + /// the number of available logical cores. + pub thread_limit: Option, + /// The kinds of safety checks to perform. + pub check: SafetyCheck, + /// A function to create a pack cache + pub make_pack_lookup_cache: F, +} + +impl Default for Options crate::cache::Never> { + fn default() -> Self { + Options { + check: Default::default(), + traversal: Default::default(), + thread_limit: None, + make_pack_lookup_cache: || crate::cache::Never, + } + } +} + +/// The outcome of the [`traverse()`][index::File::traverse()] method. +pub struct Outcome { + /// The checksum obtained when hashing the file, which matched the checksum contained within the file. + pub actual_index_checksum: gix_hash::ObjectId, + /// The statistics obtained during traversal. + pub statistics: Statistics, +} + +/// Traversal of pack data files using an index file +impl index::File +where + T: crate::FileData + Sync, +{ + /// Iterate through all _decoded objects_ in the given `pack` and handle them with a `Processor`. + /// The return value is (pack-checksum, [`Outcome`], `progress`), thus the pack traversal will always verify + /// the whole packs checksum to assure it was correct. In case of bit-rod, the operation will abort early without + /// verifying all objects using the [interrupt mechanism][gix_features::interrupt] mechanism. + /// + /// # Algorithms + /// + /// Using the [`Options::traversal`] field one can chose between two algorithms providing different tradeoffs. Both invoke + /// `new_processor()` to create functions receiving decoded objects, their object kind, index entry and a progress instance to provide + /// progress information. + /// + /// * [`Algorithm::DeltaTreeLookup`] builds an index to avoid any unnecessary computation while resolving objects, avoiding + /// the need for a cache entirely, rendering `new_cache()` unused. + /// One could also call [`traverse_with_index()`][index::File::traverse_with_index()] directly. + /// * [`Algorithm::Lookup`] uses a cache created by `new_cache()` to avoid having to re-compute all bases of a delta-chain while + /// decoding objects. + /// One could also call [`traverse_with_lookup()`][index::File::traverse_with_lookup()] directly. + /// + /// Use [`thread_limit`][Options::thread_limit] to further control parallelism and [`check`][SafetyCheck] to define how much the passed + /// objects shall be verified beforehand. + pub fn traverse( + &self, + pack: &crate::data::File, + progress: &mut dyn DynNestedProgress, + should_interrupt: &AtomicBool, + processor: Processor, + Options { + traversal, + thread_limit, + check, + make_pack_lookup_cache, + }: Options, + ) -> Result> + where + C: crate::cache::DecodeEntry, + E: std::error::Error + Send + Sync + 'static, + Processor: FnMut(gix_object::Kind, &[u8], &index::Entry, &dyn Progress) -> Result<(), E> + Send + Clone, + F: Fn() -> C + Send + Clone, + { + match traversal { + Algorithm::Lookup => self.traverse_with_lookup( + processor, + pack, + progress, + should_interrupt, + with_lookup::Options { + thread_limit, + check, + make_pack_lookup_cache, + }, + ), + Algorithm::DeltaTreeLookup => self.traverse_with_index( + pack, + processor, + progress, + should_interrupt, + with_index::Options { check, thread_limit }, + ), + } + } + + fn possibly_verify( + &self, + pack: &crate::data::File, + check: SafetyCheck, + pack_progress: &mut dyn Progress, + index_progress: &mut dyn Progress, + should_interrupt: &AtomicBool, + ) -> Result> + where + E: std::error::Error + Send + Sync + 'static, + { + Ok(if check.file_checksum() { + pack.checksum() + .verify(&self.pack_checksum()) + .map_err(Error::PackMismatch)?; + let (pack_res, id) = parallel::join( + move || pack.verify_checksum(pack_progress, should_interrupt), + move || self.verify_checksum(index_progress, should_interrupt), + ); + pack_res.map_err(Error::PackVerify)?; + id.map_err(Error::IndexVerify)? + } else { + self.index_checksum() + }) + } + + #[allow(clippy::too_many_arguments)] + fn decode_and_process_entry( + &self, + check: SafetyCheck, + pack: &crate::data::File, + cache: &mut C, + buf: &mut Vec, + inflate: &mut zlib::Inflate, + progress: &mut dyn Progress, + index_entry: &index::Entry, + processor: &mut impl FnMut(gix_object::Kind, &[u8], &index::Entry, &dyn Progress) -> Result<(), E>, + ) -> Result> + where + C: crate::cache::DecodeEntry, + E: std::error::Error + Send + Sync + 'static, + { + let pack_entry = pack.entry(index_entry.pack_offset)?; + let pack_entry_data_offset = pack_entry.data_offset; + let entry_stats = pack + .decode_entry( + pack_entry, + buf, + inflate, + &|id, _| { + let index = self.lookup(id)?; + pack.entry(self.pack_offset_at_index(index)) + .ok() + .map(crate::data::decode::entry::ResolvedBase::InPack) + }, + cache, + ) + .map_err(|e| Error::PackDecode { + source: e, + id: index_entry.oid, + offset: index_entry.pack_offset, + })?; + let object_kind = entry_stats.kind; + let header_size = (pack_entry_data_offset - index_entry.pack_offset) as usize; + let entry_len = header_size + entry_stats.compressed_size; + + process_entry( + check, + object_kind, + buf, + index_entry, + || pack.entry_crc32(index_entry.pack_offset, entry_len), + progress, + processor, + )?; + Ok(entry_stats) + } +} + +#[allow(clippy::too_many_arguments)] +fn process_entry( + check: SafetyCheck, + object_kind: gix_object::Kind, + decompressed: &[u8], + index_entry: &index::Entry, + pack_entry_crc32: impl FnOnce() -> u32, + progress: &dyn Progress, + processor: &mut impl FnMut(gix_object::Kind, &[u8], &index::Entry, &dyn Progress) -> Result<(), E>, +) -> Result<(), Error> +where + E: std::error::Error + Send + Sync + 'static, +{ + if check.object_checksum() { + gix_object::Data::new(decompressed, object_kind, index_entry.oid.kind()) + .verify_checksum(&index_entry.oid) + .map_err(|source| Error::PackObjectVerify { + offset: index_entry.pack_offset, + source, + })?; + if let Some(desired_crc32) = index_entry.crc32 { + let actual_crc32 = pack_entry_crc32(); + if actual_crc32 != desired_crc32 { + return Err(Error::Crc32Mismatch { + actual: actual_crc32, + expected: desired_crc32, + offset: index_entry.pack_offset, + kind: object_kind, + }); + } + } + } + processor(object_kind, decompressed, index_entry, progress).map_err(Error::Processor) +} diff --git a/knot2/third_party/gix-pack/src/index/traverse/reduce.rs b/knot2/third_party/gix-pack/src/index/traverse/reduce.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/index/traverse/reduce.rs @@ -0,0 +1,129 @@ +use std::{ + sync::atomic::{AtomicBool, Ordering}, + time::Instant, +}; + +use gix_features::{ + parallel, + progress::Progress, + threading::{Mutable, OwnShared, lock}, +}; + +use crate::{data, index::traverse}; + +fn add_decode_result(lhs: &mut data::decode::entry::Outcome, rhs: data::decode::entry::Outcome) { + lhs.num_deltas += rhs.num_deltas; + lhs.decompressed_size += rhs.decompressed_size; + lhs.compressed_size += rhs.compressed_size; + lhs.object_size += rhs.object_size; +} + +fn div_decode_result(lhs: &mut data::decode::entry::Outcome, div: usize) { + if div != 0 { + lhs.num_deltas = (lhs.num_deltas as f32 / div as f32) as u32; + lhs.decompressed_size /= div as u64; + lhs.compressed_size /= div; + lhs.object_size /= div as u64; + } +} + +pub struct Reducer<'a, P, E> { + progress: OwnShared>, + check: traverse::SafetyCheck, + then: Instant, + entries_seen: usize, + stats: traverse::Statistics, + should_interrupt: &'a AtomicBool, + _error: std::marker::PhantomData, +} + +impl<'a, P, E> Reducer<'a, P, E> +where + P: Progress, +{ + pub fn from_progress( + progress: OwnShared>, + pack_data_len_in_bytes: usize, + check: traverse::SafetyCheck, + should_interrupt: &'a AtomicBool, + ) -> Self { + let stats = traverse::Statistics { + pack_size: pack_data_len_in_bytes as u64, + ..Default::default() + }; + Reducer { + progress, + check, + then: Instant::now(), + entries_seen: 0, + should_interrupt, + stats, + _error: Default::default(), + } + } +} + +impl parallel::Reduce for Reducer<'_, P, E> +where + P: Progress, + E: std::error::Error + Send + Sync + 'static, +{ + type Input = Result, traverse::Error>; + type FeedProduce = (); + type Output = traverse::Statistics; + type Error = traverse::Error; + + fn feed(&mut self, input: Self::Input) -> Result<(), Self::Error> { + let chunk_stats: Vec<_> = match input { + Err(err @ traverse::Error::PackDecode { .. }) if !self.check.fatal_decode_error() => { + lock(&self.progress).info(format!("Ignoring decode error: {err}")); + return Ok(()); + } + res => res, + }?; + self.entries_seen += chunk_stats.len(); + + let chunk_total = chunk_stats.into_iter().fold( + data::decode::entry::Outcome::default_from_kind(gix_object::Kind::Tree), + |mut total, stats| { + *self.stats.objects_per_chain_length.entry(stats.num_deltas).or_insert(0) += 1; + self.stats.total_decompressed_entries_size += stats.decompressed_size; + self.stats.total_compressed_entries_size += stats.compressed_size as u64; + self.stats.total_object_size += stats.object_size; + use gix_object::Kind::*; + match stats.kind { + Commit => self.stats.num_commits += 1, + Tree => self.stats.num_trees += 1, + Blob => self.stats.num_blobs += 1, + Tag => self.stats.num_tags += 1, + } + add_decode_result(&mut total, stats); + total + }, + ); + + add_decode_result(&mut self.stats.average, chunk_total); + lock(&self.progress).set(self.entries_seen); + + if self.should_interrupt.load(Ordering::SeqCst) { + return Err(Self::Error::Interrupted); + } + Ok(()) + } + + fn finalize(mut self) -> Result { + div_decode_result(&mut self.stats.average, self.entries_seen); + + let elapsed_s = self.then.elapsed().as_secs_f32(); + let objects_per_second = (self.entries_seen as f32 / elapsed_s) as u32; + + lock(&self.progress).info(format!( + "of {} objects done in {:.2}s ({} objects/s, ~{}/s)", + self.entries_seen, + elapsed_s, + objects_per_second, + gix_features::progress::bytesize::ByteSize(self.stats.average.object_size * u64::from(objects_per_second)) + )); + Ok(self.stats) + } +} diff --git a/knot2/third_party/gix-pack/src/index/traverse/types.rs b/knot2/third_party/gix-pack/src/index/traverse/types.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/index/traverse/types.rs @@ -0,0 +1,113 @@ +use std::{collections::BTreeMap, marker::PhantomData}; + +/// Statistics regarding object encountered during execution of the [`traverse()`][crate::index::File::traverse()] method. +#[derive(Debug, PartialEq, Eq, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Statistics { + /// The average over all decoded objects + pub average: crate::data::decode::entry::Outcome, + /// A mapping of the length of the chain to the amount of objects at that length. + /// + /// A length of 0 indicates full objects, and everything more than that uses the given number + /// of delta objects. + pub objects_per_chain_length: BTreeMap, + /// The amount of bytes in all compressed streams, one per entry + pub total_compressed_entries_size: u64, + /// The amount of bytes in all decompressed streams, one per entry + pub total_decompressed_entries_size: u64, + /// The amount of bytes occupied by all undeltified, decompressed objects + pub total_object_size: u64, + /// The amount of bytes occupied by the pack itself, in bytes + pub pack_size: u64, + /// The amount of objects encountered that where commits + pub num_commits: u32, + /// The amount of objects encountered that where trees + pub num_trees: u32, + /// The amount of objects encountered that where tags + pub num_tags: u32, + /// The amount of objects encountered that where blobs + pub num_blobs: u32, +} + +impl Default for Statistics { + fn default() -> Self { + Statistics { + average: crate::data::decode::entry::Outcome::default_from_kind(gix_object::Kind::Tree), + objects_per_chain_length: Default::default(), + total_compressed_entries_size: 0, + total_decompressed_entries_size: 0, + total_object_size: 0, + pack_size: 0, + num_blobs: 0, + num_commits: 0, + num_trees: 0, + num_tags: 0, + } + } +} + +/// The ways to validate decoded objects before passing them to the processor. +#[derive(Default, Debug, PartialEq, Eq, Hash, Ord, PartialOrd, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum SafetyCheck { + /// Don't verify the validity of the checksums stored in the index and pack file + SkipFileChecksumVerification, + + /// All of the above, and also don't perform any object checksum verification + SkipFileAndObjectChecksumVerification, + + /// All of the above, and only log object decode errors. + /// + /// Useful if there is a damaged pack and you would like to traverse as many objects as possible. + SkipFileAndObjectChecksumVerificationAndNoAbortOnDecodeError, + + /// Perform all available safety checks before operating on the pack and + /// abort if any of them fails + #[default] + All, +} + +impl SafetyCheck { + pub(crate) fn file_checksum(&self) -> bool { + matches!(self, SafetyCheck::All) + } + pub(crate) fn object_checksum(&self) -> bool { + matches!(self, SafetyCheck::All | SafetyCheck::SkipFileChecksumVerification) + } + pub(crate) fn fatal_decode_error(&self) -> bool { + match self { + SafetyCheck::All + | SafetyCheck::SkipFileChecksumVerification + | SafetyCheck::SkipFileAndObjectChecksumVerification => true, + SafetyCheck::SkipFileAndObjectChecksumVerificationAndNoAbortOnDecodeError => false, + } + } +} + +/// The way we verify the pack +#[derive(Default, Debug, PartialEq, Eq, Hash, Ord, PartialOrd, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum Algorithm { + /// Build an index to allow decoding each delta and base exactly once, saving a lot of computational + /// resource at the expense of resident memory, as we will use an additional `DeltaTree` to accelerate + /// delta chain resolution. + #[default] + DeltaTreeLookup, + /// We lookup each object similarly to what would happen during normal repository use. + /// Uses more compute resources as it will resolve delta chains from back to front, but start right away + /// without indexing or investing any memory in indices. + /// + /// This option may be well suited for big packs in memory-starved system that support memory mapping. + Lookup, +} + +/// The progress ids used in [`traverse()`][crate::index::File::traverse()] . +/// +/// Use this information to selectively extract the progress of interest in case the parent application has custom visualization. +#[derive(Debug, Copy, Clone)] +pub enum ProgressId { + /// A root progress which isn't actually used, but links to the `ProgressId` of the lookup version of the algorithm. + WithLookup(PhantomData), + /// A root progress which isn't actually used, but links to the `ProgressId` of the indexed version of the algorithm. + WithIndex(PhantomData), +} diff --git a/knot2/third_party/gix-pack/src/index/traverse/with_index.rs b/knot2/third_party/gix-pack/src/index/traverse/with_index.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/index/traverse/with_index.rs @@ -0,0 +1,254 @@ +use std::sync::atomic::{AtomicBool, Ordering}; + +use gix_features::{parallel, progress::DynNestedProgress}; + +use super::Error; +use crate::{ + cache::delta::traverse, + index::{self, traverse::Outcome, util::index_entries_sorted_by_offset_ascending}, +}; + +/// Traversal options for [`traverse_with_index()`][index::File::traverse_with_index()] +#[derive(Default)] +pub struct Options { + /// If `Some`, only use the given number of threads. Otherwise, the number of threads to use will be selected based on + /// the number of available logical cores. + pub thread_limit: Option, + /// The kinds of safety checks to perform. + pub check: crate::index::traverse::SafetyCheck, +} + +/// The progress ids used in [`index::File::traverse_with_index()`]. +/// +/// Use this information to selectively extract the progress of interest in case the parent application has custom visualization. +#[derive(Debug, Copy, Clone)] +pub enum ProgressId { + /// The amount of bytes currently processed to generate a checksum of the *pack data file*. + HashPackDataBytes, + /// The amount of bytes currently processed to generate a checksum of the *pack index file*. + HashPackIndexBytes, + /// Collect all object hashes into a vector and sort it by their pack offset. + CollectSortedIndexEntries, + /// Count the objects processed when building a cache tree from all objects in a pack index. + TreeFromOffsetsObjects, + /// The amount of objects which were decoded. + DecodedObjects, + /// The amount of bytes that were decoded in total, as the sum of all bytes to represent all decoded objects. + DecodedBytes, +} + +impl From for gix_features::progress::Id { + fn from(v: ProgressId) -> Self { + match v { + ProgressId::HashPackDataBytes => *b"PTHP", + ProgressId::HashPackIndexBytes => *b"PTHI", + ProgressId::CollectSortedIndexEntries => *b"PTCE", + ProgressId::TreeFromOffsetsObjects => *b"PTDI", + ProgressId::DecodedObjects => *b"PTRO", + ProgressId::DecodedBytes => *b"PTDB", + } + } +} + +/// Traversal with index +impl index::File +where + T: crate::FileData + Sync, +{ + /// Iterate through all _decoded objects_ in the given `pack` and handle them with a `Processor`, using an index to reduce waste + /// at the cost of memory. + /// + /// For more details, see the documentation on the [`traverse()`][index::File::traverse()] method. + pub fn traverse_with_index( + &self, + pack: &crate::data::File, + mut processor: Processor, + progress: &mut dyn DynNestedProgress, + should_interrupt: &AtomicBool, + Options { + check, + thread_limit, + }: Options, + ) -> Result> + where + Processor: FnMut( + gix_object::Kind, + &[u8], + &index::Entry, + &dyn gix_features::progress::Progress, + ) -> Result<(), E> + + Send + + Clone, + E: std::error::Error + Send + Sync + 'static, + { + let (verify_result, traversal_result) = parallel::join( + { + let mut pack_progress = progress.add_child_with_id( + format!("Hash of pack '{}'", crate::source_name(pack.path())), + ProgressId::HashPackDataBytes.into(), + ); + let mut index_progress = progress.add_child_with_id( + format!("Hash of index '{}'", crate::source_name(&self.path)), + ProgressId::HashPackIndexBytes.into(), + ); + move || { + let res = self.possibly_verify( + pack, + check, + &mut pack_progress, + &mut index_progress, + should_interrupt, + ); + if res.is_err() { + should_interrupt.store(true, Ordering::SeqCst); + } + res + } + }, + || -> Result<_, Error<_>> { + let sorted_entries = index_entries_sorted_by_offset_ascending( + self, + &mut progress.add_child_with_id( + "collecting sorted index".into(), + ProgressId::CollectSortedIndexEntries.into(), + ), + ); /* Pack Traverse Collect sorted Entries */ + let tree = crate::cache::delta::Tree::from_offsets_in_pack( + pack.path(), + sorted_entries.into_iter().map(Entry::from), + &|e| e.index_entry.pack_offset, + &|id| self.lookup(id).map(|idx| self.pack_offset_at_index(idx)), + &mut progress.add_child_with_id( + "indexing".into(), + ProgressId::TreeFromOffsetsObjects.into(), + ), + should_interrupt, + self.object_hash, + )?; + let mut outcome = digest_statistics(tree.traverse( + |slice: crate::data::EntryRange, source: &crate::data::File, buf: &mut Vec| { + source.read_into(slice, buf) + }, + pack, + pack.pack_end() as u64, + move |data, + progress, + traverse::Context { + entry: pack_entry, + entry_end, + decompressed: bytes, + level, + object_kind, + }| { + data.level = level; + data.decompressed_size = pack_entry.decompressed_size; + data.object_kind = object_kind; + data.compressed_size = entry_end - pack_entry.data_offset; + data.object_size = bytes.len() as u64; + let result = index::traverse::process_entry( + check, + object_kind, + bytes, + &data.index_entry, + || { + // TODO: Fix this - we overwrite the header of 'data' which also changes the computed entry size, + // causing index and pack to seemingly mismatch. This is surprising, and should be done differently. + // debug_assert_eq!(&data.index_entry.pack_offset, &pack_entry.pack_offset()); + pack.entry_crc32( + data.index_entry.pack_offset, + (entry_end - data.index_entry.pack_offset) as usize, + ) + }, + progress, + &mut processor, + ); + match result { + Err(err @ Error::PackDecode { .. }) if !check.fatal_decode_error() => { + progress.info(format!("Ignoring decode error: {err}")); + Ok(()) + } + res => res, + } + }, + traverse::Options { + object_progress: Box::new( + progress.add_child_with_id("Resolving".into(), ProgressId::DecodedObjects.into()), + ), + size_progress: + &mut progress.add_child_with_id("Decoding".into(), ProgressId::DecodedBytes.into()), + thread_limit, + should_interrupt, + object_hash: self.object_hash, + base_spill: None, + collect_items: true, + max_object_bytes: None, + }, + )?); + outcome.pack_size = pack.data_len() as u64; + Ok(outcome) + }, + ); + Ok(Outcome { + actual_index_checksum: verify_result?, + statistics: traversal_result?, + }) + } +} + +#[derive(Clone)] +struct Entry { + index_entry: crate::index::Entry, + object_kind: gix_object::Kind, + object_size: u64, + decompressed_size: u64, + compressed_size: u64, + level: u16, +} + +impl From for Entry { + fn from(index_entry: crate::index::Entry) -> Self { + Entry { + index_entry, + level: 0, + object_kind: gix_object::Kind::Tree, + object_size: 0, + decompressed_size: 0, + compressed_size: 0, + } + } +} + +fn digest_statistics( + traverse::Outcome { items }: traverse::Outcome, +) -> index::traverse::Statistics { + let mut res = index::traverse::Statistics::default(); + let average = &mut res.average; + for item in items.iter() { + res.total_compressed_entries_size += item.data.compressed_size; + res.total_decompressed_entries_size += item.data.decompressed_size; + res.total_object_size += item.data.object_size; + *res.objects_per_chain_length + .entry(u32::from(item.data.level)) + .or_insert(0) += 1; + + average.decompressed_size += item.data.decompressed_size; + average.compressed_size += item.data.compressed_size as usize; + average.object_size += item.data.object_size; + average.num_deltas += u32::from(item.data.level); + use gix_object::Kind::*; + match item.data.object_kind { + Blob => res.num_blobs += 1, + Tree => res.num_trees += 1, + Tag => res.num_tags += 1, + Commit => res.num_commits += 1, + } + } + + let num_nodes = items.len(); + average.decompressed_size /= num_nodes as u64; + average.compressed_size /= num_nodes; + average.object_size /= num_nodes as u64; + average.num_deltas /= num_nodes as u32; + + res +} diff --git a/knot2/third_party/gix-pack/src/index/traverse/with_lookup.rs b/knot2/third_party/gix-pack/src/index/traverse/with_lookup.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/index/traverse/with_lookup.rs @@ -0,0 +1,189 @@ +use std::sync::atomic::{AtomicBool, Ordering}; + +use gix_features::{ + parallel::{self, in_parallel_if}, + progress::{self, Count, DynNestedProgress, Progress}, + threading::{Mutable, OwnShared, lock}, + zlib, +}; + +use super::{Error, Reducer}; +use crate::{ + data, exact_vec, index, + index::{traverse::Outcome, util}, +}; + +/// Traversal options for [`index::File::traverse_with_lookup()`] +pub struct Options { + /// If `Some`, only use the given number of threads. Otherwise, the number of threads to use will be selected based on + /// the number of available logical cores. + pub thread_limit: Option, + /// The kinds of safety checks to perform. + pub check: index::traverse::SafetyCheck, + /// A function to create a pack cache + pub make_pack_lookup_cache: F, +} + +impl Default for Options crate::cache::Never> { + fn default() -> Self { + Options { + check: Default::default(), + thread_limit: None, + make_pack_lookup_cache: || crate::cache::Never, + } + } +} + +/// The progress ids used in [`index::File::traverse_with_lookup()`]. +/// +/// Use this information to selectively extract the progress of interest in case the parent application has custom visualization. +#[derive(Debug, Copy, Clone)] +pub enum ProgressId { + /// The amount of bytes currently processed to generate a checksum of the *pack data file*. + HashPackDataBytes, + /// The amount of bytes currently processed to generate a checksum of the *pack index file*. + HashPackIndexBytes, + /// Collect all object hashes into a vector and sort it by their pack offset. + CollectSortedIndexEntries, + /// The amount of objects which were decoded by brute-force. + DecodedObjects, +} + +impl From for gix_features::progress::Id { + fn from(v: ProgressId) -> Self { + match v { + ProgressId::HashPackDataBytes => *b"PTHP", + ProgressId::HashPackIndexBytes => *b"PTHI", + ProgressId::CollectSortedIndexEntries => *b"PTCE", + ProgressId::DecodedObjects => *b"PTRO", + } + } +} + +/// Verify and validate the content of the index file +impl index::File +where + T: crate::FileData + Sync, +{ + /// Iterate through all _decoded objects_ in the given `pack` and handle them with a `Processor` using a cache to reduce the amount of + /// waste while decoding objects. + /// + /// For more details, see the documentation on the [`traverse()`][index::File::traverse()] method. + pub fn traverse_with_lookup( + &self, + mut processor: Processor, + pack: &data::File, + progress: &mut dyn DynNestedProgress, + should_interrupt: &AtomicBool, + Options { + thread_limit, + check, + make_pack_lookup_cache, + }: Options, + ) -> Result> + where + C: crate::cache::DecodeEntry, + E: std::error::Error + Send + Sync + 'static, + Processor: FnMut(gix_object::Kind, &[u8], &index::Entry, &dyn Progress) -> Result<(), E> + Send + Clone, + F: Fn() -> C + Send + Clone, + { + let (verify_result, traversal_result) = parallel::join( + { + let mut pack_progress = progress.add_child_with_id( + format!("Hash of pack '{}'", crate::source_name(pack.path())), + ProgressId::HashPackDataBytes.into(), + ); + let mut index_progress = progress.add_child_with_id( + format!("Hash of index '{}'", crate::source_name(&self.path)), + ProgressId::HashPackIndexBytes.into(), + ); + move || { + let res = + self.possibly_verify(pack, check, &mut pack_progress, &mut index_progress, should_interrupt); + if res.is_err() { + should_interrupt.store(true, Ordering::SeqCst); + } + res + } + }, + || { + let index_entries = util::index_entries_sorted_by_offset_ascending( + self, + &mut progress.add_child_with_id( + "collecting sorted index".into(), + ProgressId::CollectSortedIndexEntries.into(), + ), + ); + + let (chunk_size, thread_limit, available_cores) = + parallel::optimize_chunk_size_and_thread_limit(1000, Some(index_entries.len()), thread_limit, None); + let there_are_enough_entries_to_process = || index_entries.len() > chunk_size * available_cores; + let input_chunks = index_entries.chunks(chunk_size); + let reduce_progress = OwnShared::new(Mutable::new({ + let mut p = progress.add_child_with_id("Traversing".into(), ProgressId::DecodedObjects.into()); + p.init(Some(self.num_objects() as usize), progress::count("objects")); + p + })); + let state_per_thread = { + let reduce_progress = reduce_progress.clone(); + move |index| { + ( + make_pack_lookup_cache(), + Vec::with_capacity(2048), // decode buffer + zlib::Inflate::default(), + lock(&reduce_progress) + .add_child_with_id(format!("thread {index}"), gix_features::progress::UNKNOWN), // per thread progress + ) + } + }; + + in_parallel_if( + there_are_enough_entries_to_process, + input_chunks, + thread_limit, + state_per_thread, + move |entries: &[index::Entry], + (cache, buf, inflate, progress)| + -> Result, Error<_>> { + progress.init( + Some(entries.len()), + gix_features::progress::count_with_decimals("objects", 2), + ); + let mut stats = exact_vec(entries.len()); + progress.set(0); + for index_entry in entries.iter() { + let result = self.decode_and_process_entry( + check, + pack, + cache, + buf, + inflate, + progress, + index_entry, + &mut processor, + ); + progress.inc(); + let stat = match result { + Err(err @ Error::PackDecode { .. }) if !check.fatal_decode_error() => { + progress.info(format!("Ignoring decode error: {err}")); + continue; + } + res => res, + }?; + stats.push(stat); + if should_interrupt.load(Ordering::Relaxed) { + break; + } + } + Ok(stats) + }, + Reducer::from_progress(reduce_progress, pack.data_len(), check, should_interrupt), + ) + }, + ); + Ok(Outcome { + actual_index_checksum: verify_result?, + statistics: traversal_result?, + }) + } +} diff --git a/knot2/third_party/gix-pack/src/index/write/error.rs b/knot2/third_party/gix-pack/src/index/write/error.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/index/write/error.rs @@ -0,0 +1,27 @@ +/// Returned by [`crate::index::write_data_iter_to_stream()`] +#[derive(thiserror::Error, Debug)] +#[allow(missing_docs)] +pub enum Error { + #[error("An error occurred when writing the pack index file")] + Io(#[from] gix_hash::io::Error), + #[error("A pack entry could not be extracted")] + PackEntryDecode(#[from] crate::data::input::Error), + #[error("Indices of type {} cannot be written, only {} are supported", *.0 as usize, crate::index::Version::default() as usize)] + Unsupported(crate::index::Version), + #[error( + "Ref delta objects are not supported as there is no way to look them up. Resolve them beforehand." + )] + IteratorInvariantNoRefDelta, + #[error( + "The iterator failed to set a trailing hash over all prior pack entries in the last provided entry" + )] + IteratorInvariantTrailer, + #[error("Only u32::MAX objects can be stored in a pack, found {0}")] + IteratorInvariantTooManyObjects(usize), + #[error("{pack_offset} is not a valid offset for pack offset {distance}")] + IteratorInvariantBaseOffset { pack_offset: u64, distance: u64 }, + #[error(transparent)] + Tree(#[from] crate::cache::delta::Error), + #[error(transparent)] + TreeTraversal(#[from] crate::cache::delta::traverse::Error), +} diff --git a/knot2/third_party/gix-pack/src/index/write/mod.rs b/knot2/third_party/gix-pack/src/index/write/mod.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/index/write/mod.rs @@ -0,0 +1,280 @@ +pub use error::Error; + +mod error; + +#[derive(Clone)] +pub(crate) struct TreeEntry { + pub id: gix_hash::ObjectId, + pub crc32: u32, +} + +/// Information gathered while executing [`write_data_iter_to_stream()`][crate::index::write_data_iter_to_stream] +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Outcome { + /// The version of the verified index + pub index_version: crate::index::Version, + /// The verified checksum of the verified index + pub index_hash: gix_hash::ObjectId, + + /// The hash of the '.pack' file, also found in its trailing bytes + pub data_hash: gix_hash::ObjectId, + /// The amount of objects that were verified, always the amount of objects in the pack. + pub num_objects: u32, +} + +/// The progress ids used in [`write_data_iter_to_stream()`][crate::index::write_data_iter_to_stream()]. +/// +/// Use this information to selectively extract the progress of interest in case the parent application has custom visualization. +#[derive(Debug, Copy, Clone)] +pub enum ProgressId { + /// Counts the amount of objects that were index thus far. + IndexObjects, + /// The amount of bytes that were decompressed while decoding pack entries. + /// + /// This is done to determine entry boundaries. + DecompressedBytes, + /// The amount of objects whose hashes were computed. + /// + /// This is done by decoding them, which typically involves decoding delta objects. + ResolveObjects, + /// The amount of bytes that were decoded in total, as the sum of all bytes to represent all resolved objects. + DecodedBytes, + /// The amount of bytes written to the index file. + IndexBytesWritten, +} + +impl From for gix_features::progress::Id { + fn from(v: ProgressId) -> Self { + match v { + ProgressId::IndexObjects => *b"IWIO", + ProgressId::DecompressedBytes => *b"IWDB", + ProgressId::ResolveObjects => *b"IWRO", + ProgressId::DecodedBytes => *b"IWDB", + ProgressId::IndexBytesWritten => *b"IWBW", + } + } +} + +pub(super) mod function { + use std::{io, sync::atomic::AtomicBool}; + + use gix_features::progress::{self, Count, Progress, prodash::DynNestedProgress}; + + use crate::cache::delta::{Tree, traverse}; + + use super::{Error, Outcome, ProgressId, TreeEntry, modify_base}; + + /// Write information about `entries` as obtained from a pack data file into a pack index file via the `out` stream. + /// The resolver produced by `make_resolver` must resolve pack entries from the same pack data file that produced the + /// `entries` iterator. + /// + /// * `kind` is the version of pack index to produce, use [`crate::index::Version::default()`] if in doubt. + /// * `tread_limit` is used for a parallel tree traversal for obtaining object hashes with optimal performance. + /// * `root_progress` is the top-level progress to stay informed about the progress of this potentially long-running + /// computation. + /// * `object_hash` defines what kind of object hash we write into the index file. + /// * `pack_version` is the version of the underlying pack for which `entries` are read. It's used in case none of these objects are provided + /// to compute a pack-hash. + /// + /// # Remarks + /// + /// * neither in-pack nor out-of-pack Ref Deltas are supported here, these must have been resolved beforehand. + /// * `make_resolver()` will only be called after the iterator stopped returning elements and produces a function that + /// provides all bytes belonging to a pack entry writing them to the given mutable output `Vec`. + /// It should return `None` if the entry cannot be resolved from the pack that produced the `entries` iterator, causing + /// the write operation to fail. + #[allow(clippy::too_many_arguments)] + pub fn write_data_iter_to_stream( + version: crate::index::Version, + make_resolver: F, + entries: &mut dyn Iterator< + Item = Result, + >, + thread_limit: Option, + root_progress: &mut dyn DynNestedProgress, + out: &mut dyn io::Write, + should_interrupt: &AtomicBool, + object_hash: gix_hash::Kind, + pack_version: crate::data::Version, + ) -> Result + where + F: FnOnce() -> io::Result<(F2, R)>, + R: Send + Sync, + F2: Fn(crate::data::EntryRange, &R, &mut Vec) -> bool + Send + Clone, + { + if version != crate::index::Version::default() { + return Err(Error::Unsupported(version)); + } + let mut num_objects: usize = 0; + let mut last_seen_trailer = None; + let (anticipated_num_objects, upper_bound) = entries.size_hint(); + let worst_case_num_objects_after_thin_pack_resolution = + upper_bound.unwrap_or(anticipated_num_objects); + let mut tree = Tree::with_capacity(worst_case_num_objects_after_thin_pack_resolution)?; + let indexing_start = std::time::Instant::now(); + + root_progress.init(Some(4), progress::steps()); + let mut objects_progress = + root_progress.add_child_with_id("indexing".into(), ProgressId::IndexObjects.into()); + objects_progress.init(Some(anticipated_num_objects), progress::count("objects")); + let mut decompressed_progress = root_progress + .add_child_with_id("decompressing".into(), ProgressId::DecompressedBytes.into()); + decompressed_progress.init(None, progress::bytes()); + let mut pack_entries_end: u64 = 0; + + for entry in entries { + let crate::data::input::Entry { + header, + pack_offset, + crc32, + header_size, + compressed: _, + compressed_size, + decompressed_size, + trailer, + } = entry?; + + decompressed_progress.inc_by(decompressed_size as usize); + + let entry_len = u64::from(header_size) + compressed_size; + pack_entries_end = pack_offset + entry_len; + + let crc32 = crc32.expect( + "crc32 to be computed by the iterator. Caller assures correct configuration.", + ); + + use crate::data::entry::Header::*; + match header { + Tree | Blob | Commit | Tag => { + tree.add_root( + pack_offset, + TreeEntry { + id: object_hash.null(), + crc32, + }, + )?; + } + RefDelta { .. } => return Err(Error::IteratorInvariantNoRefDelta), + OfsDelta { base_distance } => { + let base_pack_offset = crate::data::entry::Header::verified_base_pack_offset( + pack_offset, + base_distance, + ) + .ok_or(Error::IteratorInvariantBaseOffset { + pack_offset, + distance: base_distance, + })?; + tree.add_child( + base_pack_offset, + pack_offset, + TreeEntry { + id: object_hash.null(), + crc32, + }, + )?; + } + } + last_seen_trailer = trailer; + num_objects += 1; + objects_progress.inc(); + } + let num_objects: u32 = num_objects + .try_into() + .map_err(|_| Error::IteratorInvariantTooManyObjects(num_objects))?; + + objects_progress.show_throughput(indexing_start); + decompressed_progress.show_throughput(indexing_start); + drop(objects_progress); + drop(decompressed_progress); + + root_progress.inc(); + + let (resolver, pack) = make_resolver().map_err(gix_hash::io::Error::from)?; + let sorted_pack_offsets_by_oid = { + let traverse::Outcome { items } = + tree.traverse( + resolver, + &pack, + pack_entries_end, + |data, + _progress, + traverse::Context { + decompressed: bytes, + object_kind, + .. + }| { modify_base(data, bytes, object_kind, object_hash) }, + traverse::Options { + object_progress: Box::new(root_progress.add_child_with_id( + "Resolving".into(), + ProgressId::ResolveObjects.into(), + )), + size_progress: &mut root_progress + .add_child_with_id("Decoding".into(), ProgressId::DecodedBytes.into()), + thread_limit, + should_interrupt, + object_hash, + base_spill: None, + collect_items: true, + max_object_bytes: None, + }, + )?; + root_progress.inc(); + + let mut items = items; + { + let _progress = root_progress + .add_child_with_id("sorting by id".into(), gix_features::progress::UNKNOWN); + items.sort_by_key(|e| e.data.id); + } + + root_progress.inc(); + items + }; + + let pack_hash = match last_seen_trailer { + Some(ph) => ph, + None if num_objects == 0 => { + let header = crate::data::header::encode(pack_version, 0); + let mut hasher = gix_hash::hasher(object_hash); + hasher.update(&header); + hasher.try_finalize().map_err(gix_hash::io::Error::from)? + } + None => return Err(Error::IteratorInvariantTrailer), + }; + let index_hash = crate::index::encode::write_to( + out, + sorted_pack_offsets_by_oid, + &pack_hash, + version, + object_hash, + &mut root_progress.add_child_with_id( + "writing index file".into(), + ProgressId::IndexBytesWritten.into(), + ), + )?; + root_progress.show_throughput_with( + indexing_start, + num_objects as usize, + progress::count("objects").expect("unit always set"), + progress::MessageLevel::Success, + ); + Ok(Outcome { + index_version: version, + index_hash, + data_hash: pack_hash, + num_objects, + }) + } +} + +fn modify_base( + entry: &mut TreeEntry, + decompressed: &[u8], + object_kind: gix_object::Kind, + hash: gix_hash::Kind, +) -> Result<(), gix_hash::hasher::Error> { + let id = gix_object::compute_hash(hash, object_kind, decompressed)?; + entry.id = id; + Ok(()) +} diff --git a/knot2/third_party/gix-pack/src/cache/delta/traverse/mod.rs b/knot2/third_party/gix-pack/src/cache/delta/traverse/mod.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/cache/delta/traverse/mod.rs @@ -0,0 +1,286 @@ +use std::os::unix::fs::FileExt; +use std::sync::{Arc, Mutex}; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; + +use gix_features::{ + parallel::in_parallel_with_slice, + progress::{self, DynNestedProgress, Progress}, + threading, + threading::{Mutable, OwnShared}, +}; + +use crate::{ + cache::delta::{Item, Tree}, + data::EntryRange, +}; + +mod resolve; + +#[derive(Clone, Copy)] +pub(crate) struct SpillRef { + offset: u64, + len: usize, +} + +pub struct BaseSpill { + budget: usize, + resident: AtomicUsize, + peak_resident: AtomicUsize, + spilled_bytes: AtomicU64, + file: std::fs::File, + write_cursor: Mutex, +} + +impl BaseSpill { + pub fn new(file: std::fs::File, budget: usize) -> Self { + Self { + budget, + resident: AtomicUsize::new(0), + peak_resident: AtomicUsize::new(0), + spilled_bytes: AtomicU64::new(0), + file, + write_cursor: Mutex::new(0), + } + } + + pub fn peak_resident(&self) -> usize { + self.peak_resident.load(Ordering::Relaxed) + } + + pub fn spilled_bytes(&self) -> u64 { + self.spilled_bytes.load(Ordering::Relaxed) + } + + fn account_push(&self, len: usize) { + let now = self.resident.fetch_add(len, Ordering::Relaxed) + len; + self.peak_resident.fetch_max(now, Ordering::Relaxed); + } + + fn account_pop_resident(&self, len: usize) { + self.resident.fetch_sub(len, Ordering::Relaxed); + } + + fn over_budget(&self) -> bool { + self.resident.load(Ordering::Relaxed) > self.budget + } + + fn spill(&self, bytes: &[u8]) -> std::io::Result { + let len = bytes.len(); + let offset = { + let mut cursor = self.write_cursor.lock().expect("base spill cursor poisoned"); + let offset = *cursor; + *cursor += len as u64; + offset + }; + self.file.write_all_at(bytes, offset)?; + self.resident.fetch_sub(len, Ordering::Relaxed); + self.spilled_bytes.fetch_add(len as u64, Ordering::Relaxed); + Ok(SpillRef { offset, len }) + } + + fn reload(&self, sref: SpillRef, out: &mut Vec) -> std::io::Result<()> { + out.resize(sref.len, 0); + self.file.read_exact_at(out, sref.offset) + } +} + +/// Returned by [`Tree::traverse()`] +#[derive(thiserror::Error, Debug)] +#[allow(missing_docs)] +pub enum Error { + #[error("{message}")] + ZlibInflate { + source: gix_features::zlib::inflate::Error, + message: &'static str, + }, + #[error("The resolver failed to obtain the pack entry bytes for the entry at {pack_offset}")] + ResolveFailed { pack_offset: u64 }, + #[error(transparent)] + EntryType(#[from] crate::data::entry::decode::Error), + #[error("One of the object inspectors failed")] + Inspect(#[from] Box), + #[error("Interrupted")] + Interrupted, + #[error( + "The base at {base_pack_offset} was referred to by a ref-delta, but it was never added to the tree as if the pack was still thin." + )] + OutOfPackRefDelta { + /// The base's offset which was from a resolved ref-delta that didn't actually get added to the tree + base_pack_offset: crate::data::Offset, + }, + #[error("Failed to spawn thread when switching to work-stealing mode")] + SpawnThread(#[from] std::io::Error), + #[error("Failed to page a delta base to the memory-budget overflow file")] + BaseSpill { source: std::io::Error }, + #[error( + "The base object at {pack_offset} decoded as a delta, but base objects cannot be deltas" + )] + UnexpectedRootDelta { pack_offset: crate::data::Offset }, + #[error(transparent)] + Delta(#[from] crate::data::delta::apply::Error), + #[error( + "The delta at {delta_pack_offset} declared a base of {declared} bytes but the resolved base is {actual} bytes" + )] + BaseSizeMismatch { + delta_pack_offset: crate::data::Offset, + declared: u64, + actual: u64, + }, + #[error( + "The delta at {delta_pack_offset} declared a reconstructed size of {declared} bytes, over the {limit}-byte object limit" + )] + DeltaResultTooLarge { + delta_pack_offset: crate::data::Offset, + declared: u64, + limit: u64, + }, +} + +/// Additional context passed to the `inspect_object(…)` function of the [`Tree::traverse()`] method. +#[allow(missing_docs)] +pub struct Context<'a> { + /// The pack entry describing the object + pub entry: &'a crate::data::Entry, + /// The offset at which `entry` ends in the pack, useful to learn about the exact range of `entry` within the pack. + pub entry_end: u64, + /// The decompressed object itself, ready to be decoded. + pub decompressed: &'a [u8], + /// The depth at which this object resides in the delta-tree. It represents the number of base objects, with 0 indicating + /// an 'undeltified' object, and higher values indicating delta objects with the given number of bases. + pub level: u16, + pub object_kind: gix_object::Kind, +} + +/// Options for [`Tree::traverse()`]. +pub struct Options<'a, 's> { + /// is a progress instance to track progress for each object in the traversal. + pub object_progress: Box, + /// is a progress instance to track the overall progress. + pub size_progress: &'s mut dyn Progress, + /// If `Some`, only use the given number of threads. Otherwise, the number of threads to use will be selected based on + /// the number of available logical cores. + pub thread_limit: Option, + /// Abort the operation if the value is `true`. + pub should_interrupt: &'a AtomicBool, + /// specifies what kind of hashes we expect to be stored in oid-delta entries, which is viable to decoding them + /// with the correct size. + pub object_hash: gix_hash::Kind, + pub base_spill: Option>, + pub collect_items: bool, + pub max_object_bytes: Option, +} + +/// The outcome of [`Tree::traverse()`] +#[allow(missing_docs)] +pub struct Outcome { + pub items: Vec>, +} + +impl Tree +where + T: Send + Sync + Clone, +{ + /// Traverse this tree of delta objects with a function `inspect_object` to process each object at will. + /// + /// * `should_run_in_parallel() -> bool` returns true if the underlying pack is big enough to warrant parallel traversal at all. + /// * `resolve(EntrySlice, &R, &mut Vec) -> bool` reads the raw pack bytes for the given `EntrySlice` into the + /// output vector, reusing its allocation. It returns `true` if the object existed in the pack, or `false` to indicate a + /// resolution error, which aborts the operation. + /// * `pack_entries_end` marks one-past-the-last byte of the last entry in the pack, as the last entries size would otherwise + /// be unknown as it's not part of the index file. + /// * `inspect_object(node_data: &mut T, progress: Progress, context: Context) -> Result<(), CustomError>` is a function + /// running for each thread receiving fully decoded objects along with contextual information, which either succeeds with `Ok(())` + /// or returns a `CustomError`. + /// Note that `node_data` can be modified to allow storing maintaining computation results on a per-object basis. It should contain + /// its own mutable per-thread data as required. + /// + /// This method returns a vector of all tree items, along with their potentially modified custom node data. + /// + /// _Note_ that this method consumed the Tree to assure safe parallel traversal with mutation support. + pub fn traverse( + self, + resolve: F, + resolve_data: &R, + pack_entries_end: u64, + inspect_object: MBFN, + Options { + thread_limit, + mut object_progress, + size_progress, + should_interrupt, + object_hash, + base_spill, + collect_items, + max_object_bytes, + }: Options<'_, '_>, + ) -> Result, Error> + where + F: Fn(EntryRange, &R, &mut Vec) -> bool + Send + Clone, + R: Send + Sync, + MBFN: FnMut(&mut T, &dyn Progress, Context<'_>) -> Result<(), E> + Send + Clone, + E: std::error::Error + Send + Sync + 'static, + { + let num_objects = self.num_items(); + let object_counter = { + let progress = &mut object_progress; + progress.init(Some(num_objects), progress::count("objects")); + progress.counter() + }; + size_progress.init(None, progress::bytes()); + let size_counter = size_progress.counter(); + let object_progress = OwnShared::new(Mutable::new(object_progress)); + + let start = std::time::Instant::now(); + let (forest, mut roots) = self.into_forest(pack_entries_end)?; + let forest = &forest; + let base_spill = base_spill.as_deref(); + let harvested = in_parallel_with_slice( + &mut roots, + thread_limit, + { + let object_progress = object_progress.clone(); + move |thread_index| resolve::State { + delta_bytes: Vec::::with_capacity(4096), + fully_resolved_delta_bytes: Vec::::with_capacity(4096), + progress: Box::new( + threading::lock(&object_progress) + .add_child(format!("thread {thread_index}")), + ), + resolve: resolve.clone(), + modify_base: inspect_object.clone(), + out: Vec::new(), + } + }, + { + move |root_id: &mut u32, state, threads_left, should_interrupt| { + resolve::deltas( + object_counter.clone(), + size_counter.clone(), + *root_id, + forest, + state, + resolve_data, + object_hash.len_in_bytes(), + base_spill, + collect_items, + max_object_bytes, + threads_left, + should_interrupt, + ) + } + }, + || { + (!should_interrupt.load(Ordering::Relaxed)) + .then(|| std::time::Duration::from_millis(50)) + }, + |state| state.out, + )?; + + threading::lock(&object_progress).show_throughput(start); + size_progress.show_throughput(start); + + Ok(Outcome { + items: harvested.into_iter().flatten().collect(), + }) + } +} diff --git a/knot2/third_party/gix-pack/src/cache/delta/traverse/resolve.rs b/knot2/third_party/gix-pack/src/cache/delta/traverse/resolve.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/cache/delta/traverse/resolve.rs @@ -0,0 +1,576 @@ +use std::sync::atomic::{AtomicBool, AtomicIsize, Ordering}; + +use gix_features::{progress::Progress, threading, zlib}; + +use crate::{ + cache::delta::{ + Item, + traverse::{Context, Error}, + tree::Forest, + }, + data, + data::EntryRange, +}; + +struct Pending { + level: u16, + node_id: u32, + entry: data::Entry, + entry_end: u64, + base_bytes: Vec, + object_kind: gix_object::Kind, + spill_ref: Option, +} + +fn enforce_budget(spill: &super::BaseSpill, stack: &mut [Pending]) -> Result<(), Error> { + (0..stack.len()).try_for_each(|index| -> Result<(), Error> { + if spill.over_budget() && stack[index].spill_ref.is_none() && !stack[index].base_bytes.is_empty() + { + let bytes = std::mem::take(&mut stack[index].base_bytes); + let sref = spill.spill(&bytes).map_err(|source| Error::BaseSpill { source })?; + stack[index].spill_ref = Some(sref); + } + Ok(()) + }) +} + +fn restore_base_bytes(spill: Option<&super::BaseSpill>, pending: &mut Pending) -> Result<(), Error> { + if let Some(spill) = spill { + match pending.spill_ref.take() { + Some(sref) => spill + .reload(sref, &mut pending.base_bytes) + .map_err(|source| Error::BaseSpill { source })?, + None => spill.account_pop_resident(pending.base_bytes.len()), + } + } + Ok(()) +} + +pub(super) struct State { + pub delta_bytes: Vec, + pub fully_resolved_delta_bytes: Vec, + pub progress: Box, + pub resolve: F, + pub modify_base: MBFN, + pub out: Vec>, +} + +pub(super) fn deltas( + objects: gix_features::progress::StepShared, + size: gix_features::progress::StepShared, + root_id: u32, + forest: &Forest, + State { + delta_bytes, + fully_resolved_delta_bytes, + progress, + resolve, + modify_base, + out, + }: &mut State, + resolve_data: &R, + hash_len: usize, + spill: Option<&super::BaseSpill>, + collect_items: bool, + max_object_bytes: Option, + threads_left: &AtomicIsize, + should_interrupt: &AtomicBool, +) -> Result<(), Error> +where + T: Send + Sync + Clone, + R: Send + Sync, + F: Fn(EntryRange, &R, &mut Vec) -> bool + Send + Clone, + MBFN: FnMut(&mut T, &dyn Progress, Context<'_>) -> Result<(), E> + Send + Clone, + E: std::error::Error + Send + Sync + 'static, +{ + let mut inflate = zlib::Inflate::default(); + let mut decompress = decompressor(&*resolve, resolve_data, hash_len, &mut inflate); + + let mut stack: Vec = Vec::new(); + + let mut root_bytes = Vec::new(); + let (root_entry, root_end) = decompress(forest.entry_slice(root_id), &mut root_bytes)?; + let object_kind = root_entry + .header + .as_kind() + .ok_or(Error::UnexpectedRootDelta { + pack_offset: forest.offset(root_id), + })?; + let mut root_data = forest.data[root_id as usize].clone(); + apply_base( + modify_base, + &mut root_data, + progress, + &root_entry, + root_end, + &root_bytes, + 0, + object_kind, + )?; + objects.fetch_add(1, Ordering::Relaxed); + size.fetch_add(root_bytes.len(), Ordering::Relaxed); + if collect_items { + out.push(forest.item(root_id, root_data)); + } + expand( + forest, + forest.children(root_id), + &root_bytes, + object_kind, + 0, + &mut stack, + delta_bytes, + fully_resolved_delta_bytes, + modify_base, + &**progress, + out, + &mut decompress, + &objects, + &size, + spill, + collect_items, + max_object_bytes, + )?; + drop(root_bytes); + if let Some(spill) = spill { + if spill.over_budget() { + enforce_budget(spill, &mut stack)?; + } + } + + loop { + if stack.len() > 1 { + if let Ok(initial_threads) = + threads_left.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |threads_available| { + (threads_available > 0).then_some(0) + }) + { + *delta_bytes = Vec::new(); + *fully_resolved_delta_bytes = Vec::new(); + return deltas_mt( + initial_threads, + stack, + objects, + size, + &**progress, + resolve.clone(), + resolve_data, + modify_base.clone(), + hash_len, + spill, + collect_items, + max_object_bytes, + forest, + out, + threads_left, + should_interrupt, + ); + } + } + + let Some(mut pending) = stack.pop() else { + break; + }; + restore_base_bytes(spill, &mut pending)?; + let Pending { + level, + node_id, + entry, + entry_end, + base_bytes, + object_kind, + .. + } = pending; + if should_interrupt.load(Ordering::Relaxed) { + return Err(Error::Interrupted); + } + + let mut node_data = forest.data[node_id as usize].clone(); + apply_base( + modify_base, + &mut node_data, + progress, + &entry, + entry_end, + &base_bytes, + level, + object_kind, + )?; + objects.fetch_add(1, Ordering::Relaxed); + size.fetch_add(base_bytes.len(), Ordering::Relaxed); + if collect_items { + out.push(forest.item(node_id, node_data)); + } + expand( + forest, + forest.children(node_id), + &base_bytes, + object_kind, + level, + &mut stack, + delta_bytes, + fully_resolved_delta_bytes, + modify_base, + &**progress, + out, + &mut decompress, + &objects, + &size, + spill, + collect_items, + max_object_bytes, + )?; + drop(base_bytes); + if let Some(spill) = spill { + if spill.over_budget() { + enforce_budget(spill, &mut stack)?; + } + } + } + + Ok(()) +} + +/// * `initial_threads` is the threads we may spawn, not accounting for our own thread which is still considered used by the parent +/// system. Since this thread will take a controlling function, we may spawn one more than that. In threaded mode, we will finish +/// all remaining work. +#[allow(clippy::too_many_arguments)] +fn deltas_mt( + mut threads_to_create: isize, + stack: Vec, + objects: gix_features::progress::StepShared, + size: gix_features::progress::StepShared, + progress: &dyn Progress, + resolve: F, + resolve_data: &R, + modify_base: MBFN, + hash_len: usize, + spill: Option<&super::BaseSpill>, + collect_items: bool, + max_object_bytes: Option, + forest: &Forest, + accumulator: &mut Vec>, + threads_left: &AtomicIsize, + should_interrupt: &AtomicBool, +) -> Result<(), Error> +where + T: Send + Sync + Clone, + R: Send + Sync, + F: Fn(EntryRange, &R, &mut Vec) -> bool + Send + Clone, + MBFN: FnMut(&mut T, &dyn Progress, Context<'_>) -> Result<(), E> + Send + Clone, + E: std::error::Error + Send + Sync + 'static, +{ + let stack = gix_features::threading::Mutable::new(stack); + threads_to_create += 1; // ourselves + let mut returned_ourselves = false; + + gix_features::parallel::threads(|s| -> Result<(), Error> { + let mut threads = Vec::new(); + let poll_interval = std::time::Duration::from_millis(100); + loop { + for tid in 0..threads_to_create { + let thread = gix_features::parallel::build_thread() + .name(format!("gix-pack.traverse_deltas.{tid}")) + .spawn_scoped(s, { + let stack = &stack; + let resolve = resolve.clone(); + let mut modify_base = modify_base.clone(); + let objects = &objects; + let size = &size; + + move || -> Result>, Error> { + let mut collected: Vec> = Vec::new(); + let mut delta_bytes = Vec::new(); + let mut fully_resolved_delta_bytes = Vec::new(); + let mut inflate = zlib::Inflate::default(); + let mut decompress = + decompressor(&resolve, resolve_data, hash_len, &mut inflate); + + loop { + let Some(mut pending) = threading::lock(stack).pop() else { + break; + }; + restore_base_bytes(spill, &mut pending)?; + let Pending { + level, + node_id, + entry, + entry_end, + base_bytes, + object_kind, + .. + } = pending; + if should_interrupt.load(Ordering::Relaxed) { + return Err(Error::Interrupted); + } + + let mut node_data = forest.data[node_id as usize].clone(); + apply_base( + &mut modify_base, + &mut node_data, + progress, + &entry, + entry_end, + &base_bytes, + level, + object_kind, + )?; + objects.fetch_add(1, Ordering::Relaxed); + size.fetch_add(base_bytes.len(), Ordering::Relaxed); + if collect_items { + collected.push(forest.item(node_id, node_data)); + } + let mut produced: Vec = Vec::new(); + expand( + forest, + forest.children(node_id), + &base_bytes, + object_kind, + level, + &mut produced, + &mut delta_bytes, + &mut fully_resolved_delta_bytes, + &mut modify_base, + progress, + &mut collected, + &mut decompress, + objects, + size, + spill, + collect_items, + max_object_bytes, + )?; + drop(base_bytes); + if !produced.is_empty() { + let mut guard = threading::lock(stack); + guard.append(&mut produced); + if let Some(spill) = spill { + if spill.over_budget() { + enforce_budget(spill, &mut guard[..])?; + } + } + } + } + Ok(collected) + } + })?; + threads.push(thread); + } + if threads_left + .fetch_update( + Ordering::SeqCst, + Ordering::SeqCst, + |threads_available: isize| { + (threads_available > 0).then(|| { + threads_to_create = + threads_available.min(threading::lock(&stack).len() as isize); + threads_available - threads_to_create + }) + }, + ) + .is_err() + { + threads_to_create = 0; + } + + // What we really want to do is either wait for one of our threads to go down + // or for another scheduled thread to become available. Unfortunately we can't do that, + // but may instead find a good way to set the polling interval instead of hard-coding it. + std::thread::sleep(poll_interval); + // Get out of threads are already starving or they would be starving soon as no work is left. + // + // Lint: ScopedJoinHandle is not the same depending on active features and is not exposed in some cases. + #[allow(clippy::redundant_closure_for_method_calls)] + if threads.iter().any(|t| t.is_finished()) { + let mut running_threads = Vec::new(); + for thread in threads.drain(..) { + if thread.is_finished() { + match thread.join() { + Ok(Err(err)) => return Err(err), + Ok(Ok(collected)) => { + accumulator.extend(collected); + if !returned_ourselves { + returned_ourselves = true; + } else { + threads_left.fetch_add(1, Ordering::SeqCst); + } + } + Err(err) => { + std::panic::resume_unwind(err); + } + } + } else { + running_threads.push(thread); + } + } + if running_threads.is_empty() && threading::lock(&stack).is_empty() { + break; + } + threads = running_threads; + } + } + + Ok(()) + }) +} + +fn decompressor<'a, F, R>( + resolve: &'a F, + resolve_data: &'a R, + hash_len: usize, + inflate: &'a mut zlib::Inflate, +) -> impl FnMut(EntryRange, &mut Vec) -> Result<(data::Entry, u64), Error> + 'a +where + F: Fn(EntryRange, &R, &mut Vec) -> bool, + R: Sync, +{ + let mut raw = Vec::new(); + move |slice: EntryRange, out: &mut Vec| -> Result<(data::Entry, u64), Error> { + if !resolve(slice.clone(), resolve_data, &mut raw) { + return Err(Error::ResolveFailed { + pack_offset: slice.start, + }); + } + let entry = data::Entry::from_bytes(&raw, slice.start, hash_len)?; + let compressed = &raw[entry.header_size()..]; + let decompressed_len = entry.decompressed_size as usize; + decompress_all_at_once_with(inflate, compressed, decompressed_len, out)?; + Ok((entry, slice.end)) + } +} + +fn apply_base( + modify_base: &mut MBFN, + data: &mut T, + progress: &dyn Progress, + entry: &data::Entry, + entry_end: u64, + decompressed: &[u8], + level: u16, + object_kind: gix_object::Kind, +) -> Result<(), Error> +where + MBFN: FnMut(&mut T, &dyn Progress, Context<'_>) -> Result<(), E>, + E: std::error::Error + Send + Sync + 'static, +{ + modify_base( + data, + progress, + Context { + entry, + entry_end, + decompressed, + level, + object_kind, + }, + ) + .map_err(|err| Box::new(err) as Box)?; + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn expand( + forest: &Forest, + children: &[u32], + base_bytes: &[u8], + object_kind: gix_object::Kind, + base_level: u16, + stack: &mut Vec, + delta_bytes: &mut Vec, + fully_resolved_delta_bytes: &mut Vec, + modify_base: &mut MBFN, + progress: &dyn Progress, + sink: &mut Vec>, + decompress: &mut F, + objects: &gix_features::progress::StepShared, + size: &gix_features::progress::StepShared, + spill: Option<&super::BaseSpill>, + collect_items: bool, + max_object_bytes: Option, +) -> Result<(), Error> +where + T: Clone, + F: FnMut(EntryRange, &mut Vec) -> Result<(data::Entry, u64), Error>, + MBFN: FnMut(&mut T, &dyn Progress, Context<'_>) -> Result<(), E>, + E: std::error::Error + Send + Sync + 'static, +{ + for &child_id in children { + let (child_entry, entry_end) = decompress(forest.entry_slice(child_id), delta_bytes)?; + let (base_size, consumed) = data::delta::decode_header_size(delta_bytes)?; + let mut header_ofs = consumed; + if base_bytes.len() != base_size as usize { + return Err(Error::BaseSizeMismatch { + delta_pack_offset: forest.offset(child_id), + declared: base_size, + actual: base_bytes.len() as u64, + }); + } + let (result_size, consumed) = data::delta::decode_header_size(&delta_bytes[consumed..])?; + header_ofs += consumed; + + if let Some(limit) = max_object_bytes { + if result_size > limit { + return Err(Error::DeltaResultTooLarge { + delta_pack_offset: forest.offset(child_id), + declared: result_size, + limit, + }); + } + } + fully_resolved_delta_bytes.resize(result_size as usize, 0); + data::delta::apply( + base_bytes, + fully_resolved_delta_bytes, + &delta_bytes[header_ofs..], + )?; + + if forest.children(child_id).is_empty() { + let mut child_data = forest.data[child_id as usize].clone(); + apply_base( + modify_base, + &mut child_data, + progress, + &child_entry, + entry_end, + fully_resolved_delta_bytes, + base_level + 1, + object_kind, + )?; + objects.fetch_add(1, Ordering::Relaxed); + size.fetch_add(base_bytes.len(), Ordering::Relaxed); + if collect_items { + sink.push(forest.item(child_id, child_data)); + } + } else { + let base_bytes = std::mem::take(fully_resolved_delta_bytes); + if let Some(spill) = spill { + spill.account_push(base_bytes.len()); + } + stack.push(Pending { + level: base_level + 1, + node_id: child_id, + entry: child_entry, + entry_end, + base_bytes, + object_kind, + spill_ref: None, + }); + } + } + Ok(()) +} + +fn decompress_all_at_once_with( + inflate: &mut zlib::Inflate, + b: &[u8], + decompressed_len: usize, + out: &mut Vec, +) -> Result<(), Error> { + out.resize(decompressed_len, 0); + inflate.reset(); + inflate.once(b, out).map_err(|err| Error::ZlibInflate { + source: err, + message: "Failed to decompress entry", + })?; + Ok(()) +} diff --git a/knot2/third_party/gix-pack/src/data/file/decode/entry.rs b/knot2/third_party/gix-pack/src/data/file/decode/entry.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/file/decode/entry.rs @@ -0,0 +1,515 @@ +use std::ops::Range; + +use gix_features::zlib; +use smallvec::SmallVec; + +use crate::{ + cache, data, + data::{File, delta, file::decode::Error}, +}; + +/// A return value of a resolve function, which given an [`ObjectId`][gix_hash::ObjectId] determines where an object can be found. +#[derive(Debug, PartialEq, Eq, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum ResolvedBase { + /// Indicate an object is within this pack, at the given entry, and thus can be looked up locally. + InPack(data::Entry), + /// Indicates the object of `kind` was found outside of the pack, and its data was written into an output + /// vector which now has a length of `end`. + #[allow(missing_docs)] + OutOfPack { kind: gix_object::Kind, end: usize }, +} + +#[derive(Debug)] +struct Delta { + data: Range, + base_size: usize, + result_size: usize, + + decompressed_size: usize, + data_offset: data::Offset, +} + +/// Additional information and statistics about a successfully decoded object produced by [`File::decode_entry()`]. +/// +/// Useful to understand the effectiveness of the pack compression or the cost of decompression. +#[derive(Debug, PartialEq, Eq, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Outcome { + /// The kind of resolved object. + pub kind: gix_object::Kind, + /// The amount of deltas in the chain of objects that had to be resolved beforehand. + /// + /// This number is affected by the [`Cache`][cache::DecodeEntry] implementation, with cache hits shortening the + /// delta chain accordingly + pub num_deltas: u32, + /// The total decompressed size of all pack entries in the delta chain + pub decompressed_size: u64, + /// The total compressed size of all pack entries in the delta chain + pub compressed_size: usize, + /// The total size of the decoded object. + pub object_size: u64, +} + +impl Outcome { + pub(crate) fn default_from_kind(kind: gix_object::Kind) -> Self { + Self { + kind, + num_deltas: 0, + decompressed_size: 0, + compressed_size: 0, + object_size: 0, + } + } + fn from_object_entry(kind: gix_object::Kind, entry: &data::Entry, compressed_size: usize) -> Self { + Self { + kind, + num_deltas: 0, + decompressed_size: entry.decompressed_size, + compressed_size, + object_size: entry.decompressed_size, + } + } +} + +/// Decompression of objects +impl File { + fn decoded_object_size(&self, size: u64) -> Result { + decoded_object_size(size, self.alloc_limit_bytes) + } + + /// Decompress the given `entry` into `out` and return the amount of bytes read from the pack data. + /// Note that `inflate` is not reset after usage, but will be reset before using it. + /// + /// _Note_ that this method does not resolve deltified objects, but merely decompresses their content + /// `out` is expected to be large enough to hold `entry.size` bytes. + pub fn decompress_entry( + &self, + entry: &data::Entry, + inflate: &mut zlib::Inflate, + out: &mut [u8], + ) -> Result { + let size: usize = entry.decompressed_size.try_into().map_err(|_| Error::OutOfMemory)?; + if out.len() < size { + return Err(Error::OutOfMemory); + } + self.decompress_entry_from_data_offset(entry.data_offset, inflate, &mut out[..size]) + } + + /// Obtain the [`Entry`][crate::data::Entry] at the given `offset` into the pack. + /// + /// The `offset` is typically obtained from the pack index file. + pub fn entry(&self, offset: data::Offset) -> Result { + let pack_offset: usize = offset.try_into().expect("offset representable by machine"); + if pack_offset > self.data_len() { + return Err(data::entry::decode::Error::Corrupt { + message: "an entry offset pointing beyond pack data", + }); + } + + let window = (self.data_len() - pack_offset).min(self.hash_len + 32); + let mut header = vec![0u8; window]; + self.read_exact_at(pack_offset, &mut header) + .map_err(|_| data::entry::decode::Error::Corrupt { + message: "failed to read entry header from pack data", + })?; + data::Entry::from_bytes(&header, offset, self.hash_len) + } + + /// Decompress the object expected at the given data offset, sans pack header. This information is only + /// known after the pack header was parsed. + /// Note that this method does not resolve deltified objects, but merely decompresses their content + /// `out` is expected to be large enough to hold `entry.size` bytes. + /// Returns the amount of packed bytes there read from the pack data file. + pub(crate) fn decompress_entry_from_data_offset( + &self, + data_offset: data::Offset, + inflate: &mut zlib::Inflate, + out: &mut [u8], + ) -> Result { + let (consumed_in, _consumed_out) = + self.decompress_complete_entry_from_data_offset(data_offset, inflate, out)?; + Ok(consumed_in) + } + + /// Like `decompress_entry_from_data_offset`, but returns `(consumed-input, consumed-output)`. + /// + /// The compressed stream must end exactly after producing `out.len()` bytes. Pack entry + /// headers are untrusted, so callers must not accept streams that stop early and + /// leave zero-filled slack in the destination buffer, nor streams that require more output + /// than the header promised. Both cases would make later delta parsing operate on bytes that + /// are not the entry payload described by the pack header. + pub(crate) fn decompress_complete_entry_from_data_offset( + &self, + data_offset: data::Offset, + inflate: &mut zlib::Inflate, + out: &mut [u8], + ) -> Result<(usize, usize), Error> { + let (status, consumed_in, consumed_out) = + self.decompress_entry_from_data_offset_unchecked(data_offset, inflate, out)?; + if status != zlib::Status::StreamEnd || consumed_out != out.len() { + return Err(data::entry::decode::Error::Corrupt { + message: "pack entry decompressed size does not match entry header", + } + .into()); + } + Ok((consumed_in, consumed_out)) + } + + /// Like [`Self::decompress_commplete_entry_from_data_offset()`], but allows callers to inspect incomplete streams. + /// + /// This is only for callers that intentionally decompress a prefix into a smaller buffer, such as + /// delta header probing. Full pack entry decoding should use [`Self::decompress_commplete_entry_from_data_offset()`]. + pub(crate) fn decompress_entry_from_data_offset_unchecked( + &self, + data_offset: data::Offset, + inflate: &mut zlib::Inflate, + out: &mut [u8], + ) -> Result<(zlib::Status, usize, usize), Error> { + let offset: usize = data_offset.try_into().expect("offset representable by machine"); + if offset >= self.data_len() { + return Err(data::entry::decode::Error::Corrupt { + message: "an entry data offset pointing beyond pack data", + } + .into()); + } + + inflate.reset(); + let mut chunk = [0u8; 8192]; + let mut in_pos = offset; + let status = loop { + let avail = (self.data_len() - in_pos).min(chunk.len()); + self.read_exact_at(in_pos, &mut chunk[..avail]).map_err(|_| { + Error::from(data::entry::decode::Error::Corrupt { + message: "failed to read pack entry data", + }) + })?; + let out_pos = inflate.state.total_out() as usize; + let before_in = inflate.state.total_in(); + let status = inflate + .state + .decompress(&chunk[..avail], &mut out[out_pos..], zlib::FlushDecompress::None) + .map_err(|err| Error::from(zlib::inflate::Error::from(err)))?; + let advanced_in = inflate.state.total_in() != before_in; + let advanced_out = inflate.state.total_out() as usize != out_pos; + in_pos = offset + inflate.state.total_in() as usize; + match status { + zlib::Status::StreamEnd => break zlib::Status::StreamEnd, + zlib::Status::Ok | zlib::Status::BufError => { + if avail == 0 || (!advanced_in && !advanced_out) { + break status; + } + } + } + }; + Ok(( + status, + inflate.state.total_in() as usize, + inflate.state.total_out() as usize, + )) + } + + /// Decode an entry, resolving delta's as needed, while growing the `out` vector if there is not enough + /// space to hold the result object. + /// + /// The `entry` determines which object to decode, and is commonly obtained with the help of a pack index file or through pack iteration. + /// `inflate` will be used for decompressing entries, and will not be reset after usage, but before first using it. + /// + /// `resolve` is a function to lookup objects with the given [`ObjectId`][gix_hash::ObjectId], in case the full object id is used to refer to + /// a base object, instead of an in-pack offset. + /// + /// `delta_cache` is a mechanism to avoid looking up base objects multiple times when decompressing multiple objects in a row. + /// Use a [Noop-Cache][cache::Never] to disable caching all together at the cost of repeating work. + pub fn decode_entry( + &self, + entry: data::Entry, + out: &mut Vec, + inflate: &mut zlib::Inflate, + resolve: &dyn Fn(&gix_hash::oid, &mut Vec) -> Option, + delta_cache: &mut dyn cache::DecodeEntry, + ) -> Result { + use crate::data::entry::Header::*; + match entry.header { + Tree | Blob | Commit | Tag => { + let size = self.decoded_object_size(entry.decompressed_size)?; + if let Some(additional) = size.checked_sub(out.len()) { + out.try_reserve(additional)?; + } + out.resize(size, 0); + self.decompress_entry(&entry, inflate, out.as_mut_slice()) + .map(|consumed_input| { + Outcome::from_object_entry( + entry.header.as_kind().expect("a non-delta entry"), + &entry, + consumed_input, + ) + }) + } + OfsDelta { .. } | RefDelta { .. } => self.resolve_deltas(entry, resolve, inflate, out, delta_cache), + } + } + + /// resolve: technically, this shouldn't ever be required as stored local packs don't refer to objects by id + /// that are outside of the pack. Unless, of course, the ref refers to an object within this pack, which means + /// it's very, very large as 20bytes are smaller than the corresponding MSB encoded number + fn resolve_deltas( + &self, + last: data::Entry, + resolve: &dyn Fn(&gix_hash::oid, &mut Vec) -> Option, + inflate: &mut zlib::Inflate, + out: &mut Vec, + cache: &mut dyn cache::DecodeEntry, + ) -> Result { + // all deltas, from the one that produces the desired object (first) to the oldest at the end of the chain + let mut chain = SmallVec::<[Delta; 10]>::default(); + let first_entry = last.clone(); + let mut cursor = last; + let mut base_buffer_size: Option = None; + let mut object_kind: Option = None; + let mut consumed_input: Option = None; + + // Find the first full base, either an undeltified object in the pack or a reference to another object. + let mut total_delta_data_size: u64 = 0; + while cursor.header.is_delta() { + if let Some((kind, packed_size)) = cache.get(self.id, cursor.data_offset, out) { + base_buffer_size = Some(out.len()); + object_kind = Some(kind); + // If the input entry is a cache hit, keep the packed size as it must be returned. + // Otherwise, the packed size will be determined later when decompressing the input delta + if total_delta_data_size == 0 { + consumed_input = Some(packed_size); + } + break; + } + // This is a pessimistic guess, as worst possible compression should not be bigger than the data itself. + // TODO: is this assumption actually true? + total_delta_data_size = total_delta_data_size + .checked_add(cursor.decompressed_size) + .ok_or(Error::OutOfMemory)?; + let decompressed_size = self.decoded_object_size(cursor.decompressed_size)?; + chain.push(Delta { + data: Range { + start: 0, + end: decompressed_size, + }, + base_size: 0, + result_size: 0, + decompressed_size, + data_offset: cursor.data_offset, + }); + use crate::data::entry::Header; + cursor = match cursor.header { + Header::OfsDelta { base_distance } => { + self.entry(cursor.checked_base_pack_offset(base_distance).ok_or( + crate::data::entry::decode::Error::Corrupt { + message: "an ofs-delta base distance pointing before pack start", + }, + )?)? + } + Header::RefDelta { base_id } => match resolve(base_id.as_ref(), out) { + Some(ResolvedBase::InPack(entry)) => entry, + Some(ResolvedBase::OutOfPack { end, kind }) => { + base_buffer_size = Some(end); + object_kind = Some(kind); + break; + } + None => return Err(Error::DeltaBaseUnresolved(base_id)), + }, + _ => unreachable!("cursor.is_delta() only allows deltas here"), + }; + } + + // This can happen if the cache held the first entry itself + // We will just treat it as an object then, even though it's technically incorrect. + if chain.is_empty() { + return Ok(Outcome::from_object_entry( + object_kind.expect("object kind as set by cache"), + &first_entry, + consumed_input.expect("consumed bytes as set by cache"), + )); + } + + // First pass will decompress all delta data and keep it in our output buffer + // []... + // so that we can find the biggest result size. + let total_delta_data_size: usize = total_delta_data_size.try_into().map_err(|_| Error::OutOfMemory)?; + + let chain_len = chain.len(); + let (first_buffer_end, second_buffer_end) = { + let delta_start = base_buffer_size.unwrap_or(0); + + let delta_range = Range { + start: delta_start, + end: delta_start + .checked_add(total_delta_data_size) + .ok_or(Error::OutOfMemory)?, + }; + out.try_reserve(delta_range.end.saturating_sub(out.len()))?; + out.resize(delta_range.end, 0); + + let mut instructions = &mut out[delta_range.clone()]; + let mut relative_delta_start = 0; + let mut biggest_result_size = 0; + for (delta_idx, delta) in chain.iter_mut().rev().enumerate() { + let (consumed_from_data_offset, consumed_out) = self.decompress_complete_entry_from_data_offset( + delta.data_offset, + inflate, + &mut instructions[..delta.decompressed_size], + )?; + let is_last_delta_to_be_applied = delta_idx + 1 == chain_len; + if is_last_delta_to_be_applied { + consumed_input = Some(consumed_from_data_offset); + } + + let current_delta = &instructions[..consumed_out]; + let (base_size, offset) = delta::decode_header_size(current_delta)?; + let mut bytes_consumed_by_header = offset; + biggest_result_size = biggest_result_size.max(base_size); + delta.base_size = self.decoded_object_size(base_size)?; + + let (result_size, offset) = delta::decode_header_size(¤t_delta[offset..])?; + bytes_consumed_by_header += offset; + biggest_result_size = biggest_result_size.max(result_size); + delta.result_size = self.decoded_object_size(result_size)?; + + // the absolute location into the instructions buffer, so we keep track of the end point of the last + delta.data.start = relative_delta_start + bytes_consumed_by_header; + delta.data.end = relative_delta_start + consumed_out; + relative_delta_start += delta.decompressed_size; + + instructions = &mut instructions[delta.decompressed_size..]; + } + + // Now we can produce a buffer like this + // [] + // from []... + if base_buffer_size.is_none() { + biggest_result_size = biggest_result_size.max(cursor.decompressed_size); + } + let biggest_result_size = self.decoded_object_size(biggest_result_size)?; + let first_buffer_size = biggest_result_size; + let second_buffer_size = first_buffer_size; + let out_size = first_buffer_size + .checked_add(second_buffer_size) + .and_then(|size| size.checked_add(total_delta_data_size)) + .ok_or(Error::OutOfMemory)?; + out.try_reserve(out_size.saturating_sub(out.len()))?; + out.resize(out_size, 0); + + // Now 'rescue' the deltas, because in the next step we possibly overwrite that portion + // of memory with the base object (in the majority of cases) + let second_buffer_end = { + let end = first_buffer_size + .checked_add(second_buffer_size) + .ok_or(Error::OutOfMemory)?; + // Move the decompressed delta instructions behind the two work buffers so they remain intact + // while we repurpose the front of `out` for base-object materialization and delta application. + out.copy_within(delta_range, end); + end + }; + + // If we don't have a out-of-pack object already, fill the base-buffer by decompressing the full object + // at which the cursor is left after the iteration + if base_buffer_size.is_none() { + let base_entry = cursor; + debug_assert!(!base_entry.header.is_delta()); + object_kind = base_entry.header.as_kind(); + let base_size = self.decoded_object_size(base_entry.decompressed_size)?; + let out_base = &mut out[..base_size]; + self.decompress_entry_from_data_offset(base_entry.data_offset, inflate, out_base)?; + } + + (first_buffer_size, second_buffer_end) + }; + + // From oldest to most recent, apply all deltas, swapping the buffer back and forth + // TODO: once we have more tests, we could optimize this memory-intensive work to + // analyse the delta-chains to only copy data once - after all, with 'copy-from-base' deltas, + // all data originates from one base at some point. + // `out` is: [source-buffer][target-buffer][max-delta-instructions-buffer] + let (buffers, instructions) = out.split_at_mut(second_buffer_end); + let (mut source_buf, mut target_buf) = buffers.split_at_mut(first_buffer_end); + + let mut last_result_size = None; + for ( + delta_idx, + Delta { + data, + base_size, + result_size, + .. + }, + ) in chain.into_iter().rev().enumerate() + { + let data = &mut instructions[data]; + if delta_idx + 1 == chain_len { + last_result_size = Some(result_size); + } + delta::apply(&source_buf[..base_size], &mut target_buf[..result_size], data)?; + // use the target as source for the next delta + std::mem::swap(&mut source_buf, &mut target_buf); + } + + let last_result_size = last_result_size.expect("at least one delta chain item"); + // uneven chains leave the target buffer after the source buffer + // FIXME(Performance) If delta-chains are uneven, we know we will have to copy bytes over here + // Instead we could use a different start buffer, to naturally end up with the result in the + // right one. + // However, this is a bit more complicated than just that - you have to deal with the base + // object, which should also be placed in the second buffer right away. You don't have that + // control/knowledge for out-of-pack bases, so this is a special case to deal with, too. + // Maybe these invariants can be represented in the type system though. + if chain_len % 2 == 1 { + // this seems inverted, but remember: we swapped the buffers on the last iteration + target_buf[..last_result_size].copy_from_slice(&source_buf[..last_result_size]); + } + debug_assert!(out.len() >= last_result_size); + out.truncate(last_result_size); + + let object_kind = object_kind.expect("a base object as root of any delta chain that we are here to resolve"); + let consumed_input = consumed_input.expect("at least one decompressed delta object"); + cache.put( + self.id, + first_entry.data_offset, + out.as_slice(), + object_kind, + consumed_input, + ); + Ok(Outcome { + kind: object_kind, + // technically depending on the cache, the chain size is not correct as it might + // have been cut short by a cache hit. The caller must deactivate the cache to get + // actual results + num_deltas: chain_len as u32, + decompressed_size: first_entry.decompressed_size, + compressed_size: consumed_input, + object_size: last_result_size as u64, + }) + } +} + +/// Convert user-controlled sizes from pack data into allocation sizes while enforcing the configured allocation cap. +fn decoded_object_size(size: u64, alloc_limit_bytes: Option) -> Result { + let size: usize = size.try_into().map_err(|_| Error::OutOfMemory)?; + if alloc_limit_bytes.is_some_and(|limit| size > limit) { + return Err(Error::OutOfMemory); + } + Ok(size) +} + +#[cfg(test)] +mod tests { + use gix_testtools::size_ok; + + use super::*; + + #[test] + fn size_of_decode_entry_outcome() { + let actual = std::mem::size_of::(); + let expected = 32; + assert!( + size_ok(actual, expected), + "this shouldn't change without use noticing as it's returned a lot: {actual} <~ {expected}" + ); + } +} diff --git a/knot2/third_party/gix-pack/src/data/file/decode/header.rs b/knot2/third_party/gix-pack/src/data/file/decode/header.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/file/decode/header.rs @@ -0,0 +1,152 @@ +use gix_features::zlib; + +use crate::{ + data, + data::{File, delta, file::decode::Error}, +}; + +/// A return value of a resolve function, which given an [`ObjectId`][gix_hash::ObjectId] determines where an object can be found. +#[derive(Debug, PartialEq, Eq, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum ResolvedBase { + /// Indicate an object is within this pack, at the given entry, and thus can be looked up locally. + InPack(data::Entry), + /// Indicates the object of `kind` was found outside of the pack. + OutOfPack { + /// The kind of object we found when reading the header of the out-of-pack base. + kind: gix_object::Kind, + /// The amount of deltas encountered if the object was packed as well. + num_deltas: Option, + }, +} + +/// Additional information and statistics about a successfully decoded object produced by [`File::decode_header()`]. +/// +/// Useful to understand the effectiveness of the pack compression or the cost of decompression. +#[derive(Debug, PartialEq, Eq, Hash, Ord, PartialOrd, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Outcome { + /// The kind of resolved object. + pub kind: gix_object::Kind, + /// The decompressed size of the object. + pub object_size: u64, + /// The amount of deltas in the chain of objects that had to be resolved beforehand. + pub num_deltas: u32, +} + +/// Obtain object information quickly. +impl File { + /// Resolve the object header information starting at `entry`, following the chain of entries as needed. + /// + /// The `entry` determines which object to decode, and is commonly obtained with the help of a pack index file or through pack iteration. + /// `inflate` will be used for (partially) decompressing entries, and will be reset before first use, but not after the last use. + /// + /// `resolve` is a function to lookup objects with the given [`ObjectId`][gix_hash::ObjectId], in case the full object id + /// is used to refer to a base object, instead of an in-pack offset. + /// + /// For delta entries, this only probes the initial delta header bytes to determine the result + /// object size. It can reject streams that end or overflow within that probe, but it does not + /// fully validate that the compressed stream produces exactly the decompressed size declared in + /// the pack entry header. Use [`File::decode_entry()`][crate::data::File::decode_entry()] when + /// callers need that full validation. + pub fn decode_header( + &self, + mut entry: data::Entry, + inflate: &mut zlib::Inflate, + resolve: &dyn Fn(&gix_hash::oid) -> Option, + ) -> Result { + use crate::data::entry::Header::*; + let mut num_deltas = 0; + let mut first_delta_decompressed_size = None::; + loop { + match entry.header { + Tree | Blob | Commit | Tag => { + return Ok(Outcome { + kind: entry.header.as_kind().expect("always valid for non-refs"), + object_size: first_delta_decompressed_size.unwrap_or(entry.decompressed_size), + num_deltas, + }); + } + OfsDelta { base_distance } => { + num_deltas += 1; + if first_delta_decompressed_size.is_none() { + first_delta_decompressed_size = Some(self.decode_delta_object_size(inflate, &entry)?); + } + entry = self.entry(entry.checked_base_pack_offset(base_distance).ok_or( + crate::data::entry::decode::Error::Corrupt { + message: "an ofs-delta base distance pointing before pack start", + }, + )?)?; + } + RefDelta { base_id } => { + num_deltas += 1; + if first_delta_decompressed_size.is_none() { + first_delta_decompressed_size = Some(self.decode_delta_object_size(inflate, &entry)?); + } + match resolve(base_id.as_ref()) { + Some(ResolvedBase::InPack(base_entry)) => entry = base_entry, + Some(ResolvedBase::OutOfPack { + kind, + num_deltas: origin_num_deltas, + }) => { + return Ok(Outcome { + kind, + object_size: first_delta_decompressed_size.unwrap_or(entry.decompressed_size), + num_deltas: origin_num_deltas.unwrap_or_default() + num_deltas, + }); + } + None => return Err(Error::DeltaBaseUnresolved(base_id)), + } + } + } + } + } + + /// Decode the result object size from the initial delta header bytes in `inflate`, using `entry` + /// for offsets. + /// + /// This intentionally mirrors Git's cheap header probe: only the first 20 decompressed bytes + /// are inspected, which is enough for the two `u64` varints that make up a valid delta header. + /// If the zlib stream ends within that probe, we can reject declared-size mismatches here. + /// Otherwise this result only proves that the delta header prefix is parseable; full + /// decompression through `decode_entry()` must still validate that the stream length matches + /// the pack entry header. + #[inline] + fn decode_delta_object_size(&self, inflate: &mut zlib::Inflate, entry: &data::Entry) -> Result { + let mut buf = [0_u8; 20]; + let max_size = entry.decompressed_size.min(buf.len() as u64) as usize; + let (status, _consumed_in, consumed_out) = + self.decompress_entry_from_data_offset_unchecked(entry.data_offset, inflate, &mut buf[..max_size])?; + if status == zlib::Status::StreamEnd { + if consumed_out as u64 != entry.decompressed_size { + return Err(data::entry::decode::Error::Corrupt { + message: "pack entry decompressed to fewer bytes than declared in the entry header", + } + .into()); + } + } else if entry.decompressed_size == max_size as u64 { + return Err(data::entry::decode::Error::Corrupt { + message: "pack entry decompressed to more bytes than declared in the entry header", + } + .into()); + } + let buf = &buf[..consumed_out]; + let (_base_size, offset) = delta::decode_header_size(buf)?; + let (result_size, _offset) = delta::decode_header_size(&buf[offset..])?; + Ok(result_size) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn size_of_decode_entry_outcome() { + assert_eq!( + std::mem::size_of::(), + 16, + "this shouldn't change without use noticing as it's returned a lot" + ); + } +} diff --git a/knot2/third_party/gix-pack/src/data/file/decode/mod.rs b/knot2/third_party/gix-pack/src/data/file/decode/mod.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/file/decode/mod.rs @@ -0,0 +1,31 @@ +use std::collections::TryReserveError; + +/// +pub mod entry; +/// +pub mod header; + +/// Returned by [`File::decode_header()`][crate::data::File::decode_header()], +/// [`File::decode_entry()`][crate::data::File::decode_entry()] and . +/// [`File::decompress_entry()`][crate::data::File::decompress_entry()] +#[derive(thiserror::Error, Debug)] +#[allow(missing_docs)] +pub enum Error { + #[error("Failed to decompress pack entry")] + ZlibInflate(#[from] gix_features::zlib::inflate::Error), + #[error("A delta chain could not be followed as the ref base with id {0} could not be found")] + DeltaBaseUnresolved(gix_hash::ObjectId), + #[error(transparent)] + EntryType(#[from] crate::data::entry::decode::Error), + #[error("Entry too large to fit in memory")] + OutOfMemory, + #[error(transparent)] + Delta(#[from] crate::data::delta::apply::Error), +} + +impl From for Error { + #[cold] + fn from(_: TryReserveError) -> Self { + Self::OutOfMemory + } +} diff --git a/knot2/third_party/gix-pack/src/data/output/count/mod.rs b/knot2/third_party/gix-pack/src/data/output/count/mod.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/output/count/mod.rs @@ -0,0 +1,49 @@ +use gix_hash::ObjectId; + +use crate::data::output::Count; + +/// Specifies how the pack location was handled during counting +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum PackLocation { + /// We did not lookup this object + NotLookedUp, + /// The object was looked up and there may be a location in a pack, along with entry information + LookedUp(Option), +} + +impl PackLocation { + /// Directly go through to `LookedUp` variant, panic otherwise + pub fn is_none(&self) -> bool { + match self { + PackLocation::LookedUp(opt) => opt.is_none(), + PackLocation::NotLookedUp => unreachable!("must have been resolved"), + } + } + /// Directly go through to `LookedUp` variant, panic otherwise + pub fn as_ref(&self) -> Option<&crate::data::entry::Location> { + match self { + PackLocation::LookedUp(opt) => opt.as_ref(), + PackLocation::NotLookedUp => unreachable!("must have been resolved"), + } + } +} + +impl Count { + /// Create a new instance from the given `oid` and its corresponding location. + pub fn from_data(oid: impl Into, location: Option) -> Self { + Count { + id: oid.into(), + entry_pack_location: PackLocation::LookedUp(location), + } + } +} + +#[path = "objects/mod.rs"] +mod objects_impl; +pub use objects_impl::{objects, objects_unthreaded}; + +/// +pub mod objects { + pub use super::objects_impl::{Error, ObjectExpansion, Options, Outcome}; +} diff --git a/knot2/third_party/gix-pack/src/data/output/entry/iter_from_counts.rs b/knot2/third_party/gix-pack/src/data/output/entry/iter_from_counts.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/output/entry/iter_from_counts.rs @@ -0,0 +1,432 @@ +pub(crate) mod function { + use std::{cmp::Ordering, sync::Arc}; + + use gix_features::{ + parallel, + parallel::SequenceId, + progress::{ + Progress, + prodash::{Count, DynNestedProgress}, + }, + }; + + use super::{Error, Mode, Options, Outcome, ProgressId, reduce, util}; + use crate::data::output; + + /// Given a known list of object `counts`, calculate entries ready to be put into a data pack. + /// + /// This allows objects to be written quite soon without having to wait for the entire pack to be built in memory. + /// A chunk of objects is held in memory and compressed using DEFLATE, and serve the output of this iterator. + /// That way slow writers will naturally apply back pressure, and communicate to the implementation that more time can be + /// spent compressing objects. + /// + /// * `counts` + /// * A list of previously counted objects to add to the pack. Duplication checks are not performed, no object is expected to be duplicated. + /// * `progress` + /// * a way to obtain progress information + /// * `options` + /// * more configuration + /// + /// _Returns_ the checksum of the pack + /// + /// ## Discussion + /// + /// ### Advantages + /// + /// * Begins writing immediately and supports back-pressure. + /// * Abstract over object databases and how input is provided. + /// + /// ### Disadvantages + /// + /// * ~~currently there is no way to easily write the pack index, even though the state here is uniquely positioned to do + /// so with minimal overhead (especially compared to `gix index-from-pack`)~~ Probably works now by chaining Iterators + /// or keeping enough state to write a pack and then generate an index with recorded data. + /// + pub fn iter_from_counts( + mut counts: Vec, + db: Find, + mut progress: Box, + Options { + version, + mode, + allow_thin_pack, + thread_limit, + chunk_size, + }: Options, + ) -> impl Iterator), Error>> + + parallel::reduce::Finalize> + where + Find: crate::Find + Send + Clone + 'static, + { + assert!( + matches!(version, crate::data::Version::V2), + "currently we can only write version 2" + ); + let (chunk_size, thread_limit, _) = + parallel::optimize_chunk_size_and_thread_limit(chunk_size, Some(counts.len()), thread_limit, None); + { + let progress = Arc::new(parking_lot::Mutex::new( + progress.add_child_with_id("resolving".into(), ProgressId::ResolveCounts.into()), + )); + progress.lock().init(None, gix_features::progress::count("counts")); + let enough_counts_present = counts.len() > 4_000; + let start = std::time::Instant::now(); + parallel::in_parallel_if( + || enough_counts_present, + counts.chunks_mut(chunk_size), + thread_limit, + |_n| Vec::::new(), + { + let progress = Arc::clone(&progress); + let db = db.clone(); + move |chunk, buf| { + let chunk_size = chunk.len(); + for count in chunk { + use crate::data::output::count::PackLocation::*; + match count.entry_pack_location { + LookedUp(_) => continue, + NotLookedUp => count.entry_pack_location = LookedUp(db.location_by_oid(&count.id, buf)), + } + } + progress.lock().inc_by(chunk_size); + Ok::<_, ()>(()) + } + }, + parallel::reduce::IdentityWithResult::<(), ()>::default(), + ) + .expect("infallible - we ignore none-existing objects"); + progress.lock().show_throughput(start); + } + let counts_range_by_pack_id = match mode { + Mode::PackCopyAndBaseObjects => { + let mut progress = progress.add_child_with_id("sorting".into(), ProgressId::SortEntries.into()); + progress.init(Some(counts.len()), gix_features::progress::count("counts")); + let start = std::time::Instant::now(); + + use crate::data::output::count::PackLocation::*; + counts.sort_by(|lhs, rhs| match (&lhs.entry_pack_location, &rhs.entry_pack_location) { + (LookedUp(None), LookedUp(None)) => Ordering::Equal, + (LookedUp(Some(_)), LookedUp(None)) => Ordering::Greater, + (LookedUp(None), LookedUp(Some(_))) => Ordering::Less, + (LookedUp(Some(lhs)), LookedUp(Some(rhs))) => lhs + .pack_id + .cmp(&rhs.pack_id) + .then(lhs.pack_offset.cmp(&rhs.pack_offset)), + (_, _) => unreachable!("counts were resolved beforehand"), + }); + + let mut index: Vec<(u32, std::ops::Range)> = Vec::new(); + let mut chunks_pack_start = counts.partition_point(|e| e.entry_pack_location.is_none()); + let mut slice = &counts[chunks_pack_start..]; + while !slice.is_empty() { + let current_pack_id = slice[0].entry_pack_location.as_ref().expect("packed object").pack_id; + let pack_end = slice.partition_point(|e| { + e.entry_pack_location.as_ref().expect("packed object").pack_id == current_pack_id + }); + index.push((current_pack_id, chunks_pack_start..chunks_pack_start + pack_end)); + slice = &slice[pack_end..]; + chunks_pack_start += pack_end; + } + + progress.set(counts.len()); + progress.show_throughput(start); + + index + } + }; + + let counts = Arc::new(counts); + let progress = Arc::new(parking_lot::Mutex::new(progress)); + let chunks = util::ChunkRanges::new(chunk_size, counts.len()); + + parallel::reduce::Stepwise::new( + chunks.enumerate(), + thread_limit, + { + let progress = Arc::clone(&progress); + move |n| { + ( + Vec::new(), // object data buffer + progress + .lock() + .add_child_with_id(format!("thread {n}"), gix_features::progress::UNKNOWN), + ) + } + }, + { + let counts = Arc::clone(&counts); + move |(chunk_id, chunk_range): (SequenceId, std::ops::Range), (buf, progress)| { + let mut out = Vec::new(); + let chunk = &counts[chunk_range]; + let mut stats = Outcome::default(); + let mut pack_offsets_to_id = None; + progress.init(Some(chunk.len()), gix_features::progress::count("objects")); + + for count in chunk.iter() { + out.push(match count + .entry_pack_location + .as_ref() + .and_then(|l| db.entry_by_location(l).map(|pe| (l, pe))) + { + Some((location, pack_entry)) => { + if let Some((cached_pack_id, _)) = &pack_offsets_to_id { + if *cached_pack_id != location.pack_id { + pack_offsets_to_id = None; + } + } + let pack_range = counts_range_by_pack_id[counts_range_by_pack_id + .binary_search_by_key(&location.pack_id, |e| e.0) + .expect("pack-id always present")] + .1 + .clone(); + let base_index_offset = pack_range.start; + let counts_in_pack = &counts[pack_range]; + let entry = output::Entry::from_pack_entry( + pack_entry, + count, + counts_in_pack, + base_index_offset, + allow_thin_pack.then_some({ + |pack_id, base_offset| { + let (cached_pack_id, cache) = pack_offsets_to_id.get_or_insert_with(|| { + db.pack_offsets_and_oid(pack_id) + .map(|mut v| { + v.sort_by_key(|e| e.0); + (pack_id, v) + }) + .expect("pack used for counts is still available") + }); + debug_assert_eq!(*cached_pack_id, pack_id); + stats.ref_delta_objects += 1; + cache + .binary_search_by_key(&base_offset, |e| e.0) + .ok() + .map(|idx| cache[idx].1) + } + }), + version, + ); + match entry { + Some(entry) => { + stats.objects_copied_from_pack += 1; + entry + } + None => match db.try_find(&count.id, buf).map_err(Error::Find)? { + Some((obj, _location)) => { + stats.decoded_and_recompressed_objects += 1; + output::Entry::from_data(count, &obj) + } + None => { + stats.missing_objects += 1; + Ok(output::Entry::invalid()) + } + }, + } + } + None => match db.try_find(&count.id, buf).map_err(Error::Find)? { + Some((obj, _location)) => { + stats.decoded_and_recompressed_objects += 1; + output::Entry::from_data(count, &obj) + } + None => { + stats.missing_objects += 1; + Ok(output::Entry::invalid()) + } + }, + }?); + progress.inc(); + } + Ok((chunk_id, out, stats)) + } + }, + reduce::Statistics::default(), + ) + } +} + +mod util { + #[derive(Clone)] + pub struct ChunkRanges { + cursor: usize, + size: usize, + len: usize, + } + + impl ChunkRanges { + pub fn new(size: usize, total: usize) -> Self { + ChunkRanges { + cursor: 0, + size, + len: total, + } + } + } + + impl Iterator for ChunkRanges { + type Item = std::ops::Range; + + fn next(&mut self) -> Option { + if self.cursor >= self.len { + None + } else { + let upper = (self.cursor + self.size).min(self.len); + let range = self.cursor..upper; + self.cursor = upper; + Some(range) + } + } + } +} + +mod reduce { + use std::marker::PhantomData; + + use gix_features::{parallel, parallel::SequenceId}; + + use super::Outcome; + use crate::data::output; + + pub struct Statistics { + total: Outcome, + _err: PhantomData, + } + + impl Default for Statistics { + fn default() -> Self { + Statistics { + total: Default::default(), + _err: PhantomData, + } + } + } + + impl parallel::Reduce for Statistics { + type Input = Result<(SequenceId, Vec, Outcome), Error>; + type FeedProduce = (SequenceId, Vec); + type Output = Outcome; + type Error = Error; + + fn feed(&mut self, item: Self::Input) -> Result { + item.map(|(cid, entries, stats)| { + self.total.aggregate(stats); + (cid, entries) + }) + } + + fn finalize(self) -> Result { + Ok(self.total) + } + } +} + +mod types { + use crate::data::output::entry; + + /// Information gathered during the run of [`iter_from_counts()`][crate::data::output::entry::iter_from_counts()]. + #[derive(Default, PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] + #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] + pub struct Outcome { + /// The amount of fully decoded objects. These are the most expensive as they are fully decoded. + pub decoded_and_recompressed_objects: usize, + /// The amount of objects that could not be located despite them being mentioned during iteration + pub missing_objects: usize, + /// The amount of base or delta objects that could be copied directly from the pack. These are cheapest as they + /// only cost a memory copy for the most part. + pub objects_copied_from_pack: usize, + /// The amount of objects that ref to their base as ref-delta, an indication for a thin back being created. + pub ref_delta_objects: usize, + } + + impl Outcome { + pub(in crate::data::output::entry) fn aggregate( + &mut self, + Outcome { + decoded_and_recompressed_objects: decoded_objects, + missing_objects, + objects_copied_from_pack, + ref_delta_objects, + }: Self, + ) { + self.decoded_and_recompressed_objects += decoded_objects; + self.missing_objects += missing_objects; + self.objects_copied_from_pack += objects_copied_from_pack; + self.ref_delta_objects += ref_delta_objects; + } + } + + /// The way the iterator operates. + #[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] + #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] + pub enum Mode { + /// Copy base objects and deltas from packs, while non-packed objects will be treated as base objects + /// (i.e. without trying to delta compress them). This is a fast way of obtaining a back while benefiting + /// from existing pack compression and spending the smallest possible time on compressing unpacked objects at + /// the cost of bandwidth. + PackCopyAndBaseObjects, + } + + /// Configuration options for the pack generation functions provided in [`iter_from_counts()`][crate::data::output::entry::iter_from_counts()]. + #[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] + #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] + pub struct Options { + /// The amount of threads to use at most when resolving the pack. If `None`, all logical cores are used. + pub thread_limit: Option, + /// The algorithm to produce a pack + pub mode: Mode, + /// If set, the resulting back can have deltas that refer to an object which is not in the pack. This can happen + /// if the initial counted objects do not contain an object that an existing packed delta refers to, for example, because + /// it wasn't part of the iteration, for instance when the iteration was performed on tree deltas or only a part of the + /// commit graph. Please note that thin packs are not valid packs at rest, thus they are only valid for packs in transit. + /// + /// If set to false, delta objects will be decompressed and recompressed as base objects. + pub allow_thin_pack: bool, + /// The amount of objects per chunk or unit of work to be sent to threads for processing + /// TODO: could this become the window size? + pub chunk_size: usize, + /// The pack data version to produce for each entry + pub version: crate::data::Version, + } + + impl Default for Options { + fn default() -> Self { + Options { + thread_limit: None, + mode: Mode::PackCopyAndBaseObjects, + allow_thin_pack: false, + chunk_size: 10, + version: Default::default(), + } + } + } + + /// The error returned by the pack generation function [`iter_from_counts()`][crate::data::output::entry::iter_from_counts()]. + #[derive(Debug, thiserror::Error)] + #[allow(missing_docs)] + pub enum Error { + #[error(transparent)] + Find(gix_object::find::Error), + #[error(transparent)] + NewEntry(#[from] entry::Error), + } + + /// The progress ids used in [`write_to_directory()`][crate::Bundle::write_to_directory()]. + /// + /// Use this information to selectively extract the progress of interest in case the parent application has custom visualization. + #[derive(Debug, Copy, Clone)] + pub enum ProgressId { + /// The amount of [`Count`][crate::data::output::Count] objects which are resolved to their pack location. + ResolveCounts, + /// Layout pack entries for placement into a pack (by pack-id and by offset). + SortEntries, + } + + impl From for gix_features::progress::Id { + fn from(v: ProgressId) -> Self { + match v { + ProgressId::ResolveCounts => *b"ECRC", + ProgressId::SortEntries => *b"ECSE", + } + } + } +} +pub use types::{Error, Mode, Options, Outcome, ProgressId}; diff --git a/knot2/third_party/gix-pack/src/data/output/entry/mod.rs b/knot2/third_party/gix-pack/src/data/output/entry/mod.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/output/entry/mod.rs @@ -0,0 +1,186 @@ +use std::io::Write; + +use gix_hash::ObjectId; + +use crate::{data, data::output, find}; + +/// +pub mod iter_from_counts; +pub use iter_from_counts::function::iter_from_counts; + +/// The kind of pack entry to be written +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum Kind { + /// A complete base object, including its kind + Base(gix_object::Kind), + /// A delta against the object with the given index. It's always an index that was already encountered to refer only + /// to object we have written already. + DeltaRef { + /// The absolute index to the object to serve as base. It's up to the writer to maintain enough state to allow producing + /// a packed delta object from it. + object_index: usize, + }, + /// A delta against the given object as identified by its `ObjectId`. + /// This is the case for thin packs only, i.e. those that are sent over the wire. + /// Note that there is the option of the `ObjectId` being used to refer to an object within + /// the same pack, but it's a discontinued practice which won't be encountered here. + DeltaOid { + /// The object serving as base for this delta + id: ObjectId, + }, +} + +/// The error returned by [`output::Entry::from_data()`]. +#[allow(missing_docs)] +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("{0}")] + ZlibDeflate(#[from] std::io::Error), + #[error(transparent)] + EntryType(#[from] crate::data::entry::decode::Error), +} + +impl output::Entry { + /// An object which can be identified as invalid easily which happens if objects didn't exist even if they were referred to. + pub fn invalid() -> output::Entry { + output::Entry { + id: gix_hash::Kind::shortest().null(), // NOTE: the actual object hash used in the repo doesn't matter here, this is a sentinel value. + kind: Kind::Base(gix_object::Kind::Blob), + decompressed_size: 0, + compressed_data: vec![], + } + } + + /// Returns true if this object doesn't really exist but still has to be handled responsibly + /// + /// Note that this is true for tree entries that are commits/git submodules, or for objects which aren't present in our local clone + /// due to shallow clones. + pub fn is_invalid(&self) -> bool { + self.id.is_null() + } + + /// Create an Entry from a previously counted object which is located in a pack. It's `entry` is provided here. + /// The `version` specifies what kind of target `Entry` version the caller desires. + pub fn from_pack_entry( + mut entry: find::Entry, + count: &output::Count, + potential_bases: &[output::Count], + bases_index_offset: usize, + pack_offset_to_oid: Option Option>, + target_version: data::Version, + ) -> Option> { + if entry.version != target_version { + return None; + } + + let pack_offset_must_be_zero = 0; + let pack_entry = match data::Entry::from_bytes(&entry.data, pack_offset_must_be_zero, count.id.as_slice().len()) + { + Ok(e) => e, + Err(err) => return Some(Err(err.into())), + }; + + use crate::data::entry::Header::*; + match pack_entry.header { + Commit => Some(output::entry::Kind::Base(gix_object::Kind::Commit)), + Tree => Some(output::entry::Kind::Base(gix_object::Kind::Tree)), + Blob => Some(output::entry::Kind::Base(gix_object::Kind::Blob)), + Tag => Some(output::entry::Kind::Base(gix_object::Kind::Tag)), + OfsDelta { base_distance } => { + let pack_location = count.entry_pack_location.as_ref().expect("packed"); + let base_offset = pack_location + .pack_offset + .checked_sub(base_distance) + .expect("pack-offset - distance is firmly within the pack"); + potential_bases + .binary_search_by(|e| { + e.entry_pack_location + .as_ref() + .expect("packed") + .pack_offset + .cmp(&base_offset) + }) + .ok() + .map(|idx| output::entry::Kind::DeltaRef { + object_index: idx + bases_index_offset, + }) + .or_else(|| { + pack_offset_to_oid + .and_then(|mut f| f(pack_location.pack_id, base_offset)) + .map(|id| output::entry::Kind::DeltaOid { id }) + }) + } + RefDelta { base_id: _ } => None, // ref deltas are for thin packs or legacy, repack them as base objects + } + .map(|kind| { + Ok(output::Entry { + id: count.id.to_owned(), + kind, + decompressed_size: pack_entry.decompressed_size as usize, + compressed_data: { + entry.data.copy_within(pack_entry.data_offset as usize.., 0); + entry.data.resize( + entry.data.len() + - usize::try_from(pack_entry.data_offset).expect("offset representable as usize"), + 0, + ); + entry.data + }, + }) + }) + } + + /// Create a new instance from the given `oid` and its corresponding git object data `obj`. + pub fn from_data(count: &output::Count, obj: &gix_object::Data<'_>) -> Result { + Ok(output::Entry { + id: count.id.to_owned(), + kind: Kind::Base(obj.kind), + decompressed_size: obj.data.len(), + compressed_data: { + let mut out = gix_features::zlib::stream::deflate::Write::new(Vec::new()); + if let Err(err) = std::io::copy(&mut &*obj.data, &mut out) { + match err.kind() { + std::io::ErrorKind::Other => return Err(Error::ZlibDeflate(err)), + err => unreachable!("Should never see other errors than zlib, but got {:?}", err), + } + } + out.flush()?; + out.into_inner() + }, + }) + } + + /// Transform ourselves into pack entry header of `version` which can be written into a pack. + /// + /// `index_to_pack(object_index) -> pack_offset` is a function to convert the base object's index into + /// the input object array (if each object is numbered) to an offset into the pack. + /// This information is known to the one calling the method. + pub fn to_entry_header( + &self, + version: data::Version, + index_to_base_distance: impl FnOnce(usize) -> u64, + ) -> data::entry::Header { + assert!( + matches!(version, data::Version::V2), + "we can only write V2 pack entries for now" + ); + + use Kind::*; + match self.kind { + Base(kind) => { + use gix_object::Kind::*; + match kind { + Tree => data::entry::Header::Tree, + Blob => data::entry::Header::Blob, + Commit => data::entry::Header::Commit, + Tag => data::entry::Header::Tag, + } + } + DeltaOid { id } => data::entry::Header::RefDelta { base_id: id.to_owned() }, + DeltaRef { object_index } => data::entry::Header::OfsDelta { + base_distance: index_to_base_distance(object_index), + }, + } + } +} diff --git a/knot2/third_party/gix-pack/src/data/output/count/objects/mod.rs b/knot2/third_party/gix-pack/src/data/output/count/objects/mod.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/output/count/objects/mod.rs @@ -0,0 +1,441 @@ +use std::{cell::RefCell, sync::atomic::AtomicBool}; + +use gix_features::parallel; +use gix_hash::ObjectId; + +use crate::data::output; + +pub(in crate::data::output::count::objects_impl) mod reduce; +mod util; + +mod types; +pub use types::{Error, ObjectExpansion, Options, Outcome}; + +mod tree; + +/// Generate [`Count`][output::Count]s from input `objects` with object expansion based on [`options`][Options] +/// to learn which objects would constitute a pack. This step is required to know exactly how many objects would +/// be in a pack while keeping data around to minimize database object access. +/// +/// A [`Count`][output::Count] object maintains enough state to greatly accelerate future access of packed objects. +/// +/// * `db` - the object store to use for accessing objects. +/// * `objects_ids` +/// * A list of objects IDs to add to the pack. Duplication checks are performed so no object is ever added to a pack twice. +/// * Objects may be expanded based on the provided [`options`][Options] +/// * `objects` +/// * count the amount of objects we encounter +/// * `should_interrupt` +/// * A flag that is set to true if the operation should stop +/// * `options` +/// * more configuration +pub fn objects( + db: Find, + objects_ids: Box>> + Send>, + objects: &dyn gix_features::progress::Count, + should_interrupt: &AtomicBool, + Options { + thread_limit, + input_object_expansion, + chunk_size, + }: Options, +) -> Result<(Vec, Outcome), Error> +where + Find: crate::Find + Send + Clone, +{ + let lower_bound = objects_ids.size_hint().0; + let (chunk_size, thread_limit, _) = parallel::optimize_chunk_size_and_thread_limit( + chunk_size, + if lower_bound == 0 { None } else { Some(lower_bound) }, + thread_limit, + None, + ); + let chunks = gix_features::iter::Chunks { + inner: objects_ids, + size: chunk_size, + }; + let seen_objs = gix_hashtable::sync::ObjectIdMap::default(); + let objects = objects.counter(); + + parallel::in_parallel( + chunks, + thread_limit, + { + move |_| { + ( + Vec::new(), // object data buffer + Vec::new(), // object data buffer 2 to hold two objects at a time + objects.clone(), + ) + } + }, + { + let seen_objs = &seen_objs; + move |oids: Vec<_>, (buf1, buf2, objects)| { + expand::this( + &db, + input_object_expansion, + seen_objs, + &mut oids.into_iter(), + buf1, + buf2, + objects, + should_interrupt, + true, /*allow pack lookups*/ + ) + } + }, + reduce::Statistics::new(), + ) +} + +/// Like [`objects()`] but using a single thread only to mostly save on the otherwise required overhead. +pub fn objects_unthreaded( + db: &dyn crate::Find, + object_ids: &mut dyn Iterator>>, + objects: &dyn gix_features::progress::Count, + should_interrupt: &AtomicBool, + input_object_expansion: ObjectExpansion, +) -> Result<(Vec, Outcome), Error> { + let seen_objs = RefCell::new(gix_hashtable::HashSet::default()); + + let (mut buf1, mut buf2) = (Vec::new(), Vec::new()); + expand::this( + db, + input_object_expansion, + &seen_objs, + object_ids, + &mut buf1, + &mut buf2, + &objects.counter(), + should_interrupt, + false, /*allow pack lookups*/ + ) +} + +mod expand { + use std::{ + cell::RefCell, + sync::atomic::{AtomicBool, Ordering}, + }; + + use gix_hash::{ObjectId, oid}; + use gix_object::{CommitRefIter, Data, TagRefIter}; + + use super::{ + tree, + types::{Error, ObjectExpansion, Outcome}, + util, + }; + use crate::{ + FindExt, + data::{output, output::count::PackLocation}, + }; + + #[allow(clippy::too_many_arguments)] + pub fn this( + db: &dyn crate::Find, + input_object_expansion: ObjectExpansion, + seen_objs: &impl util::InsertImmutable, + oids: &mut dyn Iterator>>, + buf1: &mut Vec, + #[allow(clippy::ptr_arg)] buf2: &mut Vec, + objects: &gix_features::progress::AtomicStep, + should_interrupt: &AtomicBool, + allow_pack_lookups: bool, + ) -> Result<(Vec, Outcome), Error> { + use ObjectExpansion::*; + + let mut out = Vec::new(); + let mut tree_traversal_state = gix_traverse::tree::breadthfirst::State::default(); + let mut tree_diff_state = gix_diff::tree::State::default(); + let mut parent_commit_ids = Vec::new(); + let mut traverse_delegate = tree::traverse::AllUnseen::new(seen_objs); + let mut changes_delegate = tree::changes::AllNew::new(seen_objs); + let mut outcome = Outcome::default(); + + let stats = &mut outcome; + for id in oids { + if should_interrupt.load(Ordering::Relaxed) { + return Err(Error::Interrupted); + } + + let id = id.map_err(Error::InputIteration)?; + let (obj, location) = db.find(&id, buf1)?; + stats.input_objects += 1; + match input_object_expansion { + TreeAdditionsComparedToAncestor => { + use gix_object::Kind::*; + let mut obj = obj; + let mut location = location; + let mut id = id.to_owned(); + + loop { + push_obj_count_unique(&mut out, seen_objs, &id, location, objects, stats, false); + match obj.kind { + Tree | Blob => break, + Tag => { + id = TagRefIter::from_bytes(obj.data, obj.object_hash) + .target_id() + .expect("every tag has a target"); + let tmp = db.find(&id, buf1)?; + + obj = tmp.0; + location = tmp.1; + + stats.expanded_objects += 1; + continue; + } + Commit => { + let current_tree_iter = { + let mut commit_iter = CommitRefIter::from_bytes(obj.data, obj.object_hash); + let tree_id = commit_iter.tree_id().expect("every commit has a tree"); + parent_commit_ids.clear(); + for token in commit_iter { + match token { + Ok(gix_object::commit::ref_iter::Token::Parent { id }) => { + parent_commit_ids.push(id); + } + Ok(_) => break, + Err(err) => return Err(Error::CommitDecode(err)), + } + } + let (obj, location) = db.find(&tree_id, buf1)?; + push_obj_count_unique( + &mut out, seen_objs, &tree_id, location, objects, stats, true, + ); + gix_object::TreeRefIter::from_bytes(obj.data, obj.object_hash) + }; + + let objects_ref = if parent_commit_ids.is_empty() { + traverse_delegate.clear(); + let objects = ExpandedCountingObjects::new(db, out, objects); + gix_traverse::tree::breadthfirst( + current_tree_iter, + &mut tree_traversal_state, + &objects, + &mut traverse_delegate, + ) + .map_err(Error::TreeTraverse)?; + out = objects.dissolve(stats); + &traverse_delegate.non_trees + } else { + for commit_id in &parent_commit_ids { + let parent_tree_id = { + let (parent_commit_obj, location) = db.find(commit_id, buf2)?; + + push_obj_count_unique( + &mut out, seen_objs, commit_id, location, objects, stats, true, + ); + CommitRefIter::from_bytes( + parent_commit_obj.data, + parent_commit_obj.object_hash, + ) + .tree_id() + .expect("every commit has a tree") + }; + let parent_tree = { + let (parent_tree_obj, location) = db.find(&parent_tree_id, buf2)?; + push_obj_count_unique( + &mut out, + seen_objs, + &parent_tree_id, + location, + objects, + stats, + true, + ); + gix_object::TreeRefIter::from_bytes( + parent_tree_obj.data, + parent_tree_obj.object_hash, + ) + }; + + changes_delegate.clear(); + let objects = CountingObjects::new(db); + gix_diff::tree( + parent_tree, + current_tree_iter, + &mut tree_diff_state, + &objects, + &mut changes_delegate, + ) + .map_err(Error::TreeChanges)?; + stats.decoded_objects += objects.into_count(); + } + &changes_delegate.objects + }; + for id in objects_ref.iter() { + out.push(id_to_count(db, buf2, id, objects, stats, allow_pack_lookups)); + } + break; + } + } + } + } + TreeContents => { + use gix_object::Kind::*; + let mut id = id; + let mut obj = (obj, location); + loop { + push_obj_count_unique(&mut out, seen_objs, &id, obj.1.clone(), objects, stats, false); + match obj.0.kind { + Tree => { + traverse_delegate.clear(); + { + let objects = ExpandedCountingObjects::new(db, out, objects); + gix_traverse::tree::breadthfirst( + gix_object::TreeRefIter::from_bytes(obj.0.data, obj.0.object_hash), + &mut tree_traversal_state, + &objects, + &mut traverse_delegate, + ) + .map_err(Error::TreeTraverse)?; + out = objects.dissolve(stats); + } + for id in &traverse_delegate.non_trees { + out.push(id_to_count(db, buf1, id, objects, stats, allow_pack_lookups)); + } + break; + } + Commit => { + id = CommitRefIter::from_bytes(obj.0.data, obj.0.object_hash) + .tree_id() + .expect("every commit has a tree"); + stats.expanded_objects += 1; + obj = db.find(&id, buf1)?; + continue; + } + Blob => break, + Tag => { + id = TagRefIter::from_bytes(obj.0.data, obj.0.object_hash) + .target_id() + .expect("every tag has a target"); + stats.expanded_objects += 1; + obj = db.find(&id, buf1)?; + continue; + } + } + } + } + AsIs => push_obj_count_unique(&mut out, seen_objs, &id, location, objects, stats, false), + } + } + outcome.total_objects = out.len(); + Ok((out, outcome)) + } + + #[inline] + fn push_obj_count_unique( + out: &mut Vec, + all_seen: &impl util::InsertImmutable, + id: &oid, + location: Option, + objects: &gix_features::progress::AtomicStep, + statistics: &mut Outcome, + count_expanded: bool, + ) { + let inserted = all_seen.insert(id.to_owned()); + if inserted { + objects.fetch_add(1, Ordering::Relaxed); + statistics.decoded_objects += 1; + if count_expanded { + statistics.expanded_objects += 1; + } + out.push(output::Count::from_data(id, location)); + } + } + + #[inline] + fn id_to_count( + db: &dyn crate::Find, + buf: &mut Vec, + id: &oid, + objects: &gix_features::progress::AtomicStep, + statistics: &mut Outcome, + allow_pack_lookups: bool, + ) -> output::Count { + objects.fetch_add(1, Ordering::Relaxed); + statistics.expanded_objects += 1; + output::Count { + id: id.to_owned(), + entry_pack_location: if allow_pack_lookups { + PackLocation::LookedUp(db.location_by_oid(id, buf)) + } else { + PackLocation::NotLookedUp + }, + } + } + + struct CountingObjects<'a> { + decoded_objects: std::cell::RefCell, + objects: &'a dyn crate::Find, + } + + impl<'a> CountingObjects<'a> { + fn new(objects: &'a dyn crate::Find) -> Self { + Self { + decoded_objects: Default::default(), + objects, + } + } + + fn into_count(self) -> usize { + self.decoded_objects.into_inner() + } + } + + impl gix_object::Find for CountingObjects<'_> { + fn try_find<'a>(&self, id: &oid, buffer: &'a mut Vec) -> Result>, gix_object::find::Error> { + let res = Ok(self.objects.try_find(id, buffer)?.map(|t| t.0)); + *self.decoded_objects.borrow_mut() += 1; + res + } + } + + struct ExpandedCountingObjects<'a> { + decoded_objects: std::cell::RefCell, + expanded_objects: std::cell::RefCell, + out: std::cell::RefCell>, + objects_count: &'a gix_features::progress::AtomicStep, + objects: &'a dyn crate::Find, + } + + impl<'a> ExpandedCountingObjects<'a> { + fn new( + objects: &'a dyn crate::Find, + out: Vec, + objects_count: &'a gix_features::progress::AtomicStep, + ) -> Self { + Self { + decoded_objects: Default::default(), + expanded_objects: Default::default(), + out: RefCell::new(out), + objects_count, + objects, + } + } + + fn dissolve(self, stats: &mut Outcome) -> Vec { + stats.decoded_objects += self.decoded_objects.into_inner(); + stats.expanded_objects += self.expanded_objects.into_inner(); + self.out.into_inner() + } + } + + impl gix_object::Find for ExpandedCountingObjects<'_> { + fn try_find<'a>(&self, id: &oid, buffer: &'a mut Vec) -> Result>, gix_object::find::Error> { + let maybe_obj = self.objects.try_find(id, buffer)?; + *self.decoded_objects.borrow_mut() += 1; + match maybe_obj { + None => Ok(None), + Some((obj, location)) => { + self.objects_count.fetch_add(1, Ordering::Relaxed); + *self.expanded_objects.borrow_mut() += 1; + self.out.borrow_mut().push(output::Count::from_data(id, location)); + Ok(Some(obj)) + } + } + } + } +} diff --git a/knot2/third_party/gix-pack/src/data/output/count/objects/reduce.rs b/knot2/third_party/gix-pack/src/data/output/count/objects/reduce.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/output/count/objects/reduce.rs @@ -0,0 +1,40 @@ +use std::marker::PhantomData; + +use gix_features::parallel; + +use super::Outcome; +use crate::data::output; + +pub struct Statistics { + total: Outcome, + counts: Vec, + _err: PhantomData, +} + +impl Statistics { + pub fn new() -> Self { + Statistics { + total: Default::default(), + counts: Default::default(), + _err: PhantomData, + } + } +} + +impl parallel::Reduce for Statistics { + type Input = Result<(Vec, Outcome), E>; + type FeedProduce = (); + type Output = (Vec, Outcome); + type Error = E; + + fn feed(&mut self, item: Self::Input) -> Result { + let (counts, stats) = item?; + self.total.aggregate(stats); + self.counts.extend(counts); + Ok(()) + } + + fn finalize(self) -> Result { + Ok((self.counts, self.total)) + } +} diff --git a/knot2/third_party/gix-pack/src/data/output/count/objects/tree.rs b/knot2/third_party/gix-pack/src/data/output/count/objects/tree.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/output/count/objects/tree.rs @@ -0,0 +1,127 @@ +pub mod changes { + use gix_diff::tree::{ + Visit, + visit::{Action, Change}, + }; + use gix_hash::ObjectId; + use gix_object::bstr::BStr; + + use crate::data::output::count::objects_impl::util::InsertImmutable; + + pub struct AllNew<'a, H> { + pub objects: Vec, + all_seen: &'a H, + } + + impl<'a, H> AllNew<'a, H> + where + H: InsertImmutable, + { + pub fn new(all_seen: &'a H) -> Self { + AllNew { + objects: Default::default(), + all_seen, + } + } + pub fn clear(&mut self) { + self.objects.clear(); + } + } + + impl Visit for AllNew<'_, H> + where + H: InsertImmutable, + { + fn pop_front_tracked_path_and_set_current(&mut self) {} + + fn push_back_tracked_path_component(&mut self, _component: &BStr) {} + + fn push_path_component(&mut self, _component: &BStr) {} + + fn pop_path_component(&mut self) {} + + fn visit(&mut self, change: Change) -> Action { + match change { + Change::Addition { + oid, + entry_mode, + relation: _, + } + | Change::Modification { oid, entry_mode, .. } => { + if entry_mode.is_commit() { + return std::ops::ControlFlow::Continue(()); + } + let inserted = self.all_seen.insert(oid); + if inserted { + self.objects.push(oid); + } + } + Change::Deletion { .. } => {} + } + std::ops::ControlFlow::Continue(()) + } + } +} + +pub mod traverse { + use gix_hash::ObjectId; + use gix_object::{bstr::BStr, tree::EntryRef}; + use gix_traverse::tree::{Visit, visit::Action}; + + use crate::data::output::count::objects_impl::util::InsertImmutable; + + pub struct AllUnseen<'a, H> { + pub non_trees: Vec, + all_seen: &'a H, + } + + impl<'a, H> AllUnseen<'a, H> + where + H: InsertImmutable, + { + pub fn new(all_seen: &'a H) -> Self { + AllUnseen { + non_trees: Default::default(), + all_seen, + } + } + pub fn clear(&mut self) { + self.non_trees.clear(); + } + } + + impl Visit for AllUnseen<'_, H> + where + H: InsertImmutable, + { + fn pop_back_tracked_path_and_set_current(&mut self) {} + + fn pop_front_tracked_path_and_set_current(&mut self) {} + + fn push_back_tracked_path_component(&mut self, _component: &BStr) {} + + fn push_path_component(&mut self, _component: &BStr) {} + + fn pop_path_component(&mut self) {} + + fn visit_tree(&mut self, entry: &EntryRef<'_>) -> Action { + let inserted = self.all_seen.insert(entry.oid.to_owned()); + if inserted { + std::ops::ControlFlow::Continue(true) + } else { + std::ops::ControlFlow::Continue(false) + } + } + + fn visit_nontree(&mut self, entry: &EntryRef<'_>) -> Action { + if entry.mode.is_commit() { + return std::ops::ControlFlow::Continue(true); + } + let inserted = self.all_seen.insert(entry.oid.to_owned()); + if inserted { + self.non_trees.push(entry.oid.to_owned()); + } + std::ops::ControlFlow::Continue(true) + } + } +} diff --git a/knot2/third_party/gix-pack/src/data/output/count/objects/types.rs b/knot2/third_party/gix-pack/src/data/output/count/objects/types.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/output/count/objects/types.rs @@ -0,0 +1,96 @@ +/// Information gathered during the run of [`iter_from_objects()`][super::objects()]. +#[derive(Default, PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Outcome { + /// The amount of objects provided to start the iteration. + pub input_objects: usize, + /// The amount of objects that have been expanded from the input source. + /// It's desirable to do that as expansion happens on multiple threads, allowing the amount of input objects to be small. + /// `expanded_objects - decoded_objects` is the 'cheap' object we found without decoding the object itself. + pub expanded_objects: usize, + /// The amount of fully decoded objects. These are the most expensive as they are fully decoded + pub decoded_objects: usize, + /// The total amount of encountered objects. Should be `expanded_objects + input_objects`. + pub total_objects: usize, +} + +impl Outcome { + pub(in crate::data::output::count) fn aggregate( + &mut self, + Outcome { + input_objects, + decoded_objects, + expanded_objects, + total_objects, + }: Self, + ) { + self.input_objects += input_objects; + self.decoded_objects += decoded_objects; + self.expanded_objects += expanded_objects; + self.total_objects += total_objects; + } +} + +/// The way input objects are handled +#[derive(Default, PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum ObjectExpansion { + /// Don't do anything with the input objects except for transforming them into pack entries + #[default] + AsIs, + /// If the input object is a Commit then turn it into a pack entry. Additionally obtain its tree, turn it into a pack entry + /// along with all of its contents, that is nested trees, and any other objects reachable from it. + /// Otherwise, the same as [`AsIs`][ObjectExpansion::AsIs]. + /// + /// This mode is useful if all reachable objects should be added, as in cloning a repository. + TreeContents, + /// If the input is a commit, obtain its ancestors and turn them into pack entries. Obtain the ancestor trees along with the commits + /// tree and turn them into pack entries. Finally obtain the added/changed objects when comparing the ancestor trees with the + /// current tree and turn them into entries as well. + /// Otherwise, the same as [`AsIs`][ObjectExpansion::AsIs]. + /// + /// This mode is useful to build a pack containing only new objects compared to a previous state. + TreeAdditionsComparedToAncestor, +} + +/// Configuration options for the pack generation functions provided in [this module][crate::data::output]. +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Options { + /// The amount of threads to use at most when resolving the pack. If `None`, all logical cores are used. + /// If more than one thread is used, the order of returned [counts][crate::data::output::Count] is not deterministic anymore + /// especially when tree traversal is involved. Thus deterministic ordering requires `Some(1)` to be set. + pub thread_limit: Option, + /// The amount of objects per chunk or unit of work to be sent to threads for processing + pub chunk_size: usize, + /// The way input objects are handled + pub input_object_expansion: ObjectExpansion, +} + +impl Default for Options { + fn default() -> Self { + Options { + thread_limit: None, + chunk_size: 10, + input_object_expansion: Default::default(), + } + } +} + +/// The error returned by the pack generation iterator [`bytes::FromEntriesIter`][crate::data::output::bytes::FromEntriesIter]. +#[derive(Debug, thiserror::Error)] +#[allow(missing_docs)] +pub enum Error { + #[error(transparent)] + CommitDecode(gix_object::decode::Error), + #[error(transparent)] + FindExisting(#[from] gix_object::find::existing::Error), + #[error(transparent)] + InputIteration(Box), + #[error(transparent)] + TreeTraverse(gix_traverse::tree::breadthfirst::Error), + #[error(transparent)] + TreeChanges(gix_diff::tree::Error), + #[error("Operation interrupted")] + Interrupted, +} diff --git a/knot2/third_party/gix-pack/src/data/output/count/objects/util.rs b/knot2/third_party/gix-pack/src/data/output/count/objects/util.rs new file mode 100644 --- /dev/null +++ b/knot2/third_party/gix-pack/src/data/output/count/objects/util.rs @@ -0,0 +1,24 @@ +pub trait InsertImmutable { + fn insert(&self, id: gix_hash::ObjectId) -> bool; +} + +mod trait_impls { + use std::cell::RefCell; + + use gix_hash::ObjectId; + use gix_hashtable::HashSet; + + use super::InsertImmutable; + + impl InsertImmutable for gix_hashtable::sync::ObjectIdMap<()> { + fn insert(&self, id: ObjectId) -> bool { + self.insert(id, ()).is_none() + } + } + + impl InsertImmutable for RefCell> { + fn insert(&self, item: ObjectId) -> bool { + self.borrow_mut().insert(item) + } + } +}