diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..56dea2f --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +target/ +local/ diff --git a/LICENSE.Apache-2.0 b/LICENSE.Apache-2.0 new file mode 100644 index 0000000..93f59f2 --- /dev/null +++ b/LICENSE.Apache-2.0 @@ -0,0 +1,190 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +Copyright 2026 microcosm + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/LICENSE.MIT b/LICENSE.MIT new file mode 100644 index 0000000..21246cc --- /dev/null +++ b/LICENSE.MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 microcosm + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/interop/star-lite/algorithm.json b/interop/star-lite/algorithm.json new file mode 100644 index 0000000..e69de29 diff --git a/reference-impl/rust/Cargo.lock b/reference-impl/rust/Cargo.lock new file mode 100644 index 0000000..9126c2a --- /dev/null +++ b/reference-impl/rust/Cargo.lock @@ -0,0 +1,812 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "base-x" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" + +[[package]] +name = "base256emoji" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e9430d9a245a77c92176e649af6e275f20839a48389859d1661e9a128d077c" +dependencies = [ + "const-str", + "match-lookup", +] + +[[package]] +name = "bitflags" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cbor4ii" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544cf8c89359205f4f990d0e6f3828db42df85b5dac95d09157a250eb0749c4" +dependencies = [ + "serde", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cid" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a304f95f84d169a6f31c4d0a30d784643aaa0bbc9c1e449a2c23e963ec4971" +dependencies = [ + "multibase", + "multihash", + "serde", + "serde_bytes", + "unsigned-varint", +] + +[[package]] +name = "const-str" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f421161cb492475f1661ddc9815a745a1c894592070661180fdec3d4872e9c3" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "data-encoding-macro" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3259c913752a86488b501ed8680446a5ed2d5aeac6e596cb23ba3800768ea32c" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccc2776f0c61eca1ca32528f85548abd1a4be8fb53d1b21c013e4f18da1e7090" +dependencies = [ + "data-encoding", + "syn", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", + "wasip3", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.0", + "serde", + "serde_core", +] + +[[package]] +name = "ipld-core" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090f624976d72f0b0bb71b86d58dc16c15e069193067cb3a3a09d655246cbbda" +dependencies = [ + "cid", + "serde", + "serde_bytes", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "match-lookup" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "757aee279b8bdbb9f9e676796fd459e4207a1f986e87886700abf589f5abf771" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "multibase" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8694bb4835f452b0e3bb06dbebb1d6fc5385b6ca1caf2e55fd165c042390ec77" +dependencies = [ + "base-x", + "base256emoji", + "data-encoding", + "data-encoding-macro", +] + +[[package]] +name = "multihash" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "577c63b00ad74d57e8c9aa870b5fccebf2fd64a308a5aee9f1bb88e4aea19447" +dependencies = [ + "serde", + "unsigned-varint", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_ipld_dagcbor" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46182f4f08349a02b45c998ba3215d3f9de826246ba02bb9dddfe9a2a2100778" +dependencies = [ + "cbor4ii", + "ipld-core", + "scopeguard", + "serde", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "star-lite" +version = "0.1.0" +dependencies = [ + "ipld-core", + "rand", + "serde", + "serde_bytes", + "serde_ipld_dagcbor", + "star-repo", + "tempfile", + "thiserror", + "tokio", +] + +[[package]] +name = "star-repo" +version = "0.1.0" +dependencies = [ + "ipld-core", + "serde", + "serde_bytes", + "serde_ipld_dagcbor", + "sha2", + "thiserror", + "unsigned-varint", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio" +version = "1.52.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "110a78583f19d5cdb2c5ccf321d1290344e71313c6c37d43520d386027d18386" +dependencies = [ + "pin-project-lite", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "typenum" +version = "1.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "unsigned-varint" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.3+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +dependencies = [ + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "zerocopy" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/reference-impl/rust/Cargo.toml b/reference-impl/rust/Cargo.toml new file mode 100644 index 0000000..9c262f4 --- /dev/null +++ b/reference-impl/rust/Cargo.toml @@ -0,0 +1,15 @@ +[workspace] +resolver = "2" +members = [ + "star-repo", + "star-lite", +] + +[workspace.dependencies] +ipld-core = { version = "0.4", features = ["serde"] } +serde = { version = "1", features = ["derive"] } +serde_bytes = "0.11" +serde_ipld_dagcbor = "0.6" +sha2 = "0.10" +thiserror = "1" +unsigned-varint = "0.8" diff --git a/reference-impl/rust/star-lite/Cargo.toml b/reference-impl/rust/star-lite/Cargo.toml new file mode 100644 index 0000000..4915d5d --- /dev/null +++ b/reference-impl/rust/star-lite/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "star-lite" +version = "0.1.0" +description = "STAR-lite archive format for atproto repositories" +repository = "https://tangled.org/microcosm.blue/star" +homepage = "https://microcosm.tngl.io/star/" +keywords = ["atproto", "car", "mst"] +categories = ["encoding", "data-structures"] +readme = "readme.md" +edition = "2024" +license = "MIT OR Apache-2.0" + +[dependencies] +ipld-core = { workspace = true } +serde_bytes = { workspace = true } +star-repo = { path = "../star-repo", version = "0.1.0" } +tempfile = "3" +thiserror = { workspace = true } + +[dev-dependencies] +rand = "0.8" +serde = { workspace = true } +serde_ipld_dagcbor = { workspace = true } +tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync"] } diff --git a/reference-impl/rust/star-lite/LICENSE.Apache-2.0 b/reference-impl/rust/star-lite/LICENSE.Apache-2.0 new file mode 100644 index 0000000..93f59f2 --- /dev/null +++ b/reference-impl/rust/star-lite/LICENSE.Apache-2.0 @@ -0,0 +1,190 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +Copyright 2026 microcosm + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/reference-impl/rust/star-lite/LICENSE.MIT b/reference-impl/rust/star-lite/LICENSE.MIT new file mode 100644 index 0000000..21246cc --- /dev/null +++ b/reference-impl/rust/star-lite/LICENSE.MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 microcosm + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/reference-impl/rust/star-lite/examples/async_bridge.rs b/reference-impl/rust/star-lite/examples/async_bridge.rs new file mode 100644 index 0000000..c7eae88 --- /dev/null +++ b/reference-impl/rust/star-lite/examples/async_bridge.rs @@ -0,0 +1,171 @@ +//! Async-from-sync bridge example. +//! +//! Star-lite's `CarBuilder` is sync and may block on internal I/O (the +//! spilling byte-log writes/reads to a tempfile). To use it from an async +//! runtime — for instance, an axum handler that streams a CAR body in +//! response to a request whose entries arrive over a websocket — the +//! standard pattern is: +//! +//! 1. **Two bounded mpsc channels**, one per direction, providing +//! back-pressure both ways. +//! 2. **An async producer** task pushes `(key, record)` pairs into the +//! input channel. +//! 3. **The blocking core** runs inside `tokio::task::spawn_blocking`. It +//! drains the input channel synchronously and writes CAR bytes into a +//! `ChannelWriter` that pushes them to the output channel. +//! 4. **An async consumer** receives chunks from the output channel — +//! typically wrapped in `ReceiverStream` and fed to an axum +//! `Body::from_stream`. +//! +//! This example simulates both ends with synthetic data and prints stats. +//! Drop into a real service by replacing the producer (read from your +//! async source) and the consumer (return a streaming response). +//! +//! cargo run --release --example async_bridge + +use std::io::{self, Write}; + +use ipld_core::cid::Cid; +use serde::Serialize; +use tokio::sync::mpsc; + +use star_lite::ConvertOptions; +use star_lite::convert::CarBuilder; +use star_lite::mst::{BuilderOptions, MstStack}; +use star_lite::verify::VerifyBackend; + +const CHANNEL_CAPACITY: usize = 32; + +#[tokio::main(flavor = "multi_thread", worker_threads = 2)] +async fn main() -> Result<(), Box> { + // Build a synthetic corpus + commit. In a real service these come from + // wherever your data lives — a websocket frame parser, a database + // cursor, etc. + let n: usize = std::env::var("STAR_ASYNC_N") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(10_000); + let entries = build_synthetic(n); + let commit_bytes = build_synthetic_commit(&entries); + + // Channels in both directions. Bounded capacity keeps memory usage + // predictable and provides natural back-pressure. + let (in_tx, mut in_rx) = mpsc::channel::<(String, Vec)>(CHANNEL_CAPACITY); + let (out_tx, mut out_rx) = mpsc::channel::>(CHANNEL_CAPACITY); + + // Async producer: feed entries into the input channel as if they were + // arriving from a stream. + let producer = tokio::spawn(async move { + for entry in entries { + // `.send().await` back-pressures if the blocking core falls + // behind on processing. + if in_tx.send(entry).await.is_err() { + break; // consumer dropped — give up + } + } + // Dropping `in_tx` here closes the channel and signals EOF to the + // blocking side. + }); + + // Blocking core: a `spawn_blocking` task that drains the input + // channel, drives `CarBuilder`, and writes CAR bytes to the output + // channel via `ChannelWriter`. Default `ConvertOptions` keeps disk + // spilling enabled (64 MiB threshold), which is critical for highly + // concurrent use — peak memory per request stays bounded even under + // load. + let blocking_core = tokio::task::spawn_blocking(move || -> Result<(), star_lite::Error> { + // BufWriter aggregates the many small writes our emit loop makes + // (one per CAR frame) into fewer, larger channel sends. + let writer = io::BufWriter::new(ChannelWriter { tx: out_tx }); + let opts = ConvertOptions::default(); + let mut builder = CarBuilder::new(&commit_bytes, writer, opts)?; + while let Some((key, record)) = in_rx.blocking_recv() { + builder.add_entry(&key, &record)?; + } + builder.finish() + }); + + // Async consumer: receive CAR chunks. In a real axum handler this + // would be wrapped in `tokio_stream::wrappers::ReceiverStream` and + // returned as `Body::from_stream(...)`. + let mut total_bytes: usize = 0; + let mut chunks: usize = 0; + while let Some(chunk) = out_rx.recv().await { + total_bytes += chunk.len(); + chunks += 1; + } + + // Surface any errors from either side. + producer.await?; + blocking_core.await??; + + println!("received {chunks} chunks totaling {total_bytes} bytes from {n} entries"); + Ok(()) +} + +/// Sync `io::Write` adapter: every write becomes a `blocking_send` into a +/// tokio mpsc channel. Used inside `spawn_blocking` — the receiver lives +/// on the async side. +struct ChannelWriter { + tx: mpsc::Sender>, +} + +impl Write for ChannelWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + // `blocking_send` parks this thread (which is fine — we're in + // spawn_blocking) until the consumer drains a slot. That's the + // back-pressure: if the network is slow, fjall iteration will + // also slow down, all the way back to `producer`. + self.tx + .blocking_send(buf.to_vec()) + .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "receiver dropped"))?; + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +// ---- corpus helpers ---- + +fn build_synthetic(n: usize) -> Vec<(String, Vec)> { + let mut entries: Vec<(String, Vec)> = (0..n) + .map(|i| { + ( + format!("c.col/rkey-{i:08}"), + format!("record-body-{i}").into_bytes(), + ) + }) + .collect(); + entries.sort_by(|a, b| a.0.cmp(&b.0)); + entries +} + +#[derive(Debug, Serialize)] +struct CommitForSigning<'a> { + did: &'a str, + version: u64, + data: Cid, + rev: &'a str, + prev: Option, + sig: serde_bytes::ByteBuf, +} + +fn build_synthetic_commit(entries: &[(String, Vec)]) -> Vec { + let mut stack = MstStack::new(VerifyBackend, BuilderOptions::default()); + for (k, v) in entries { + stack.insert(k.as_bytes(), v).unwrap(); + } + let (data, _) = stack.finish().unwrap(); + + let commit = CommitForSigning { + did: "did:plc:test", + version: 3, + data, + rev: "3l3lalalalala", + prev: None, + sig: serde_bytes::ByteBuf::from(vec![0u8; 64]), + }; + serde_ipld_dagcbor::to_vec(&commit).unwrap() +} diff --git a/reference-impl/rust/star-lite/examples/encode_entries.rs b/reference-impl/rust/star-lite/examples/encode_entries.rs new file mode 100644 index 0000000..adf116a --- /dev/null +++ b/reference-impl/rust/star-lite/examples/encode_entries.rs @@ -0,0 +1,111 @@ +//! Encode a STAR-lite archive from `(key, record)` pairs. +//! +//! Use this when you already have a commit and the data CID in hand, and +//! just want to stream entries into an archive. The output is whatever +//! [`Write`] you hand in — wrap a file, a `Vec`, an `os_pipe::PipeWriter`, +//! a zstd encoder, etc. +//! +//! Usage: +//! +//! cargo run --example encode_entries -- path/to/out.starlite +//! +//! Constraints: +//! +//! - Keys must arrive in strict lex byte order; duplicates are rejected. +//! - `data_cid` must be CIDv1 / dag-cbor / sha-256 (36 bytes on the wire). +//! - The header carries `commit minus its data field` as the partial commit; +//! [`CommitInfo::to_partial_bytes`] produces that. + +use std::fs::File; +use std::io::{BufWriter, Write}; + +use ipld_core::cid::Cid; +use serde::Serialize; + +use star_lite::commit::parse_commit; +use star_lite::mst::{BuilderOptions, MstStack}; +use star_lite::verify::VerifyBackend; +use star_lite::{CommitInfo, Writer}; + +/// Encode a STAR-lite archive into `output` from an iterator of +/// `(key, record)` pairs. Keys must already be in lex order. +fn encode_starlite( + output: W, + data_cid: Cid, + commit: &CommitInfo, + entries: I, +) -> star_lite::Result<()> +where + W: Write, + I: IntoIterator)>, + K: AsRef, +{ + let partial = commit.to_partial_bytes()?; + let mut w = Writer::new(output); + w.write_header(data_cid, Some(&partial))?; + for (k, v) in entries { + w.write_entry(k.as_ref(), &v)?; + } + w.finish() +} + +fn main() { + let out_path = std::env::args().nth(1).unwrap_or_else(|| { + eprintln!("usage: encode_entries "); + std::process::exit(2); + }); + + // Synthetic entries (sorted by key). + let entries: Vec<(String, Vec)> = (0..1000) + .map(|i| { + ( + format!("app.bsky.feed.post/{i:08}"), + format!("record-{i}").into_bytes(), + ) + }) + .collect(); + + // In a real caller these come from upstream; here we compute them so the + // example is self-contained. + let data_cid = compute_mst_root(&entries); + let commit = synthesize_commit("did:plc:example", "3kfoo", data_cid); + + let file = File::create(&out_path).expect("create output"); + let writer = BufWriter::new(file); + encode_starlite(writer, data_cid, &commit, entries).expect("encode"); + + eprintln!("wrote {out_path}"); +} + +/// Compute the MST root CID without retaining records or MST nodes. +fn compute_mst_root(entries: &[(String, Vec)]) -> Cid { + let mut stack = MstStack::new(VerifyBackend, BuilderOptions::default()); + for (k, v) in entries { + stack.insert(k.as_bytes(), v).expect("insert"); + } + stack.finish().expect("finish").0 +} + +/// Synthesize a v3 commit object with a fake signature. Real callers already +/// have a [`CommitInfo`] from their upstream pipeline. +fn synthesize_commit(did: &str, rev: &str, data: Cid) -> CommitInfo { + #[derive(Serialize)] + struct Commit<'a> { + did: &'a str, + version: u64, + data: Cid, + rev: &'a str, + prev: Option, + sig: serde_bytes::ByteBuf, + } + let bytes = serde_ipld_dagcbor::to_vec(&Commit { + did, + version: 3, + data, + rev, + prev: None, + sig: serde_bytes::ByteBuf::from(vec![0u8; 64]), + }) + .expect("encode commit"); + parse_commit(&bytes).expect("parse commit") +} diff --git a/reference-impl/rust/star-lite/examples/from_car.rs b/reference-impl/rust/star-lite/examples/from_car.rs new file mode 100644 index 0000000..aec16f0 --- /dev/null +++ b/reference-impl/rust/star-lite/examples/from_car.rs @@ -0,0 +1,29 @@ +//! Convert a CAR file (atproto repository) to a STAR-lite archive. +//! +//! Usage: +//! cargo run --example from_car -- path/to/repo.car path/to/out.starlite + +use star_lite::car_to_star_lite; +use std::fs::File; +use std::io::{BufReader, BufWriter}; + +fn main() { + let mut args = std::env::args().skip(1); + let in_path = args.next().unwrap_or_else(|| { + eprintln!("usage: from_car "); + std::process::exit(2); + }); + let out_path = args.next().unwrap_or_else(|| { + eprintln!("usage: from_car "); + std::process::exit(2); + }); + + let input = BufReader::new(File::open(&in_path).expect("open car")); + let output = BufWriter::new(File::create(&out_path).expect("create archive")); + + if let Err(e) = car_to_star_lite(input, output) { + eprintln!("conversion failed: {e}"); + std::process::exit(1); + } + eprintln!("wrote {out_path}"); +} diff --git a/reference-impl/rust/star-lite/examples/inspect.rs b/reference-impl/rust/star-lite/examples/inspect.rs new file mode 100644 index 0000000..ab4baab --- /dev/null +++ b/reference-impl/rust/star-lite/examples/inspect.rs @@ -0,0 +1,43 @@ +//! Dump the structure of a STAR-lite archive. +//! +//! Usage: +//! cargo run --example inspect -- path/to/archive.starlite + +use star_lite::Reader; +use std::fs::File; +use std::io::BufReader; + +fn main() { + let path = std::env::args().nth(1).unwrap_or_else(|| { + eprintln!("usage: inspect "); + std::process::exit(2); + }); + + let file = File::open(&path).expect("open archive"); + let reader = Reader::new(BufReader::new(file)).expect("parse header"); + + println!("header cid: {}", reader.header_cid()); + match reader.partial_commit() { + Some(p) => println!("partial commit: {} bytes", p.len()), + None => println!("partial commit: (absent)"), + } + println!("entries:"); + + let mut count = 0u64; + let mut total_key_bytes = 0u64; + let mut total_record_bytes = 0u64; + + for entry in reader { + let entry = entry.expect("parse entry"); + count += 1; + total_key_bytes += entry.key.len() as u64; + total_record_bytes += entry.record.len() as u64; + println!(" {} ({} bytes)", entry.key, entry.record.len()); + } + + println!(); + println!("summary:"); + println!(" {count} entries"); + println!(" {total_key_bytes} bytes of keys"); + println!(" {total_record_bytes} bytes of records"); +} diff --git a/reference-impl/rust/star-lite/examples/to_car.rs b/reference-impl/rust/star-lite/examples/to_car.rs new file mode 100644 index 0000000..453032b --- /dev/null +++ b/reference-impl/rust/star-lite/examples/to_car.rs @@ -0,0 +1,29 @@ +//! Convert a STAR-lite archive to a CAR file. +//! +//! Usage: +//! cargo run --example to_car -- path/to/archive.starlite path/to/out.car + +use star_lite::{ConvertOptions, star_lite_to_car}; +use std::fs::File; +use std::io::{BufReader, BufWriter}; + +fn main() { + let mut args = std::env::args().skip(1); + let in_path = args.next().unwrap_or_else(|| { + eprintln!("usage: to_car "); + std::process::exit(2); + }); + let out_path = args.next().unwrap_or_else(|| { + eprintln!("usage: to_car "); + std::process::exit(2); + }); + + let input = BufReader::new(File::open(&in_path).expect("open archive")); + let output = BufWriter::new(File::create(&out_path).expect("create car")); + + if let Err(e) = star_lite_to_car(input, output, ConvertOptions::default()) { + eprintln!("conversion failed: {e}"); + std::process::exit(1); + } + eprintln!("wrote {out_path}"); +} diff --git a/reference-impl/rust/star-lite/readme.md b/reference-impl/rust/star-lite/readme.md new file mode 100644 index 0000000..e9b376a --- /dev/null +++ b/reference-impl/rust/star-lite/readme.md @@ -0,0 +1,3 @@ +# star-lite (rust) + +WARNING: NOT FULLY REVIEWED YET diff --git a/reference-impl/rust/star-lite/src/convert/from_car.rs b/reference-impl/rust/star-lite/src/convert/from_car.rs new file mode 100644 index 0000000..2ac30fe --- /dev/null +++ b/reference-impl/rust/star-lite/src/convert/from_car.rs @@ -0,0 +1,104 @@ +//! CAR -> STAR-lite conversion. +//! +//! Loads the CAR fully into memory, walks the MST in key order, and writes +//! each leaf as a STAR-lite entry. Memory cost is ~the size of the repo. +//! Almost all atproto repos fit comfortably; for outliers, callers should +//! drive [`crate::convert::CarBuilder`] from an iterator that does its own +//! streaming/spilling. + +use std::io::{Read, Write}; + +use ipld_core::cid::Cid; + +use star_repo::Blockstore; +use star_repo::commit::parse_commit; +use star_repo::error::MstError; +use star_repo::mst::node::{MstNode, decode_node}; + +use crate::error::{FormatError, Result}; +use crate::writer::Writer; + +/// Read a CARv1 file from `input` and write a STAR-lite archive to `output`. +pub fn car_to_star_lite(input: R, output: W) -> Result<()> { + let blockstore = Blockstore::from_reader(input)?; + let commit_cid = blockstore.root()?; + let commit_bytes = blockstore.get(&commit_cid)?.to_vec(); + let commit = parse_commit(&commit_bytes)?; + + let mut writer = Writer::new(output); + let partial = commit.to_partial_bytes()?; + writer.write_header(commit.data, Some(&partial))?; + + { + let mut emit = |key: &str, record: &[u8]| writer.write_entry(key, record); + walk_mst(&blockstore, &commit.data, &mut emit)?; + } + + writer.finish() +} + +/// Walk an MST starting at `root_cid`, calling `f(key, record_bytes)` for +/// each leaf in key order. Useful for callers that want to drive their own +/// pipeline rather than producing a STAR-lite archive directly. +pub fn walk_mst(blockstore: &Blockstore, root_cid: &Cid, f: &mut F) -> Result<()> +where + F: FnMut(&str, &[u8]) -> Result<()>, +{ + walk_node(blockstore, root_cid, f) +} + +fn walk_node(blockstore: &Blockstore, node_cid: &Cid, f: &mut F) -> Result<()> +where + F: FnMut(&str, &[u8]) -> Result<()>, +{ + let bytes = blockstore.get(node_cid)?; + let node: MstNode = decode_node(bytes)?; + + if let Some(left) = node.left { + walk_node(blockstore, &left, f)?; + } + + let mut prev_key: Option> = None; + for entry in node.entries { + let prefix_len = entry.prefix_len as usize; + let key = match &prev_key { + None => { + if prefix_len != 0 { + return Err(MstError::InvalidEntry(format!( + "first entry in mst node has nonzero prefix length {prefix_len}" + )) + .into()); + } + entry.key_suffix.into_vec() + } + Some(prev) => { + if prefix_len > prev.len() { + return Err(MstError::InvalidEntry(format!( + "prefix length {prefix_len} exceeds previous key length {}", + prev.len() + )) + .into()); + } + let mut k = prev[..prefix_len].to_vec(); + k.extend_from_slice(&entry.key_suffix); + k + } + }; + + // Validate key is utf-8. + let key_str = std::str::from_utf8(&key) + .map_err(FormatError::InvalidUtf8)? + .to_owned(); + + let record_bytes = blockstore.get(&entry.value)?; + f(&key_str, record_bytes)?; + + if let Some(right) = entry.right { + walk_node(blockstore, &right, f)?; + } + + prev_key = Some(key); + } + + Ok(()) +} diff --git a/reference-impl/rust/star-lite/src/convert/mod.rs b/reference-impl/rust/star-lite/src/convert/mod.rs new file mode 100644 index 0000000..9a05103 --- /dev/null +++ b/reference-impl/rust/star-lite/src/convert/mod.rs @@ -0,0 +1,9 @@ +//! Conversion between STAR-lite and CARv1. + +pub mod from_car; +pub mod options; +pub mod to_car; + +pub use from_car::{car_to_star_lite, walk_mst}; +pub use options::ConvertOptions; +pub use to_car::{CarBuilder, star_lite_to_car}; diff --git a/reference-impl/rust/star-lite/src/convert/options.rs b/reference-impl/rust/star-lite/src/convert/options.rs new file mode 100644 index 0000000..b2ee835 --- /dev/null +++ b/reference-impl/rust/star-lite/src/convert/options.rs @@ -0,0 +1,46 @@ +//! Shared options for the conversion APIs. + +use std::path::PathBuf; + +use crate::mst::BuilderOptions; +use crate::storage::StorageOptions; + +/// Tunables for the streaming conversion. `Default` is sensible for most +/// repos; tweak for very-large repos or constrained-memory environments. +#[derive(Debug, Clone, Default)] +pub struct ConvertOptions { + pub mst: BuilderOptions, + pub storage: StorageOptions, +} + +impl ConvertOptions { + pub fn with_spill_threshold(mut self, threshold: u64) -> Self { + self.storage.spill_threshold = threshold; + self + } + + /// Disable disk spilling — equivalent to + /// [`Self::with_spill_threshold`]`(u64::MAX)`. The entire CAR body + /// stays in memory; peak RAM ≈ full CAR size. + /// + /// Intended for benchmarks and tests where you want to isolate + /// in-memory behavior. **Not recommended for production async use:** + /// in highly-concurrent environments (many simultaneous repo + /// conversions), the bound-on-peak-memory that disk spilling provides + /// is the whole point. Prefer the default threshold + a + /// `spawn_blocking` bridge (see the `async_bridge` example). + pub fn no_spill(mut self) -> Self { + self.storage.spill_threshold = u64::MAX; + self + } + + pub fn with_tempdir(mut self, dir: PathBuf) -> Self { + self.storage.tempdir = Some(dir); + self + } + + pub fn with_max_entries_per_node(mut self, n: usize) -> Self { + self.mst.max_entries_per_node = n; + self + } +} diff --git a/reference-impl/rust/star-lite/src/convert/to_car.rs b/reference-impl/rust/star-lite/src/convert/to_car.rs new file mode 100644 index 0000000..a02b51a --- /dev/null +++ b/reference-impl/rust/star-lite/src/convert/to_car.rs @@ -0,0 +1,148 @@ +//! Streaming STAR-lite -> CAR conversion. +//! +//! [`CarBuilder`] takes the commit bytes plus an arbitrary stream of +//! `(key, record)` pairs and produces a CARv1 file. It's deliberately not +//! tied to the STAR-lite reader: callers can drive it from any source. The +//! convenience function [`star_lite_to_car`] is a thin wrapper that pulls +//! entries from a [`crate::Reader`] and feeds them in. +//! +//! ## Output ordering +//! +//! Per [AT-Repo §2.8.3][spec], blocks are emitted in preorder traversal: +//! +//! 1. CAR header (roots = [commit_cid]) +//! 2. Commit block +//! 3. The MST in preorder. For each node: the node frame, then its +//! leftmost subtree (recursively), then for each leaf entry the record +//! block followed by the leaf's right subtree (recursively). +//! +//! The builder produces this ordering implicitly: every `FrozenSubtree`'s +//! `emit_plan` is already in CAR-preorder for that subtree, so emit is a +//! flat walk over the root subtree's frame offsets. +//! +//! [spec]: https://www.ietf.org/archive/id/draft-holmgren-at-repository-01.txt + +use std::io::{Cursor, Read, Write}; + +use ipld_core::cid::Cid; + +use star_repo::commit::{CommitInfo, parse_commit, parse_partial_commit}; +use star_repo::error::CommitError; +use star_repo::frame::{build_frame, write_header}; +use star_repo::{compute_cid_dag_cbor, varint}; + +use crate::convert::options::ConvertOptions; +use crate::error::{Result, VerifyError}; +use crate::mst::{CarBackend, MstStack}; +use crate::reader::{Entry, Reader}; +use crate::storage::Storage; + +pub struct CarBuilder { + output: W, + stack: MstStack, + commit: CommitInfo, + commit_cid: Cid, +} + +impl CarBuilder { + /// Construct a new CAR builder. The commit is parsed and its CID is + /// computed up front so we can validate against `commit.data` at + /// finalization time. + pub fn new(commit_bytes: &[u8], output: W, options: ConvertOptions) -> Result { + let commit = parse_commit(commit_bytes)?; + let commit_cid = compute_cid_dag_cbor(&commit.raw); + + let backend = CarBackend::new(Storage::new(options.storage)); + let stack = MstStack::new(backend, options.mst); + + Ok(Self { + output, + stack, + commit, + commit_cid, + }) + } + + /// CID of the parsed commit (root of the eventual CAR file). + pub fn commit_cid(&self) -> Cid { + self.commit_cid + } + + pub fn commit(&self) -> &CommitInfo { + &self.commit + } + + /// Append one `(key, record)` entry. The record's bytes are framed and + /// appended to the backend's byte log immediately; an MST leaf for it is + /// queued in the stack. + pub fn add_entry(&mut self, key: &str, record: &[u8]) -> Result<()> { + self.stack.insert(key.as_bytes(), record)?; + Ok(()) + } + + /// Finalize the CAR file: encode the MST root, verify it matches the + /// commit's `data` CID, then stream out the CAR. + pub fn finish(mut self) -> Result<()> { + let (root, backend) = self.stack.finish()?; + + if root.root_cid != self.commit.data { + return Err( + VerifyError::RootMismatch(Box::new(crate::error::RootMismatch { + declared: self.commit.data, + computed: root.root_cid, + })) + .into(), + ); + } + + // 1. CAR header. + write_header(&mut self.output, &[self.commit_cid])?; + + // 2. Commit block. The commit is small (typically <300 bytes); we + // just frame it directly rather than spilling to storage. + let commit_frame = build_frame(&self.commit_cid, &self.commit.raw)?; + self.output.write_all(&commit_frame)?; + + // 3. The MST in preorder. Each entry in the emit plan is the + // storage offset of one CARv1 frame; the frame's extent is + // given by its leading varint payload-length prefix. + let mut storage = backend.into_storage(); + let mut varint_buf = [0u8; varint::MAX_BYTES]; + for &offset in &root.emit_plan { + let n = storage.read_at(offset, &mut varint_buf)?; + let mut cursor = Cursor::new(&varint_buf[..n]); + let payload_len = varint::read_required(&mut cursor)?; + let frame_len = cursor.position() + payload_len; + storage.copy_range_to(offset, frame_len, &mut self.output)?; + } + + self.output.flush()?; + Ok(()) + } +} + +/// Read a STAR-lite archive from `input` and write a CAR to `output`. +/// +/// The archive must carry a partial commit in its header — CAR files require +/// a commit at the root. Headerless archives error with +/// [`CommitError::MissingPartialCommit`]. +pub fn star_lite_to_car( + input: R, + output: W, + options: ConvertOptions, +) -> Result<()> { + let mut reader = Reader::new(input)?; + let header_cid = reader.header_cid(); + let partial = reader + .partial_commit() + .ok_or(CommitError::MissingPartialCommit)? + .to_vec(); + let commit = parse_partial_commit(&partial, header_cid)?; + + let mut builder = CarBuilder::new(&commit.raw, output, options)?; + while let Some(entry) = reader.next_entry()? { + let Entry { key, record } = entry; + builder.add_entry(&key, &record)?; + } + builder.finish() +} diff --git a/reference-impl/rust/star-lite/src/error.rs b/reference-impl/rust/star-lite/src/error.rs new file mode 100644 index 0000000..637258e --- /dev/null +++ b/reference-impl/rust/star-lite/src/error.rs @@ -0,0 +1,111 @@ +//! Error types for STAR-lite. +//! +//! The top-level [`Error`] enum groups failures by phase. Each phase has its +//! own detailed enum so callers can match precisely on what went wrong without +//! parsing strings. + +use std::io; + +use ipld_core::cid::Cid; +use thiserror::Error; + +pub type Result = std::result::Result; + +/// Top-level error for all crate operations. +#[derive(Debug, Error)] +pub enum Error { + #[error("io: {0}")] + Io(#[from] io::Error), + + #[error("star-lite framing: {0}")] + Format(#[from] FormatError), + + #[error("repo: {0}")] + Repo(#[from] star_repo::Error), + + #[error("storage: {0}")] + Storage(#[from] StorageError), + + #[error("verify: {0}")] + Verify(#[from] VerifyError), +} + +// `?` shortcuts so call sites can return star-repo error subtypes directly +// instead of double-wrapping through `star_repo::Error`. + +impl From for Error { + fn from(e: star_repo::error::MstError) -> Self { + Error::Repo(e.into()) + } +} + +impl From for Error { + fn from(e: star_repo::error::CommitError) -> Self { + Error::Repo(e.into()) + } +} + +/// Errors specific to the on-disk STAR-lite framing. +#[derive(Debug, Error)] +pub enum FormatError { + #[error("not a STAR-lite file: bad or missing magic bytes")] + InvalidMagic, + + #[error( + "invalid header CID: must start with 0x01711220 (CIDv1, dag-cbor, sha-256) and be 36 bytes" + )] + InvalidHeaderCid, + + #[error("unexpected end of file")] + UnexpectedEof, + + #[error("invalid utf-8 in key")] + InvalidUtf8(#[from] std::str::Utf8Error), + + #[error("keys out of order: {prev:?} must sort strictly before {curr:?}")] + KeyOrder { prev: String, curr: String }, + + #[error("declared size {actual} exceeds limit {limit} for field {field}")] + SizeLimit { + field: &'static str, + actual: u64, + limit: u64, + }, + + #[error("writer used incorrectly: {0}")] + BadWriterState(&'static str), + + #[error("empty key is not allowed")] + EmptyKey, +} + +/// Errors from the spilling byte storage. +#[derive(Debug, Error)] +pub enum StorageError { + #[error("storage io: {0}")] + Io(#[from] io::Error), + + #[error("range [{offset}, {offset}+{length}) exceeds storage size {total}")] + OutOfRange { + offset: u64, + length: u64, + total: u64, + }, +} + +/// Errors from archive verification. +#[derive(Debug, Error)] +pub enum VerifyError { + #[error("{0}")] + RootMismatch(Box), +} + +/// Payload for [`VerifyError::RootMismatch`]. Boxed inside `VerifyError` so +/// the `Result` discriminant doesn't carry two CIDs of dead weight on the +/// success path. +#[derive(Debug, Error)] +#[error("computed mst root {computed} does not match commit.data {declared}")] +pub struct RootMismatch { + pub declared: Cid, + pub computed: Cid, +} diff --git a/reference-impl/rust/star-lite/src/lib.rs b/reference-impl/rust/star-lite/src/lib.rs new file mode 100644 index 0000000..4be80e0 --- /dev/null +++ b/reference-impl/rust/star-lite/src/lib.rs @@ -0,0 +1,38 @@ +//! Reference implementation of the STAR-lite archive format for atproto +//! repositories. +//! +//! See `README.md` (also the format spec) and the IETF draft +//! [draft-holmgren-at-repository][spec] for full background. +//! +//! Top-level capabilities: +//! +//! - [`Reader`] / [`Writer`] — streaming STAR-lite parser/serializer. +//! - [`verify_archive`] — full-archive verification: reconstructs the MST and +//! matches its root against the commit's `data` CID. No disk usage. +//! - [`star_lite_to_car`] / [`car_to_star_lite`] — bidirectional conversion. +//! - [`CarBuilder`] — lower-level builder for callers who already have an +//! iterator of `(key, record)` pairs (e.g. driving from a different MST +//! walker). +//! +//! Atproto repo primitives (signed-commit parsing, MST construction, CARv1 +//! framing) live in the companion [`star_repo`] crate; this crate +//! re-exports the most commonly used items below. +//! +//! [spec]: https://www.ietf.org/archive/id/draft-holmgren-at-repository-01.txt + +pub mod convert; +pub mod error; +pub mod mst; +pub mod reader; +pub mod storage; +pub mod verify; +pub mod writer; + +pub use convert::{CarBuilder, ConvertOptions, car_to_star_lite, star_lite_to_car}; +pub use error::{Error, Result}; +pub use ipld_core::cid::Cid; +pub use mst::VerifyBackend; +pub use reader::{Entry, Limits, Reader}; +pub use star_repo::{CommitInfo, commit}; +pub use verify::verify_archive; +pub use writer::Writer; diff --git a/reference-impl/rust/star-lite/src/mst/backend.rs b/reference-impl/rust/star-lite/src/mst/backend.rs new file mode 100644 index 0000000..a2e1b56 --- /dev/null +++ b/reference-impl/rust/star-lite/src/mst/backend.rs @@ -0,0 +1,36 @@ +//! Generic MST backends. + +use ipld_core::cid::Cid; + +use crate::error::Result; +use crate::mst::stack::{LevelEntry, MstBackend}; + +/// Hash-and-forget backend: each slot is just a CID. No bytes are retained +/// past their CID computation. Useful when you only need the MST root. +pub struct VerifyBackend; + +impl MstBackend for VerifyBackend { + /// Verify doesn't track anything per leaf — the leaf's CID lives in + /// [`LevelEntry::leaf_cid`], and the bytes have already been hashed and + /// dropped by the stack's call to [`MstBackend::ingest_leaf`]. + type LeafRef = (); + type Slot = Cid; + + fn ingest_leaf(&mut self, _record_cid: Cid, _record_bytes: &[u8]) -> Result { + Ok(()) + } + + fn build_slot( + &mut self, + node_cid: Cid, + _node_bytes: &[u8], + _left: Option, + _entries: Vec>, + ) -> Result { + Ok(node_cid) + } + + fn slot_cid(slot: &Self::Slot) -> Cid { + *slot + } +} diff --git a/reference-impl/rust/star-lite/src/mst/car_backend.rs b/reference-impl/rust/star-lite/src/mst/car_backend.rs new file mode 100644 index 0000000..ef75fae --- /dev/null +++ b/reference-impl/rust/star-lite/src/mst/car_backend.rs @@ -0,0 +1,142 @@ +//! CAR-shaped MST backend: frames blocks into a byte log as they're produced +//! by the stack, and tracks each subtree as a CAR-preorder list of *frame +//! offsets* into [`Storage`]. +//! +//! ## Self-delimiting frames, offset-only emit plans +//! +//! Every CARv1 block frame begins with a varint payload-length prefix, so +//! a frame is fully self-describing given its starting offset: read the +//! varint, add its byte count to the declared payload length, that's the +//! frame's total extent. We exploit that here — each subtree's +//! `emit_plan` is just a `Vec` of frame-start offsets in CAR +//! preorder. At emit time, we peek the varint at each offset to discover +//! the frame's length, then copy that many bytes to the output. +//! +//! ## Memory shape +//! +//! Each emit plan entry is 8 bytes (one `u64` offset). For an N-record +//! atproto repo there are roughly 2N frame offsets at the root (one per +//! record + one per MST node), so ~16N bytes of bookkeeping — independent +//! of record sizes. Per-byte overhead at scale is dominated by the byte +//! log itself, which spills to disk past the configured threshold. +//! +//! ## What changed from earlier versions +//! +//! Earlier iterations stored `(offset, length)` pairs (a `BlockRun`) and +//! coalesced storage-contiguous adjacent runs at build time. We dropped +//! both: the length is redundant (frames are varint-prefixed), and +//! coalescing required the length. The trade-off is that emit now does a +//! varint parse per frame, but varint parsing is cheap (≤ 9 bytes, a +//! handful of shifts). + +use ipld_core::cid::Cid; + +use star_repo::frame::write_frame; + +use crate::error::Result; +use crate::mst::stack::{LevelEntry, MstBackend}; +use crate::storage::Storage; + +/// Reference to a CARv1 block frame in [`Storage`]: the absolute offset +/// where the frame begins. The frame's extent is implicit in its leading +/// varint payload-length prefix, so this 8-byte offset is sufficient to +/// locate and extract the framed bytes. +pub type CarFrameRef = u64; + +/// A finalized subtree: its root CID and the CAR-preorder emit plan whose +/// frame refs, walked in order, point at this subtree's bytes in a CARv1 +/// body. +#[derive(Debug)] +pub struct FrozenSubtree { + pub root_cid: Cid, + pub emit_plan: Vec, +} + +/// Stack backend that frames every block (records and MST nodes) into a +/// shared byte log as they arrive, and tracks subtrees as flat lists of +/// frame offsets in CAR preorder. +pub struct CarBackend { + storage: Storage, +} + +impl CarBackend { + pub fn new(storage: Storage) -> Self { + Self { storage } + } + + pub fn storage(&self) -> &Storage { + &self.storage + } + + pub fn storage_mut(&mut self) -> &mut Storage { + &mut self.storage + } + + pub fn into_storage(self) -> Storage { + self.storage + } + + /// Frame `bytes` (under `cid`) as a CARv1 block, write to storage, + /// and return a [`CarFrameRef`] pointing at the frame's start. + fn append_frame(&mut self, cid: &Cid, bytes: &[u8]) -> Result { + let offset = self.storage.len(); + write_frame(&mut self.storage, cid, bytes)?; + Ok(offset) + } +} + +impl MstBackend for CarBackend { + /// Per-leaf bookkeeping is a [`CarFrameRef`] — the offset of the + /// record's CARv1 frame in storage. The CID lives in + /// [`LevelEntry::leaf_cid`]. + type LeafRef = CarFrameRef; + type Slot = FrozenSubtree; + + fn ingest_leaf(&mut self, record_cid: Cid, record_bytes: &[u8]) -> Result { + self.append_frame(&record_cid, record_bytes) + } + + fn build_slot( + &mut self, + node_cid: Cid, + node_bytes: &[u8], + left: Option, + entries: Vec>, + ) -> Result { + let node_ref = self.append_frame(&node_cid, node_bytes)?; + + // Assemble the emit plan in CAR preorder: this node's frame ref, + // then leftmost subtree's plan, then for each entry its record + // frame ref followed by its right subtree's plan. + let estimated = 1 + + left.as_ref().map_or(0, |s| s.emit_plan.len()) + + entries + .iter() + .map(|e| 1 + e.right.as_ref().map_or(0, |s| s.emit_plan.len())) + .sum::(); + let mut emit_plan: Vec = Vec::with_capacity(estimated); + + emit_plan.push(node_ref); + if let Some(left) = left { + emit_plan.extend(left.emit_plan); + } + for LevelEntry { + leaf_ref, right, .. + } in entries + { + emit_plan.push(leaf_ref); + if let Some(right) = right { + emit_plan.extend(right.emit_plan); + } + } + + Ok(FrozenSubtree { + root_cid: node_cid, + emit_plan, + }) + } + + fn slot_cid(slot: &Self::Slot) -> Cid { + slot.root_cid + } +} diff --git a/reference-impl/rust/star-lite/src/mst/mod.rs b/reference-impl/rust/star-lite/src/mst/mod.rs new file mode 100644 index 0000000..2ad2c77 --- /dev/null +++ b/reference-impl/rust/star-lite/src/mst/mod.rs @@ -0,0 +1,16 @@ +//! MST construction. +//! +//! The atproto MST schema and key-layer hash live in [`star_repo::mst`] and +//! are re-exported here. The construction engine ([`MstStack`] + +//! [`MstBackend`]) and the two concrete backends ([`CarBackend`], +//! [`VerifyBackend`]) are STAR-lite-shaped and live in this crate. + +pub mod backend; +pub mod car_backend; +pub mod stack; + +pub use backend::VerifyBackend; +pub use car_backend::{CarBackend, CarFrameRef, FrozenSubtree}; +pub use stack::{BuilderOptions, DEFAULT_MAX_ENTRIES_PER_NODE, LevelEntry, MstBackend, MstStack}; + +pub use star_repo::mst::{MstEntry, MstNode, decode_node, encode_node, layer_of}; diff --git a/reference-impl/rust/star-lite/src/mst/stack.rs b/reference-impl/rust/star-lite/src/mst/stack.rs new file mode 100644 index 0000000..771f428 --- /dev/null +++ b/reference-impl/rust/star-lite/src/mst/stack.rs @@ -0,0 +1,336 @@ +//! Streaming MST construction stack — the algorithm-of-record for +//! STAR-lite-shaped MST construction. +//! +//! Specialized for sorted-key input: `(key, record_bytes)` pairs arrive in +//! strict lex order, and the stack produces a canonical atproto MST. No +//! rebalancing, no random updates, no general-purpose semantics. +//! +//! ## Algorithm +//! +//! The stack maintains one partial node per MST layer. When a new key K at +//! layer D arrives: +//! +//! 1. For each layer L in `0..D`, finalize whatever's accumulated: encode +//! its slots as a single MST node (always — no trivial-pass-through; per +//! AT-Repo §2.5.4 a layer with just a subtree slot becomes the +//! intermediate empty-entry node `{l: cid, e: []}`). Push the resulting +//! subtree onto layer L+1. +//! 2. Push a leaf for K onto layer D. +//! +//! At [`MstStack::finish`], we fold the remaining layers bottom-up the same +//! way. Empty layers with a carry from below produce the bridge nodes +//! described above. Empty layers with no carry produce nothing — that's how +//! "top pruning" (the spec's term for not adding empty wrappers above the +//! highest-layer key) and "leaf-position pruning" both fall out naturally. +//! +//! For the empty-repo case (no inserts) we emit the single empty node +//! `{l: null, e: []}` per §2.5.4. +//! +//! ## What the stack owns vs. what backends own +//! +//! The stack owns *everything* that affects the wire format: ordering checks, +//! layer promotion, the entry cap, MST node encoding, prefix elision. There +//! is exactly one source of truth for these — verifiers and converters cannot +//! desync, because they all run the same code. +//! +//! Backends own *only* what to do with bytes the stack produces: persist +//! them, hash and discard them, etc. See [`MstBackend`]. + +use ipld_core::cid::Cid; + +use star_repo::compute_cid_dag_cbor; +use star_repo::error::MstError; +use star_repo::mst::layer_of; +use star_repo::mst::node::{MstEntry, MstNode, encode_node, shared_prefix_len}; + +use crate::error::Result; + +/// Default cap on entries in any single MST node. Atproto MSTs naturally have +/// a few entries per node (fanout 4); 200 is a generous safety margin +/// matching the per-commit operation cap from §4.2.1. +pub const DEFAULT_MAX_ENTRIES_PER_NODE: usize = 200; + +#[derive(Debug, Clone)] +pub struct BuilderOptions { + pub max_entries_per_node: usize, +} + +impl Default for BuilderOptions { + fn default() -> Self { + Self { + max_entries_per_node: DEFAULT_MAX_ENTRIES_PER_NODE, + } + } +} + +/// Connects the [`MstStack`] state machine to a per-format storage strategy. +/// +/// Implementations choose what to do with the bytes the stack produces — some +/// (verify) hash and discard, some (CAR conversion) persist into a byte log. +/// Backends never see ordering, layer-promotion, encoding, or cap-enforcement +/// logic; those live in the stack, so two backends running on the same input +/// cannot disagree about the wire format. +pub trait MstBackend { + /// Per-leaf data the backend tracks. `()` if it doesn't need any. + type LeafRef; + /// Per-subtree handle the backend produces when a layer finalizes. The + /// CID is required (via [`Self::slot_cid`]) so the stack can name this + /// slot in the parent node above it. + type Slot; + + /// Take ownership of an inserted record's bytes. The stack has already + /// computed `record_cid`; the backend may persist the bytes to a byte + /// log, hash them out-of-band, or simply ignore them. + fn ingest_leaf(&mut self, record_cid: Cid, record_bytes: &[u8]) -> Result; + + /// Take ownership of a finalized layer: a fully-encoded MST node (CID + + /// bytes), plus its children. Produce a single slot representing the + /// resulting subtree. + /// + /// `node_bytes` is the canonical DAG-CBOR encoding the stack just + /// produced, and `node_cid` is its CID. Backends that persist bytes + /// should use these directly rather than re-encoding. + fn build_slot( + &mut self, + node_cid: Cid, + node_bytes: &[u8], + left: Option, + entries: Vec>, + ) -> Result; + + /// Read the root CID out of a slot. Called by the stack when assembling + /// parent nodes (the parent needs each child's CID to encode its `l` and + /// `t` fields). + fn slot_cid(slot: &Self::Slot) -> Cid; +} + +/// One leaf entry, plus the optional right subtree that follows it. Mirrors +/// the encoded MST node 1:1. +#[derive(Debug)] +pub struct LevelEntry { + pub key: Vec, + pub leaf_cid: Cid, + pub leaf_ref: L, + pub right: Option, +} + +/// One layer of the stack: an optional left subtree, followed by leaves +/// (each with an optional right subtree). +struct Level { + left: Option, + entries: Vec>, +} + +impl Default for Level { + fn default() -> Self { + Self { + left: None, + entries: Vec::new(), + } + } +} + +impl Level { + fn is_empty(&self) -> bool { + self.left.is_none() && self.entries.is_empty() + } + + /// Push a freshly-promoted subtree at the right end of this layer. + /// Subtrees attach as `left` if no leaves have been seen yet, otherwise + /// as `right` of the most recent leaf. + fn push_subtree(&mut self, s: S) -> Result<()> { + if self.entries.is_empty() { + if self.left.is_some() { + return Err( + MstError::Invariant("two consecutive subtree slots before any leaf").into(), + ); + } + self.left = Some(s); + } else { + let tail = self + .entries + .last_mut() + .expect("entries non-empty by branch check"); + if tail.right.is_some() { + return Err( + MstError::Invariant("two consecutive subtree slots after a leaf").into(), + ); + } + tail.right = Some(s); + } + Ok(()) + } +} + +pub struct MstStack { + options: BuilderOptions, + backend: B, + layers: Vec>, + last_key: Option>, +} + +impl MstStack { + pub fn new(backend: B, options: BuilderOptions) -> Self { + Self { + options, + backend, + layers: Vec::new(), + last_key: None, + } + } + + /// Insert a `(key, record)` pair. Keys must arrive in strict lex order. + pub fn insert(&mut self, key: &[u8], record: &[u8]) -> Result<()> { + if key.is_empty() { + return Err(MstError::EmptyKey.into()); + } + if let Some(prev) = &self.last_key + && key <= prev.as_slice() + { + return Err(MstError::KeyOrder { + prev: String::from_utf8_lossy(prev).into_owned(), + curr: String::from_utf8_lossy(key).into_owned(), + } + .into()); + } + + let layer = layer_of(key) as usize; + while self.layers.len() <= layer { + self.layers.push(Level::default()); + } + + // Promote: every layer below this key's layer is now frozen. + for level_idx in 0..layer { + let level = std::mem::take(&mut self.layers[level_idx]); + if level.is_empty() { + continue; + } + let promoted = encode_level(&mut self.backend, level, &self.options)?; + self.layers[level_idx + 1].push_subtree(promoted)?; + } + + // Eager cap check: enforce the same limit at accumulation time as + // encode_level enforces at finalization. Without this, an adversary + // submitting a repo with a long run of low-layer keys (cheap to + // grind: ~75% of random keys are layer 0) could grow this layer's + // entries vector unboundedly before the encode-time check fires, + // OOMing the process. Living in the stack — not in each backend — + // means verifiers and converters fail at *exactly* the same input + // boundary, so an attacker can't ship a repo that one accepts and + // another rejects. + if self.layers[layer].entries.len() >= self.options.max_entries_per_node { + return Err(MstError::NodeTooLarge { + count: self.layers[layer].entries.len() + 1, + limit: self.options.max_entries_per_node, + } + .into()); + } + + let record_cid = compute_cid_dag_cbor(record); + let leaf_ref = self.backend.ingest_leaf(record_cid, record)?; + self.layers[layer].entries.push(LevelEntry { + key: key.to_vec(), + leaf_cid: record_cid, + leaf_ref, + right: None, + }); + self.last_key = Some(key.to_vec()); + Ok(()) + } + + /// Finalize the tree. Returns `(root_slot, backend)` so callers can read + /// the root CID and continue using any resources the backend holds (e.g. + /// the populated byte log a CAR emitter needs to copy from). + /// + /// If no inserts happened, returns the canonical empty MST node. + pub fn finish(mut self) -> Result<(B::Slot, B)> { + let mut carry: Option = None; + let depth = self.layers.len(); + for level_idx in 0..depth { + let mut level = std::mem::take(&mut self.layers[level_idx]); + if let Some(c) = carry.take() { + // The carry attaches at the right end. push_subtree handles + // both empty-layer (becomes left) and non-empty-layer + // (attaches to last entry's right) cases, and rejects the + // invariant violations. + level.push_subtree(c)?; + } + if level.is_empty() { + continue; + } + // Always encode — never trivial-collapse a single-subtree layer. + // Per §2.5.4 a layer with just a subtree carry produces the + // intermediate `{l: cid, e: []}` bridge node, *unless* it ends up + // being the root. By construction this can't happen here: the + // topmost stack layer always has at least one leaf (it's the + // layer of the highest-layer key, which was inserted as a leaf + // there). So if `level_idx` is below the top, encoding a bridge + // is correct; if `level_idx` is the top, the encoded node has + // entries. + carry = Some(encode_level(&mut self.backend, level, &self.options)?); + } + + let root = match carry { + Some(r) => r, + None => encode_empty_root(&mut self.backend)?, + }; + Ok((root, self.backend)) + } +} + +/// Encode a layer as a single MST node and hand the result to the backend. +fn encode_level( + backend: &mut B, + level: Level, + options: &BuilderOptions, +) -> Result { + debug_assert!(!level.is_empty(), "encode_level called with empty level"); + + // Backstop. The eager check in `MstStack::insert` should make this + // unreachable in practice, but `encode_level` may be called with + // synthesized layers during `finish` (when a carry from below attaches + // to a layer that was empty), so we keep this as a safety net. + if level.entries.len() > options.max_entries_per_node { + return Err(MstError::NodeTooLarge { + count: level.entries.len(), + limit: options.max_entries_per_node, + } + .into()); + } + + let Level { left, entries } = level; + + let left_cid = left.as_ref().map(B::slot_cid); + let mut mst_entries: Vec = Vec::with_capacity(entries.len()); + let mut prev_key: Option<&[u8]> = None; + for entry in &entries { + let prefix_len = match prev_key { + None => 0, + Some(p) => shared_prefix_len(p, &entry.key), + }; + let suffix = entry.key[prefix_len..].to_vec(); + mst_entries.push(MstEntry { + prefix_len: prefix_len as u32, + key_suffix: serde_bytes::ByteBuf::from(suffix), + value: entry.leaf_cid, + right: entry.right.as_ref().map(B::slot_cid), + }); + prev_key = Some(&entry.key); + } + + let node = MstNode { + entries: mst_entries, + left: left_cid, + }; + let (cid, bytes) = encode_node(&node)?; + backend.build_slot(cid, &bytes, left, entries) +} + +fn encode_empty_root(backend: &mut B) -> Result { + let node = MstNode { + entries: Vec::new(), + left: None, + }; + let (cid, bytes) = encode_node(&node)?; + backend.build_slot(cid, &bytes, None, Vec::new()) +} diff --git a/reference-impl/rust/star-lite/src/reader.rs b/reference-impl/rust/star-lite/src/reader.rs new file mode 100644 index 0000000..301763c --- /dev/null +++ b/reference-impl/rust/star-lite/src/reader.rs @@ -0,0 +1,235 @@ +//! Streaming reader for STAR-lite archives. +//! +//! The reader holds the parsed header (CID + optional partial commit bytes) in +//! memory after construction. Each entry is read lazily, one at a time, and +//! the per-entry memory cost is bounded by [`Limits`]. + +use std::io::Read; + +use ipld_core::cid::Cid; + +use star_repo::varint; + +use crate::error::{Error, FormatError, Result}; + +/// `*l\0`: ASCII "star", "**l**ite", version 0. +pub const MAGIC: [u8; 3] = [0x2A, 0x6C, 0x00]; + +/// CID prefix in atproto format: CIDv1, dag-cbor codec, sha-256 multihash, 32- +/// byte digest length. The header CID is always exactly this prefix followed by +/// 32 digest bytes. +pub const HEADER_CID_PREFIX: [u8; 4] = [0x01, 0x71, 0x12, 0x20]; + +/// Total size of the header CID on the wire: 4-byte prefix + 32-byte digest. +pub const HEADER_CID_LEN: usize = 36; + +/// Per the format spec, partial commits larger than 4096 bytes are rejected as +/// implausible. Hard-coded; any change requires a spec revision. +pub const MAX_PARTIAL_COMMIT_LEN: u64 = 4096; + +/// Limits on the size of individual fields, applied during parsing. +/// +/// These exist to prevent malformed or malicious archives from causing +/// unbounded allocations. +#[derive(Debug, Clone, Copy)] +pub struct Limits { + pub max_key_len: u64, + pub max_record_len: u64, +} + +impl Default for Limits { + fn default() -> Self { + Self { + max_key_len: 1024, // 1 KB + max_record_len: 5 * 1024 * 1024, // 5 MB + } + } +} + +/// A single key/record pair from a STAR-lite archive. +#[derive(Debug, Clone)] +pub struct Entry { + pub key: String, + pub record: Vec, +} + +/// Streaming reader over a STAR-lite archive. +/// +/// Construct with [`Reader::new`] (or [`Reader::with_limits`]); this consumes +/// the magic bytes, header CID, and optional partial commit. Then iterate via +/// [`Reader::next_entry`] or by treating the reader as an +/// `Iterator>`. +pub struct Reader { + inner: R, + limits: Limits, + header_cid: Cid, + partial_commit: Option>, + /// Bytes of the previous key, used to enforce strict lex ordering. + prev_key: Option>, + /// Set once we hit EOF or an error. + finished: bool, +} + +impl Reader { + pub fn new(inner: R) -> Result { + Self::with_limits(inner, Limits::default()) + } + + pub fn with_limits(mut inner: R, limits: Limits) -> Result { + // Magic: 3 bytes. A short read here means "not a STAR-lite file" rather + // than "truncated entry", so we surface InvalidMagic for any short read. + let mut magic = [0u8; 3]; + let mut filled = 0; + while filled < magic.len() { + let n = inner.read(&mut magic[filled..])?; + if n == 0 { + return Err(FormatError::InvalidMagic.into()); + } + filled += n; + } + if magic != MAGIC { + return Err(FormatError::InvalidMagic.into()); + } + + // Header CID: fixed 36 bytes, prefix-asserted. + let mut cid_buf = [0u8; HEADER_CID_LEN]; + read_exact_or_eof(&mut inner, &mut cid_buf)?; + if cid_buf[..HEADER_CID_PREFIX.len()] != HEADER_CID_PREFIX { + return Err(FormatError::InvalidHeaderCid.into()); + } + let header_cid = Cid::try_from(&cid_buf[..]).map_err(|_| FormatError::InvalidHeaderCid)?; + + // Partial commit: varint-prefixed bytes; len == 0 means absent. + let commit_len = varint::read_required(&mut inner)?; + if commit_len > MAX_PARTIAL_COMMIT_LEN { + return Err(FormatError::SizeLimit { + field: "partial commit", + actual: commit_len, + limit: MAX_PARTIAL_COMMIT_LEN, + } + .into()); + } + let partial_commit = if commit_len == 0 { + None + } else { + let mut buf = vec![0u8; commit_len as usize]; + read_exact_or_eof(&mut inner, &mut buf)?; + Some(buf) + }; + + Ok(Self { + inner, + limits, + header_cid, + partial_commit, + prev_key: None, + finished: false, + }) + } + + /// CID of the MST root, as declared in the archive header. Equal to the + /// `data` field of the (full) commit object. + pub fn header_cid(&self) -> Cid { + self.header_cid + } + + /// Partial commit bytes, if present. The partial encoding omits the `data` + /// field; reconstruct the full commit via + /// [`crate::commit::parse_partial_commit`]. + pub fn partial_commit(&self) -> Option<&[u8]> { + self.partial_commit.as_deref() + } + + pub fn limits(&self) -> &Limits { + &self.limits + } + + /// Read the next entry, or `Ok(None)` at clean EOF. + pub fn next_entry(&mut self) -> Result> { + if self.finished { + return Ok(None); + } + let key_len = match varint::read(&mut self.inner)? { + Some(n) => n, + None => { + self.finished = true; + return Ok(None); + } + }; + if key_len == 0 { + self.finished = true; + return Err(FormatError::EmptyKey.into()); + } + if key_len > self.limits.max_key_len { + self.finished = true; + return Err(FormatError::SizeLimit { + field: "key", + actual: key_len, + limit: self.limits.max_key_len, + } + .into()); + } + + let mut key_bytes = vec![0u8; key_len as usize]; + read_exact_or_eof(&mut self.inner, &mut key_bytes)?; + + // Strict ordering check (against raw bytes, before utf-8 validation). + if let Some(prev) = &self.prev_key + && key_bytes.as_slice() <= prev.as_slice() + { + self.finished = true; + let prev_str = String::from_utf8_lossy(prev).into_owned(); + let curr_str = String::from_utf8_lossy(&key_bytes).into_owned(); + return Err(FormatError::KeyOrder { + prev: prev_str, + curr: curr_str, + } + .into()); + } + + let key = std::str::from_utf8(&key_bytes) + .map_err(|e| Error::Format(FormatError::InvalidUtf8(e)))? + .to_owned(); + + let record_len = varint::read_required(&mut self.inner)?; + if record_len > self.limits.max_record_len { + self.finished = true; + return Err(FormatError::SizeLimit { + field: "record", + actual: record_len, + limit: self.limits.max_record_len, + } + .into()); + } + let mut record = vec![0u8; record_len as usize]; + read_exact_or_eof(&mut self.inner, &mut record)?; + + self.prev_key = Some(key_bytes); + Ok(Some(Entry { key, record })) + } +} + +impl Iterator for Reader { + type Item = Result; + + fn next(&mut self) -> Option { + match self.next_entry() { + Ok(Some(e)) => Some(Ok(e)), + Ok(None) => None, + Err(e) => Some(Err(e)), + } + } +} + +/// Read exactly `buf.len()` bytes; treat any short read as `UnexpectedEof`. +fn read_exact_or_eof(r: &mut R, buf: &mut [u8]) -> Result<()> { + let mut filled = 0; + while filled < buf.len() { + let n = r.read(&mut buf[filled..])?; + if n == 0 { + return Err(FormatError::UnexpectedEof.into()); + } + filled += n; + } + Ok(()) +} diff --git a/reference-impl/rust/star-lite/src/storage.rs b/reference-impl/rust/star-lite/src/storage.rs new file mode 100644 index 0000000..b99ae2a --- /dev/null +++ b/reference-impl/rust/star-lite/src/storage.rs @@ -0,0 +1,258 @@ +//! Append-only byte storage that spills to disk on memory pressure. +//! +//! The store is a single growing buffer. Callers `append` raw bytes (typically +//! CARv1 block frames) and get back an absolute offset. Once the in-memory +//! buffer exceeds `spill_threshold`, the buffer is flushed to a tempfile and +//! cleared; the next appends accumulate in the buffer again. Multiple flushes +//! are supported. Offsets remain stable across spills. +//! +//! Reads via [`Storage::copy_range_to`] are aware of the split between the +//! file (older bytes) and the buffer (newest bytes) and can span both. + +use std::fs::File; +use std::io::{self, Read, Seek, SeekFrom, Write}; +use std::path::PathBuf; + +use crate::error::{Error, Result, StorageError}; + +#[derive(Debug, Clone)] +pub struct StorageOptions { + /// In-memory buffer threshold before spilling to disk. Defaults to 64 MiB. + pub spill_threshold: u64, + /// Optional directory for the spill tempfile. None = system tempdir. + pub tempdir: Option, +} + +impl Default for StorageOptions { + fn default() -> Self { + Self { + spill_threshold: 64 * 1024 * 1024, + tempdir: None, + } + } +} + +pub struct Storage { + options: StorageOptions, + /// In-memory buffer for the most recently appended bytes. + buffer: Vec, + /// Total bytes already moved out to `spill_file`. + spilled_bytes: u64, + /// Lazily-created spill file. None until first spill. + spill_file: Option, +} + +impl Storage { + pub fn new(options: StorageOptions) -> Self { + Self { + options, + buffer: Vec::new(), + spilled_bytes: 0, + spill_file: None, + } + } + + /// Total logical bytes (in buffer + spilled). + pub fn len(&self) -> u64 { + self.spilled_bytes + self.buffer.len() as u64 + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Append `data` and return the absolute offset where it starts. + pub fn append(&mut self, data: &[u8]) -> Result { + let abs_offset = self.len(); + self.buffer.extend_from_slice(data); + if self.buffer.len() as u64 >= self.options.spill_threshold { + self.spill()?; + } + Ok(abs_offset) + } + + /// Force the in-memory buffer out to disk. + fn spill(&mut self) -> Result<()> { + if self.buffer.is_empty() { + return Ok(()); + } + let file = match &mut self.spill_file { + Some(f) => f, + None => { + let f = match &self.options.tempdir { + Some(dir) => tempfile::tempfile_in(dir).map_err(StorageError::Io)?, + None => tempfile::tempfile().map_err(StorageError::Io)?, + }; + self.spill_file = Some(f); + self.spill_file.as_mut().unwrap() + } + }; + file.seek(SeekFrom::Start(self.spilled_bytes)) + .map_err(StorageError::Io)?; + file.write_all(&self.buffer).map_err(StorageError::Io)?; + self.spilled_bytes += self.buffer.len() as u64; + self.buffer.clear(); + Ok(()) + } + + /// Copy `length` bytes starting at absolute `offset` to `out`. + /// Spans the buffer/file split if needed. + pub fn copy_range_to(&mut self, offset: u64, length: u64, out: &mut W) -> Result<()> { + let total = self.len(); + if offset + .checked_add(length) + .map(|end| end > total) + .unwrap_or(true) + { + return Err(StorageError::OutOfRange { + offset, + length, + total, + } + .into()); + } + if length == 0 { + return Ok(()); + } + + let end = offset + length; + + // Three cases: entirely in file, entirely in buffer, or straddle. + if end <= self.spilled_bytes { + self.copy_from_file(offset, length, out)?; + } else if offset >= self.spilled_bytes { + let buf_start = (offset - self.spilled_bytes) as usize; + let buf_end = buf_start + length as usize; + out.write_all(&self.buffer[buf_start..buf_end])?; + } else { + // straddle + let prefix = self.spilled_bytes - offset; + self.copy_from_file(offset, prefix, out)?; + let suffix = length - prefix; + out.write_all(&self.buffer[..suffix as usize])?; + } + Ok(()) + } + + /// Read up to `buf.len()` bytes starting at absolute `offset` into + /// `buf`. Returns the number of bytes filled, which is + /// `min(buf.len(), self.len() - offset)`. Spans the buffer/file split + /// just like [`Self::copy_range_to`]. + /// + /// Used at emit time by the CAR builder to peek the varint payload + /// length prefix of a frame whose offset (but not length) is known. + pub fn read_at(&mut self, offset: u64, buf: &mut [u8]) -> Result { + let total = self.len(); + if offset > total { + return Err(StorageError::OutOfRange { + offset, + length: buf.len() as u64, + total, + } + .into()); + } + let want = ((buf.len() as u64).min(total - offset)) as usize; + if want == 0 { + return Ok(0); + } + let mut cursor = io::Cursor::new(&mut buf[..want]); + self.copy_range_to(offset, want as u64, &mut cursor)?; + Ok(want) + } + + fn copy_from_file(&mut self, offset: u64, length: u64, out: &mut W) -> Result<()> { + let file = self + .spill_file + .as_mut() + .expect("copy_from_file called with no spill file"); + file.seek(SeekFrom::Start(offset)) + .map_err(StorageError::Io)?; + let mut remaining = length; + let mut buf = vec![0u8; 64 * 1024]; + while remaining > 0 { + let want = remaining.min(buf.len() as u64) as usize; + let n = file.read(&mut buf[..want]).map_err(StorageError::Io)?; + if n == 0 { + return Err(StorageError::OutOfRange { + offset, + length, + total: self.spilled_bytes, + } + .into()); + } + out.write_all(&buf[..n])?; + remaining -= n as u64; + } + Ok(()) + } +} + +/// `io::Write` impl so `Storage` can be passed to anything that writes to a +/// generic sink (e.g. CAR frame writers, varint writers). Errors during +/// append are mapped back into `io::Error` — the underlying I/O error is +/// passed through directly, while everything else gets wrapped as a +/// generic "other" io error carrying the original message. +impl Write for Storage { + fn write(&mut self, buf: &[u8]) -> io::Result { + match self.append(buf) { + Ok(_) => Ok(buf.len()), + Err(Error::Storage(StorageError::Io(e))) => Err(e), + Err(other) => Err(io::Error::other(other)), + } + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn make() -> Storage { + Storage::new(StorageOptions { + spill_threshold: 16, + tempdir: None, + }) + } + + #[test] + fn append_and_read_in_memory() { + let mut s = Storage::new(StorageOptions::default()); + let off = s.append(b"hello world").unwrap(); + assert_eq!(off, 0); + let mut out = Vec::new(); + s.copy_range_to(0, 11, &mut out).unwrap(); + assert_eq!(out, b"hello world"); + } + + #[test] + fn spilling_preserves_offsets() { + let mut s = make(); + let a = s.append(&[0xAA; 8]).unwrap(); + let b = s.append(&[0xBB; 12]).unwrap(); // triggers spill (>= 16) + let c = s.append(&[0xCC; 4]).unwrap(); + assert_eq!(a, 0); + assert_eq!(b, 8); + assert_eq!(c, 20); + + let mut out = Vec::new(); + s.copy_range_to(0, 24, &mut out).unwrap(); + assert!(out[..8].iter().all(|&x| x == 0xAA)); + assert!(out[8..20].iter().all(|&x| x == 0xBB)); + assert!(out[20..24].iter().all(|&x| x == 0xCC)); + } + + #[test] + fn straddle_read_works() { + let mut s = make(); + s.append(&[1u8; 10]).unwrap(); + s.append(&[2u8; 10]).unwrap(); // spills + s.append(&[3u8; 5]).unwrap(); // back in buffer + let mut out = Vec::new(); + // range 18..22 spans the file (18..20) + buffer (20..22) + s.copy_range_to(18, 4, &mut out).unwrap(); + assert_eq!(out, vec![2, 2, 3, 3]); + } +} diff --git a/reference-impl/rust/star-lite/src/verify.rs b/reference-impl/rust/star-lite/src/verify.rs new file mode 100644 index 0000000..ef5cc01 --- /dev/null +++ b/reference-impl/rust/star-lite/src/verify.rs @@ -0,0 +1,60 @@ +//! Full archive verification. +//! +//! Reconstructs the MST from a STAR-lite archive's `(key, record)` stream, +//! computes the root CID, and asserts it matches the header CID (= the +//! commit's `data` field, when a partial commit is present). +//! +//! This is the cheapest verification mode: records are hashed as they stream +//! in and discarded, MST nodes are encoded and discarded the moment their +//! CID is folded into the parent. No byte log, no disk spill, no record +//! retention. The structural algorithm is the same one CAR conversion uses +//! — they share [`star_repo::mst::MstStack`] — so any divergence between +//! verify and convert is impossible by construction. + +use std::io::Read; + +use ipld_core::cid::Cid; + +use star_repo::commit::{CommitInfo, parse_partial_commit}; + +use crate::error::{Result, VerifyError}; +use crate::mst::{BuilderOptions, MstStack}; +use crate::reader::Reader; + +/// Re-exported here so callers can write `star_lite::verify::VerifyBackend`. +/// The canonical location is [`crate::mst::VerifyBackend`]. +pub use crate::mst::VerifyBackend; + +/// Verify a STAR-lite archive end-to-end. +/// +/// Returns `(verified_root_cid, parsed_commit)` on success. The commit is +/// `Some` iff the archive carried a partial commit in its header. Returns +/// [`VerifyError::RootMismatch`] if the reconstructed MST root doesn't match +/// the archive's header CID. +pub fn verify_archive(input: R) -> Result<(Cid, Option)> { + let mut reader = Reader::new(input)?; + let header_cid = reader.header_cid(); + let partial = reader.partial_commit().map(|s| s.to_vec()); + + let mut stack = MstStack::new(VerifyBackend, BuilderOptions::default()); + while let Some(entry) = reader.next_entry()? { + stack.insert(entry.key.as_bytes(), &entry.record)?; + } + let (root_cid, _) = stack.finish()?; + + if root_cid != header_cid { + return Err( + VerifyError::RootMismatch(Box::new(crate::error::RootMismatch { + declared: header_cid, + computed: root_cid, + })) + .into(), + ); + } + + let commit = match partial { + Some(bytes) => Some(parse_partial_commit(&bytes, header_cid)?), + None => None, + }; + Ok((root_cid, commit)) +} diff --git a/reference-impl/rust/star-lite/src/writer.rs b/reference-impl/rust/star-lite/src/writer.rs new file mode 100644 index 0000000..261e4c4 --- /dev/null +++ b/reference-impl/rust/star-lite/src/writer.rs @@ -0,0 +1,133 @@ +//! Streaming writer for STAR-lite archives. +//! +//! Enforces strict lex ordering of keys at write time. Forgetting to call +//! [`Writer::finish`] is fine — the format has no trailer — but using the +//! writer in the wrong order (entries before header, etc.) is a hard error. + +use std::io::Write; + +use ipld_core::cid::Cid; + +use star_repo::varint; + +use crate::error::{FormatError, Result}; +use crate::reader::{HEADER_CID_LEN, HEADER_CID_PREFIX, MAGIC, MAX_PARTIAL_COMMIT_LEN}; + +/// Streaming writer that produces a STAR-lite archive. +pub struct Writer { + inner: W, + state: State, + prev_key: Option>, +} + +#[derive(Debug, PartialEq, Eq)] +enum State { + NeedsHeader, + Body, + Finished, +} + +impl Writer { + pub fn new(inner: W) -> Self { + Self { + inner, + state: State::NeedsHeader, + prev_key: None, + } + } + + /// Write the archive prologue: magic, header CID (= MST root), and + /// optional partial commit. Must be called exactly once, before any + /// [`write_entry`]. + /// + /// `data_cid` must be a CIDv1 / dag-cbor / sha-256 link (36 bytes on the + /// wire); other CID shapes are rejected. + pub fn write_header(&mut self, data_cid: Cid, partial_commit: Option<&[u8]>) -> Result<()> { + if self.state != State::NeedsHeader { + return Err(FormatError::BadWriterState("header already written").into()); + } + + self.inner.write_all(&MAGIC)?; + + let cid_bytes = data_cid.to_bytes(); + if cid_bytes.len() != HEADER_CID_LEN + || cid_bytes[..HEADER_CID_PREFIX.len()] != HEADER_CID_PREFIX + { + return Err(FormatError::InvalidHeaderCid.into()); + } + self.inner.write_all(&cid_bytes)?; + + let partial = partial_commit.unwrap_or(&[]); + let partial_len = partial.len() as u64; + if partial_len > MAX_PARTIAL_COMMIT_LEN { + return Err(FormatError::SizeLimit { + field: "partial commit", + actual: partial_len, + limit: MAX_PARTIAL_COMMIT_LEN, + } + .into()); + } + varint::write(&mut self.inner, partial_len)?; + self.inner.write_all(partial)?; + + self.state = State::Body; + Ok(()) + } + + /// Append a `(key, record)` entry. Keys must arrive in strict lex byte + /// order; duplicates are rejected. + pub fn write_entry(&mut self, key: &str, record: &[u8]) -> Result<()> { + match self.state { + State::NeedsHeader => { + return Err(FormatError::BadWriterState("must call write_header first").into()); + } + State::Finished => { + return Err(FormatError::BadWriterState("writer is finished").into()); + } + State::Body => {} + } + + let key_bytes = key.as_bytes(); + if key_bytes.is_empty() { + return Err(FormatError::EmptyKey.into()); + } + + if let Some(prev) = &self.prev_key + && key_bytes <= prev.as_slice() + { + let prev_str = String::from_utf8_lossy(prev).into_owned(); + return Err(FormatError::KeyOrder { + prev: prev_str, + curr: key.to_owned(), + } + .into()); + } + + varint::write(&mut self.inner, key_bytes.len() as u64)?; + self.inner.write_all(key_bytes)?; + varint::write(&mut self.inner, record.len() as u64)?; + self.inner.write_all(record)?; + + self.prev_key = Some(key_bytes.to_vec()); + Ok(()) + } + + /// Mark the writer as done. Optional, but flushes the underlying writer + /// and prevents further use. + pub fn finish(mut self) -> Result<()> { + match self.state { + State::NeedsHeader => { + return Err(FormatError::BadWriterState("no header was written").into()); + } + State::Finished => return Ok(()), + State::Body => {} + } + self.inner.flush()?; + self.state = State::Finished; + Ok(()) + } + + pub fn into_inner(self) -> W { + self.inner + } +} diff --git a/reference-impl/rust/star-lite/tests/basic.rs b/reference-impl/rust/star-lite/tests/basic.rs new file mode 100644 index 0000000..2dee4db --- /dev/null +++ b/reference-impl/rust/star-lite/tests/basic.rs @@ -0,0 +1,201 @@ +//! Reader/writer round-trip tests and strict ordering checks. + +use ipld_core::cid::Cid; +use star_lite::error::FormatError; +use star_lite::reader::{HEADER_CID_PREFIX, MAGIC}; +use star_lite::{Error, Reader, Writer}; +use std::io::Cursor; + +/// A minimal valid header CID: the standard atproto prefix + 32 zero bytes. +/// The digest is not a real hash of anything; tests that don't compute MST +/// roots just need *some* well-formed header CID. +fn synthetic_cid() -> Cid { + let mut bytes = [0u8; 36]; + bytes[..4].copy_from_slice(&HEADER_CID_PREFIX); + Cid::try_from(&bytes[..]).expect("valid CID prefix") +} + +/// Hand-rolled header: magic + synthetic CID + varint(0) for an absent +/// partial commit. +fn handcrafted_header(buf: &mut Vec) { + buf.extend_from_slice(&MAGIC); + buf.extend_from_slice(&HEADER_CID_PREFIX); + buf.extend_from_slice(&[0u8; 32]); + buf.push(0); +} + +fn build_archive(entries: &[(&str, &[u8])]) -> Vec { + let mut buf = Vec::new(); + let mut w = Writer::new(&mut buf); + w.write_header(synthetic_cid(), None).unwrap(); + for (k, v) in entries { + w.write_entry(k, v).unwrap(); + } + w.finish().unwrap(); + buf +} + +#[test] +fn empty_repo_roundtrips() { + let archive = build_archive(&[]); + + assert_eq!(&archive[..3], &MAGIC); + + let reader = Reader::new(Cursor::new(&archive)).unwrap(); + assert_eq!(reader.header_cid(), synthetic_cid()); + assert!(reader.partial_commit().is_none()); + + let collected: Vec<_> = reader.collect::>().unwrap(); + assert!(collected.is_empty()); +} + +#[test] +fn full_roundtrip_preserves_keys_and_records() { + let entries: &[(&str, &[u8])] = &[ + ("app.bsky.feed.post/3kabcabc", b"\xA1\x64body\x65hello"), + ("app.bsky.feed.post/3kabcabd", b"\xA1\x64body\x65world"), + ("app.bsky.graph.follow/3kfollow", b"\xA0"), + ("app.bsky.graph.follow/3lfollow", b"\xA0"), + ]; + let archive = build_archive(entries); + + let reader = Reader::new(Cursor::new(&archive)).unwrap(); + let parsed: Vec<_> = reader.map(|r| r.unwrap()).collect(); + + assert_eq!(parsed.len(), entries.len()); + for (out, (k, v)) in parsed.iter().zip(entries) { + assert_eq!(out.key, *k); + assert_eq!(out.record, *v); + } +} + +#[test] +fn writer_rejects_out_of_order_keys() { + let mut buf = Vec::new(); + let mut w = Writer::new(&mut buf); + w.write_header(synthetic_cid(), None).unwrap(); + w.write_entry("b/2", &[]).unwrap(); + + let err = w.write_entry("a/1", &[]).unwrap_err(); + assert!(matches!(err, Error::Format(FormatError::KeyOrder { .. }))); +} + +#[test] +fn writer_rejects_duplicate_keys() { + let mut buf = Vec::new(); + let mut w = Writer::new(&mut buf); + w.write_header(synthetic_cid(), None).unwrap(); + w.write_entry("a/1", &[]).unwrap(); + + let err = w.write_entry("a/1", &[]).unwrap_err(); + assert!(matches!(err, Error::Format(FormatError::KeyOrder { .. }))); +} + +#[test] +fn writer_requires_header_first() { + let mut buf = Vec::new(); + let mut w = Writer::new(&mut buf); + let err = w.write_entry("a/1", &[]).unwrap_err(); + assert!(matches!(err, Error::Format(FormatError::BadWriterState(_)))); +} + +#[test] +fn reader_rejects_out_of_order_archive() { + // Hand-crafted archive: two keys in descending order. + // Format: magic || cid(36) || varint(commit_len) || partial || + // varint(klen) || k || varint(rlen) || r || ... + let mut buf = Vec::new(); + handcrafted_header(&mut buf); + // "b" first + buf.push(0x01); + buf.push(b'b'); + buf.push(0x00); + // then "a" — out of order + buf.push(0x01); + buf.push(b'a'); + buf.push(0x00); + + let mut reader = Reader::new(Cursor::new(&buf)).unwrap(); + assert_eq!(reader.next_entry().unwrap().unwrap().key, "b"); + let err = reader.next_entry().unwrap_err(); + assert!(matches!(err, Error::Format(FormatError::KeyOrder { .. }))); +} + +#[test] +fn reader_rejects_bad_magic() { + let buf = b"XXXextra-bytes"; + let err = match Reader::new(Cursor::new(&buf[..])) { + Ok(_) => panic!("expected error for bad magic"), + Err(e) => e, + }; + assert!(matches!(err, Error::Format(FormatError::InvalidMagic))); +} + +#[test] +fn reader_rejects_empty_input() { + let buf: &[u8] = &[]; + let err = match Reader::new(Cursor::new(buf)) { + Ok(_) => panic!("expected error for empty input"), + Err(e) => e, + }; + assert!(matches!(err, Error::Format(FormatError::InvalidMagic))); +} + +#[test] +fn reader_rejects_bad_header_cid_prefix() { + let mut buf = Vec::new(); + buf.extend_from_slice(&MAGIC); + // Wrong prefix: not 0x01711220. + buf.extend_from_slice(&[0xFF, 0xFF, 0xFF, 0xFF]); + buf.extend_from_slice(&[0u8; 32]); + buf.push(0); + let err = match Reader::new(Cursor::new(&buf)) { + Ok(_) => panic!("expected error for bad header CID prefix"), + Err(e) => e, + }; + assert!(matches!(err, Error::Format(FormatError::InvalidHeaderCid))); +} + +#[test] +fn reader_rejects_truncated_record() { + let mut buf = Vec::new(); + handcrafted_header(&mut buf); + buf.push(0x05); // declares 5-byte key + buf.extend_from_slice(b"ab"); // only 2 bytes provided + + let mut reader = Reader::new(Cursor::new(&buf)).unwrap(); + let err = reader.next_entry().unwrap_err(); + assert!(matches!(err, Error::Format(FormatError::UnexpectedEof))); +} + +#[test] +fn reader_enforces_size_limits() { + use star_lite::Limits; + + let mut buf = Vec::new(); + handcrafted_header(&mut buf); + // Declare a 1024-byte key (exceeds our test limit of 4). + // 1024 = 0x400 → varint: 0x80 0x08 + buf.push(0x80); + buf.push(0x08); + + let limits = Limits { + max_key_len: 4, + ..Limits::default() + }; + let mut reader = Reader::with_limits(Cursor::new(&buf), limits).unwrap(); + let err = reader.next_entry().unwrap_err(); + assert!(matches!( + err, + Error::Format(FormatError::SizeLimit { field: "key", .. }) + )); +} + +#[test] +fn writer_rejects_empty_key() { + let mut buf = Vec::new(); + let mut w = Writer::new(&mut buf); + w.write_header(synthetic_cid(), None).unwrap(); + let err = w.write_entry("", b"x").unwrap_err(); + assert!(matches!(err, Error::Format(FormatError::EmptyKey))); +} diff --git a/reference-impl/rust/star-lite/tests/mst.rs b/reference-impl/rust/star-lite/tests/mst.rs new file mode 100644 index 0000000..011f720 --- /dev/null +++ b/reference-impl/rust/star-lite/tests/mst.rs @@ -0,0 +1,208 @@ +//! Unit tests for the MST stack + backends. +//! +//! These cover the structural invariants from AT-Repo §2.5: empty repo, +//! single-leaf root, and the bridge-node case (intermediate empty-entry +//! nodes between a high-layer key and its low-layer sibling). +//! +//! The asserts here run the same input through `MstStack` and +//! `MstStack` and compare root CIDs: if both backends produce +//! the same root CID for the same input, the structural rules are consistent +//! across the two implementations. (Since both share `MstStack`, they cannot +//! disagree on structure — but the cross-check still guards against the +//! backend impls accidentally introducing divergence.) + +use ipld_core::cid::Cid; +use star_lite::mst::{BuilderOptions, CarBackend, MstStack}; +use star_lite::storage::{Storage, StorageOptions}; +use star_lite::verify::VerifyBackend; + +fn frozen_root(entries: &[(&str, &[u8])]) -> Cid { + let backend = CarBackend::new(Storage::new(StorageOptions::default())); + let mut stack = MstStack::new(backend, BuilderOptions::default()); + for (k, v) in entries { + stack.insert(k.as_bytes(), v).unwrap(); + } + stack.finish().unwrap().0.root_cid +} + +fn verify_root(entries: &[(&str, &[u8])]) -> Cid { + let mut stack = MstStack::new(VerifyBackend, BuilderOptions::default()); + for (k, v) in entries { + stack.insert(k.as_bytes(), v).unwrap(); + } + stack.finish().unwrap().0 +} + +#[test] +fn empty_repo_produces_canonical_empty_node() { + let f = frozen_root(&[]); + let v = verify_root(&[]); + assert_eq!(f, v); +} + +#[test] +fn single_leaf_at_layer_zero() { + use star_lite::mst::layer_of; + // pick a key we know is at layer 0 + let key = "app.bsky.feed.post/abc"; + assert_eq!(layer_of(key.as_bytes()), 0); + let f = frozen_root(&[(key, b"hello")]); + let v = verify_root(&[(key, b"hello")]); + assert_eq!(f, v); +} + +#[test] +fn frozen_and_verify_agree_on_random_corpus() { + // Larger random sample to exercise the stack-fold across multiple layers. + let mut entries = Vec::new(); + for i in 0..500 { + let k = format!("c.col/rkey-{i:05}"); + let v = format!("record-body-{i}").into_bytes(); + entries.push((k, v)); + } + entries.sort_by(|a, b| a.0.cmp(&b.0)); + let entry_refs: Vec<(&str, &[u8])> = entries + .iter() + .map(|(k, v)| (k.as_str(), v.as_slice())) + .collect(); + + let f = frozen_root(&entry_refs); + let v = verify_root(&entry_refs); + assert_eq!(f, v); +} + +#[test] +fn search_and_use_high_layer_keys() { + use star_lite::mst::layer_of; + // Find a couple of keys at unusually high layers to force the bridge-node + // codepath. Layers are 2-bit groups, so layer >= 2 means the SHA-256 of + // the key starts with 4+ leading zero bits — about 1 in 16 keys. + let mut high_layer_keys: Vec = Vec::new(); + for i in 0..10_000 { + let k = format!("c.col/k-{i:08}"); + if layer_of(k.as_bytes()) >= 2 { + high_layer_keys.push(k); + if high_layer_keys.len() >= 2 { + break; + } + } + } + assert!( + high_layer_keys.len() >= 2, + "expected to find some layer-2+ keys in 10k samples" + ); + + // Mix with a layer-0 key sandwich. + let mut entries: Vec<(String, Vec)> = Vec::new(); + entries.push(("c.col/aaaaaaaa".into(), b"a".to_vec())); + for k in &high_layer_keys { + entries.push((k.clone(), b"h".to_vec())); + } + entries.push(("c.col/zzzzzzzz".into(), b"z".to_vec())); + entries.sort_by(|a, b| a.0.cmp(&b.0)); + + let entry_refs: Vec<(&str, &[u8])> = entries + .iter() + .map(|(k, v)| (k.as_str(), v.as_slice())) + .collect(); + let f = frozen_root(&entry_refs); + let v = verify_root(&entry_refs); + assert_eq!(f, v); +} + +#[test] +fn rejects_out_of_order_keys() { + let backend = CarBackend::new(Storage::new(StorageOptions::default())); + let mut stack = MstStack::new(backend, BuilderOptions::default()); + stack.insert(b"b", b"x").unwrap(); + let err = stack.insert(b"a", b"x"); + assert!(err.is_err()); +} + +/// The cap is enforced eagerly during insert — not only at encode/finish +/// time. Without this, an attacker submitting a long run of low-layer keys +/// (cheap to construct: ~75% of random keys are layer 0) could grow a +/// layer's entries vector without bound, OOMing the process before any cap +/// fires. +/// +/// The check now lives in `MstStack::insert`, so verify and CAR conversion +/// share it by construction. We test it here through both backends to +/// confirm both fail at the same boundary. +#[test] +fn stack_enforces_cap_eagerly_through_car_backend() { + use star_lite::Error; + use star_repo::error::MstError; + use star_repo::mst::layer_of; + + let opts = BuilderOptions { + max_entries_per_node: 4, + }; + let backend = CarBackend::new(Storage::new(StorageOptions::default())); + let mut stack = MstStack::new(backend, opts); + + // Find five layer-0 keys in lex order; they all want to live at the + // same level, and the 5th must trigger the cap. + let mut layer_zero: Vec = (0..200) + .map(|i| format!("c.col/k-{i:08}")) + .filter(|k| layer_of(k.as_bytes()) == 0) + .take(5) + .collect(); + layer_zero.sort(); + assert_eq!(layer_zero.len(), 5, "need 5 layer-0 keys for this test"); + + for k in &layer_zero[..4] { + stack.insert(k.as_bytes(), k.as_bytes()).unwrap(); + } + + let err = stack + .insert(layer_zero[4].as_bytes(), layer_zero[4].as_bytes()) + .unwrap_err(); + assert!( + matches!( + err, + Error::Repo(star_repo::Error::Mst(MstError::NodeTooLarge { + count: 5, + limit: 4 + })) + ), + "expected NodeTooLarge {{ count: 5, limit: 4 }}, got {err:?}" + ); +} + +#[test] +fn stack_enforces_cap_eagerly_through_verify_backend() { + use star_lite::Error; + use star_repo::error::MstError; + use star_repo::mst::layer_of; + + let opts = BuilderOptions { + max_entries_per_node: 4, + }; + let mut stack = MstStack::new(VerifyBackend, opts); + + let mut layer_zero: Vec = (0..200) + .map(|i| format!("c.col/k-{i:08}")) + .filter(|k| layer_of(k.as_bytes()) == 0) + .take(5) + .collect(); + layer_zero.sort(); + assert_eq!(layer_zero.len(), 5); + + for k in &layer_zero[..4] { + stack.insert(k.as_bytes(), k.as_bytes()).unwrap(); + } + + let err = stack + .insert(layer_zero[4].as_bytes(), layer_zero[4].as_bytes()) + .unwrap_err(); + assert!( + matches!( + err, + Error::Repo(star_repo::Error::Mst(MstError::NodeTooLarge { + count: 5, + limit: 4 + })) + ), + "expected NodeTooLarge {{ count: 5, limit: 4 }}, got {err:?}" + ); +} diff --git a/reference-impl/rust/star-lite/tests/roundtrip.rs b/reference-impl/rust/star-lite/tests/roundtrip.rs new file mode 100644 index 0000000..e476c7e --- /dev/null +++ b/reference-impl/rust/star-lite/tests/roundtrip.rs @@ -0,0 +1,199 @@ +//! End-to-end round-trip: STAR-lite -> CAR -> STAR-lite, asserting +//! byte-identical output and matching commit/MST structure. +//! +//! Since the CAR builder validates that `commit.data` equals the computed +//! MST root CID, our synthetic commits have to be constructed in the right +//! order: first build the records, compute the MST root via the (cheaper) +//! verify backend, *then* construct the commit with that CID embedded. + +use ipld_core::cid::Cid; +use serde::Serialize; +use std::io::Cursor; + +use star_lite::commit::parse_commit; +use star_lite::mst::{BuilderOptions, MstStack}; +use star_lite::verify::VerifyBackend; +use star_lite::{ + ConvertOptions, Reader, Writer, car_to_star_lite, star_lite_to_car, verify_archive, +}; + +#[derive(Debug, Serialize)] +struct CommitForSigning<'a> { + did: &'a str, + version: u64, + data: Cid, + rev: &'a str, + prev: Option, + sig: serde_bytes::ByteBuf, +} + +/// Build a synthetic "signed" commit. The signature is a fake 64-byte zero +/// blob — we never verify signatures in this crate, only field structure. +fn synthetic_commit(did: &str, rev: &str, data: Cid) -> Vec { + let commit = CommitForSigning { + did, + version: 3, + data, + rev, + prev: None, + sig: serde_bytes::ByteBuf::from(vec![0u8; 64]), + }; + serde_ipld_dagcbor::to_vec(&commit).unwrap() +} + +/// Compute the MST root CID for a sorted list of `(key, record_bytes)`. +fn mst_root_for(entries: &[(&str, &[u8])]) -> Cid { + let mut stack = MstStack::new(VerifyBackend, BuilderOptions::default()); + for (k, v) in entries { + stack.insert(k.as_bytes(), v).unwrap(); + } + stack.finish().unwrap().0 +} + +/// Build a STAR-lite archive from a list of entries plus a synthetic commit. +fn build_starlite(entries: &[(&str, &[u8])]) -> Vec { + let data = mst_root_for(entries); + let commit_bytes = synthetic_commit("did:plc:test", "3l3lalalalala", data); + let commit = parse_commit(&commit_bytes).unwrap(); + let partial = commit.to_partial_bytes().unwrap(); + + let mut buf = Vec::new(); + let mut w = Writer::new(&mut buf); + w.write_header(data, Some(&partial)).unwrap(); + for (k, v) in entries { + w.write_entry(k, v).unwrap(); + } + w.finish().unwrap(); + buf +} + +#[test] +fn empty_repo_roundtrips_through_car() { + let starlite = build_starlite(&[]); + + // STAR-lite -> CAR + let mut car = Vec::new(); + star_lite_to_car(Cursor::new(&starlite), &mut car, ConvertOptions::default()).unwrap(); + + // CAR -> STAR-lite + let mut starlite_back = Vec::new(); + car_to_star_lite(Cursor::new(&car), &mut starlite_back).unwrap(); + + assert_eq!(starlite_back, starlite); +} + +#[test] +fn small_repo_roundtrips_through_car() { + let entries: &[(&str, &[u8])] = &[ + ("app.bsky.feed.post/3kabcabc", b"\xA1\x64body\x65hello"), + ("app.bsky.feed.post/3kabcabd", b"\xA1\x64body\x65world"), + ("app.bsky.graph.follow/3kfollow", b"\xA0"), + ]; + let starlite = build_starlite(entries); + + let mut car = Vec::new(); + star_lite_to_car(Cursor::new(&starlite), &mut car, ConvertOptions::default()).unwrap(); + + let mut starlite_back = Vec::new(); + car_to_star_lite(Cursor::new(&car), &mut starlite_back).unwrap(); + + assert_eq!(starlite_back, starlite); +} + +#[test] +fn larger_repo_roundtrips_through_car() { + let mut entries: Vec<(String, Vec)> = Vec::new(); + for i in 0..200 { + let k = format!("app.bsky.feed.post/{i:08}"); + let v = format!("post body {i}").into_bytes(); + entries.push((k, v)); + } + entries.sort_by(|a, b| a.0.cmp(&b.0)); + let refs: Vec<(&str, &[u8])> = entries + .iter() + .map(|(k, v)| (k.as_str(), v.as_slice())) + .collect(); + + let starlite = build_starlite(&refs); + + let mut car = Vec::new(); + star_lite_to_car(Cursor::new(&starlite), &mut car, ConvertOptions::default()).unwrap(); + + let mut starlite_back = Vec::new(); + car_to_star_lite(Cursor::new(&car), &mut starlite_back).unwrap(); + + assert_eq!(starlite_back, starlite); +} + +#[test] +fn verify_archive_succeeds_for_valid_archive() { + let entries: &[(&str, &[u8])] = &[ + ("app.bsky.feed.post/3kaaa", b"\xA0"), + ("app.bsky.feed.post/3kbbb", b"\xA0"), + ]; + let starlite = build_starlite(entries); + let (root, info) = verify_archive(Cursor::new(&starlite)).unwrap(); + assert_eq!(root, mst_root_for(entries)); + let info = info.expect("partial commit should be present"); + assert_eq!(info.did, "did:plc:test"); + assert_eq!(info.version, 3); +} + +#[test] +fn verify_archive_fails_when_data_cid_mismatches() { + let entries: &[(&str, &[u8])] = &[("app.bsky.feed.post/3kaaa", b"\xA0")]; + // Use a header CID for a *different* entry set, so the reconstructed MST + // root won't match. + let wrong_data = mst_root_for(&[("c.x/y", b"different")]); + let commit_bytes = synthetic_commit("did:plc:test", "3l3lalala", wrong_data); + let commit = parse_commit(&commit_bytes).unwrap(); + let partial = commit.to_partial_bytes().unwrap(); + + let mut starlite = Vec::new(); + let mut w = Writer::new(&mut starlite); + w.write_header(wrong_data, Some(&partial)).unwrap(); + for (k, v) in entries { + w.write_entry(k, v).unwrap(); + } + w.finish().unwrap(); + + let err = verify_archive(Cursor::new(&starlite)).unwrap_err(); + let msg = format!("{err}"); + assert!(msg.contains("does not match"), "got: {msg}"); +} + +#[test] +fn star_lite_to_car_validates_commit_data() { + // Same as above, but exercising the to_car path — which also verifies. + let entries: &[(&str, &[u8])] = &[("c.col/key", b"\xA0")]; + let wrong_data = mst_root_for(&[("c.col/different", b"\xA0")]); + let commit_bytes = synthetic_commit("did:plc:test", "3lzzz", wrong_data); + let commit = parse_commit(&commit_bytes).unwrap(); + let partial = commit.to_partial_bytes().unwrap(); + + let mut starlite = Vec::new(); + let mut w = Writer::new(&mut starlite); + w.write_header(wrong_data, Some(&partial)).unwrap(); + for (k, v) in entries { + w.write_entry(k, v).unwrap(); + } + w.finish().unwrap(); + + let mut car = Vec::new(); + let err = + star_lite_to_car(Cursor::new(&starlite), &mut car, ConvertOptions::default()).unwrap_err(); + let msg = format!("{err}"); + assert!(msg.contains("does not match"), "got: {msg}"); +} + +#[test] +fn reader_iterates_commit_metadata() { + let entries: &[(&str, &[u8])] = &[("c.col/k1", b"r1"), ("c.col/k2", b"r2")]; + let starlite = build_starlite(entries); + + let reader = Reader::new(Cursor::new(&starlite)).unwrap(); + let parsed: Vec<_> = reader.collect::, _>>().unwrap(); + assert_eq!(parsed.len(), 2); + assert_eq!(parsed[0].key, "c.col/k1"); + assert_eq!(parsed[1].key, "c.col/k2"); +} diff --git a/reference-impl/rust/star-repo/Cargo.toml b/reference-impl/rust/star-repo/Cargo.toml new file mode 100644 index 0000000..111f316 --- /dev/null +++ b/reference-impl/rust/star-repo/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "star-repo" +version = "0.1.0" +description = "CARv1 and MST helpers for star-* crates" +repository = "https://tangled.org/microcosm.blue/star" +readme = "readme.md" +edition = "2024" +license = "MIT OR Apache-2.0" + +[dependencies] +ipld-core = { workspace = true } +serde = { workspace = true } +serde_bytes = { workspace = true } +serde_ipld_dagcbor = { workspace = true } +sha2 = { workspace = true } +thiserror = { workspace = true } +unsigned-varint = { workspace = true } diff --git a/reference-impl/rust/star-repo/LICENSE.Apache-2.0 b/reference-impl/rust/star-repo/LICENSE.Apache-2.0 new file mode 100644 index 0000000..93f59f2 --- /dev/null +++ b/reference-impl/rust/star-repo/LICENSE.Apache-2.0 @@ -0,0 +1,190 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +Copyright 2026 microcosm + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/reference-impl/rust/star-repo/LICENSE.MIT b/reference-impl/rust/star-repo/LICENSE.MIT new file mode 100644 index 0000000..21246cc --- /dev/null +++ b/reference-impl/rust/star-repo/LICENSE.MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 microcosm + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/reference-impl/rust/star-repo/readme.md b/reference-impl/rust/star-repo/readme.md new file mode 100644 index 0000000..b79e931 --- /dev/null +++ b/reference-impl/rust/star-repo/readme.md @@ -0,0 +1,5 @@ +# star-repo (rust) + +WARNING: NOT FULLY REVIEWED YET + +the CAR stuff is probably a bit hack -- we should be pulling in other ecosystem crates. *maybe* with feature-flags to pick your favourite? diff --git a/reference-impl/rust/star-repo/src/blockstore.rs b/reference-impl/rust/star-repo/src/blockstore.rs new file mode 100644 index 0000000..9b00435 --- /dev/null +++ b/reference-impl/rust/star-repo/src/blockstore.rs @@ -0,0 +1,46 @@ +//! Reading a CARv1 file into an in-memory blockstore (STUB). +//! +//! For converting CAR -> STAR-lite we need random access to blocks by CID: +//! given the commit, look up the MST root, walk the tree, and dereference +//! each leaf record. This uses a `HashMap>`, which costs roughly +//! the size of the repo in memory. +//! +//! `repo-stream` is better for this, and might eventually replace this. + +use std::collections::HashMap; +use std::io::Read; + +use ipld_core::cid::Cid; + +use crate::error::{Error, Result}; +use crate::frame::{CarHeader, read_frame, read_header}; + +pub struct Blockstore { + pub header: CarHeader, + pub blocks: HashMap>, +} + +impl Blockstore { + pub fn from_reader(mut r: R) -> Result { + let header = read_header(&mut r)?; + let mut blocks = HashMap::new(); + while let Some((cid, bytes)) = read_frame(&mut r)? { + if blocks.insert(cid, bytes).is_some() { + return Err(Error::DuplicateBlock(Box::new(cid))); + } + } + Ok(Self { header, blocks }) + } + + pub fn root(&self) -> Result { + // atproto CARs always declare a single root: the commit. + self.header.roots.first().copied().ok_or(Error::NoRoots) + } + + pub fn get(&self, cid: &Cid) -> Result<&[u8]> { + self.blocks + .get(cid) + .map(|v| v.as_slice()) + .ok_or_else(|| Error::BlockNotFound(Box::new(*cid))) + } +} diff --git a/reference-impl/rust/star-repo/src/commit.rs b/reference-impl/rust/star-repo/src/commit.rs new file mode 100644 index 0000000..1ec9a87 --- /dev/null +++ b/reference-impl/rust/star-repo/src/commit.rs @@ -0,0 +1,184 @@ +//! Parsing the atproto signed commit object. +//! +//! Per [AT-Repo §2.4][spec] commits have: +//! +//! ```text +//! { did: string, version: 3, data: Cid, rev: string, +//! prev: Cid | null, sig: bytes } +//! ``` +//! +//! All six fields are *required* on the wire, with `prev` being nullable +//! (always present, possibly null). +//! +//! ## Signature verification +//! +//! We do not verify signatures here. Atproto allows two signing curves +//! (secp256k1 and p256), needs DID resolution to find the public key, and +//! requires low-S canonicalization (§2.6.3) — bundling all of that into +//! this crate would be an awkward fit. +//! +//! Instead we expose [`CommitInfo::signed_bytes`]: a re-encoded canonical +//! DAG-CBOR of the unsigned commit (everything except `sig`). Per §2.6.1 +//! the signature was made over `sha256(unsigned_commit_cbor)`. A caller +//! can resolve the DID, fetch the public key, and verify themselves. +//! +//! Caveat: the re-encoded `signed_bytes` matches the original signing input +//! bit-for-bit only if the original was canonical DAG-CBOR. The spec +//! requires this (§2.7), but a misbehaving PDS could produce a signed but +//! non-canonical commit, in which case re-encoding would not validate. +//! There's no clean way around that without holding the original byte slice +//! and excising the `sig` field in place — more work than it's worth here. +//! +//! [spec]: https://www.ietf.org/archive/id/draft-holmgren-at-repository-01.txt + +use ipld_core::cid::Cid; +use serde::{Deserialize, Serialize}; + +use crate::error::{CommitError, Result}; + +/// The version number expected in atproto v3 commits. +pub const SUPPORTED_COMMIT_VERSION: u64 = 3; + +/// Metadata extracted from an atproto signed commit, plus the bytes a caller +/// needs to verify the signature themselves. +#[derive(Debug, Clone)] +pub struct CommitInfo { + pub did: String, + pub version: u64, + pub data: Cid, + pub rev: String, + /// Always present in v3 commits per the spec, but typically null. + pub prev: Option, + pub sig: Vec, + /// Canonical DAG-CBOR encoding of the unsigned commit (the five non-`sig` + /// fields). Per §2.6.1 the signature was made over the SHA-256 hash of + /// this byte string. + pub signed_bytes: Vec, + /// The original signed commit bytes as provided. Useful for passing + /// through to a CAR file unchanged. + pub raw: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +struct SignedCommit { + did: String, + version: u64, + data: Cid, + rev: String, + prev: Option, + sig: serde_bytes::ByteBuf, +} + +#[derive(Debug, Serialize, Deserialize)] +struct UnsignedCommit { + did: String, + version: u64, + data: Cid, + rev: String, + prev: Option, +} + +/// Commit-minus-`data`: the on-the-wire form of the partial commit embedded in +/// a STAR-lite header. `data` is recovered from the header CID at parse time. +#[derive(Debug, Serialize, Deserialize)] +struct PartialCommit { + did: String, + version: u64, + rev: String, + prev: Option, + sig: serde_bytes::ByteBuf, +} + +/// Parse the bytes of an atproto signed commit. +pub fn parse_commit(bytes: &[u8]) -> Result { + let signed: SignedCommit = + serde_ipld_dagcbor::from_slice(bytes).map_err(|e| CommitError::Decode(e.to_string()))?; + + if signed.version != SUPPORTED_COMMIT_VERSION { + return Err(CommitError::UnsupportedVersion(signed.version).into()); + } + + let unsigned = UnsignedCommit { + did: signed.did.clone(), + version: signed.version, + data: signed.data, + rev: signed.rev.clone(), + prev: signed.prev, + }; + let signed_bytes = + serde_ipld_dagcbor::to_vec(&unsigned).map_err(|e| CommitError::Encode(e.to_string()))?; + + Ok(CommitInfo { + did: signed.did, + version: signed.version, + data: signed.data, + rev: signed.rev, + prev: signed.prev, + sig: signed.sig.into_vec(), + signed_bytes, + raw: bytes.to_vec(), + }) +} + +/// Parse a STAR-lite partial commit (commit minus `data`) plus the header CID +/// (= `data`) into a full [`CommitInfo`]. +/// +/// `raw` on the returned struct is the canonical DAG-CBOR encoding of the full +/// signed commit, re-built from `partial_bytes` and `data`. As with +/// [`parse_commit`], the re-encoded bytes match the original signing input +/// only when `partial_bytes` was canonical. +pub fn parse_partial_commit(partial_bytes: &[u8], data: Cid) -> Result { + let partial: PartialCommit = serde_ipld_dagcbor::from_slice(partial_bytes) + .map_err(|e| CommitError::Decode(e.to_string()))?; + + if partial.version != SUPPORTED_COMMIT_VERSION { + return Err(CommitError::UnsupportedVersion(partial.version).into()); + } + + let unsigned = UnsignedCommit { + did: partial.did.clone(), + version: partial.version, + data, + rev: partial.rev.clone(), + prev: partial.prev, + }; + let signed_bytes = + serde_ipld_dagcbor::to_vec(&unsigned).map_err(|e| CommitError::Encode(e.to_string()))?; + + let signed = SignedCommit { + did: partial.did.clone(), + version: partial.version, + data, + rev: partial.rev.clone(), + prev: partial.prev, + sig: partial.sig.clone(), + }; + let raw = + serde_ipld_dagcbor::to_vec(&signed).map_err(|e| CommitError::Encode(e.to_string()))?; + + Ok(CommitInfo { + did: partial.did, + version: partial.version, + data, + rev: partial.rev, + prev: partial.prev, + sig: partial.sig.into_vec(), + signed_bytes, + raw, + }) +} + +impl CommitInfo { + /// Re-encode this commit *without* its `data` field, in canonical + /// DAG-CBOR. This is the form embedded in a STAR-lite header. + pub fn to_partial_bytes(&self) -> Result> { + let partial = PartialCommit { + did: self.did.clone(), + version: self.version, + rev: self.rev.clone(), + prev: self.prev, + sig: serde_bytes::ByteBuf::from(self.sig.clone()), + }; + serde_ipld_dagcbor::to_vec(&partial).map_err(|e| CommitError::Encode(e.to_string()).into()) + } +} diff --git a/reference-impl/rust/star-repo/src/error.rs b/reference-impl/rust/star-repo/src/error.rs new file mode 100644 index 0000000..e8eeb97 --- /dev/null +++ b/reference-impl/rust/star-repo/src/error.rs @@ -0,0 +1,115 @@ +//! Errors for the repo helper crate. + +use std::io; + +use ipld_core::cid::Cid; +use thiserror::Error; + +pub type Result = std::result::Result; + +#[derive(Debug, Error)] +pub enum Error { + #[error("io: {0}")] + Io(#[from] io::Error), + + #[error("invalid varint (too long, non-minimal, or truncated)")] + InvalidVarint, + + #[error("unexpected end of file")] + UnexpectedEof, + + #[error("invalid car header: {0}")] + InvalidHeader(String), + + #[error("unsupported car version: {0}")] + UnsupportedVersion(u64), + + #[error("block declared with size {actual} exceeds limit {limit}")] + BlockTooLarge { actual: u64, limit: u64 }, + + #[error("block size {block} too small to contain a CID")] + BlockTooSmall { block: u64 }, + + #[error("invalid CID encoding in block frame: {0}")] + InvalidCid(String), + + #[error("{0}")] + HashMismatch(Box), + + #[error("car file references missing block: {0}")] + BlockNotFound(Box), + + #[error("car file has duplicate block: {0}")] + DuplicateBlock(Box), + + #[error("unsupported codec for atproto block: {0:#x}")] + UnsupportedCodec(u64), + + #[error("unsupported multihash code for atproto block: {0:#x}")] + UnsupportedMultihash(u64), + + #[error("car file declared no roots; expected exactly one (the commit)")] + NoRoots, + + #[error("dag-cbor: {0}")] + Cbor(String), + + #[error("mst: {0}")] + Mst(#[from] MstError), + + #[error("commit: {0}")] + Commit(#[from] CommitError), +} + +/// Payload for [`Error::HashMismatch`]. Boxed inside `Error` so the +/// `Result` discriminant doesn't carry two CIDs of dead weight on the +/// success path. +#[derive(Debug, Error)] +#[error("block content does not match its declared CID: declared {declared}, computed {computed}")] +pub struct HashMismatch { + pub declared: Cid, + pub computed: Cid, +} + +/// Errors during MST construction or traversal. +#[derive(Debug, Error)] +pub enum MstError { + /// A single MST node would have more entries than the configured cap. + /// In atproto MSTs this is a strong signal of pathological input. + #[error("mst node would exceed entry cap: {count} entries (limit {limit})")] + NodeTooLarge { count: usize, limit: usize }, + + #[error("internal builder invariant violated: {0}")] + Invariant(&'static str), + + #[error("mst node encoding failed: {0}")] + EncodingFailed(String), + + /// A decoded MST node entry was structurally invalid: the first entry's + /// prefix length must be zero, subsequent entries' prefix lengths must + /// not exceed the previous key's length, etc. + #[error("invalid mst entry: {0}")] + InvalidEntry(String), + + #[error("empty key is not allowed")] + EmptyKey, + + #[error("keys out of order: {prev:?} must sort strictly before {curr:?}")] + KeyOrder { prev: String, curr: String }, +} + +/// Errors related to parsing or interpreting the atproto commit object. +#[derive(Debug, Error)] +pub enum CommitError { + #[error("commit cbor failed to decode: {0}")] + Decode(String), + + #[error("commit re-encoding failed (was the commit canonical dag-cbor?): {0}")] + Encode(String), + + #[error("unsupported commit version: {0}")] + UnsupportedVersion(u64), + + #[error("archive has no partial commit; required for this operation")] + MissingPartialCommit, +} diff --git a/reference-impl/rust/star-repo/src/frame.rs b/reference-impl/rust/star-repo/src/frame.rs new file mode 100644 index 0000000..197ed2b --- /dev/null +++ b/reference-impl/rust/star-repo/src/frame.rs @@ -0,0 +1,147 @@ +//! CARv1 frame encoding and decoding. +//! +//! A CARv1 file is: +//! +//! ```text +//! varint(header_len) || dag-cbor(header) +//! ( varint(payload_len) || cid_bytes || block_bytes )* +//! ``` +//! +//! The header is `{roots: [Cid], version: 1}`. Each block frame is a +//! length-prefixed concatenation of the CID and the block bytes; the varint +//! prefix counts both. + +use std::io::{Read, Write}; + +use ipld_core::cid::Cid; +use serde::{Deserialize, Serialize}; + +use crate::error::{Error, Result}; +use crate::hash::{DAG_CBOR_CODEC, SHA2_256_CODE, compute_cid_dag_cbor}; +use crate::varint; + +/// Per-block size cap used by the CAR reader to bound memory. +pub const MAX_BLOCK_LEN: u64 = 16 * 1024 * 1024; + +#[derive(Debug, Serialize, Deserialize)] +pub struct CarHeader { + pub roots: Vec, + pub version: u64, +} + +/// Write a CARv1 header to `w`. +pub fn write_header(w: &mut W, roots: &[Cid]) -> Result<()> { + let header = CarHeader { + roots: roots.to_vec(), + version: 1, + }; + let bytes = serde_ipld_dagcbor::to_vec(&header) + .map_err(|e| Error::Cbor(format!("car header encode: {e}")))?; + varint::write(w, bytes.len() as u64)?; + w.write_all(&bytes)?; + Ok(()) +} + +/// Read and validate a CARv1 header from `r`. +pub fn read_header(r: &mut R) -> Result { + let len = varint::read_required(r)?; + if len > MAX_BLOCK_LEN { + return Err(Error::BlockTooLarge { + actual: len, + limit: MAX_BLOCK_LEN, + }); + } + let mut buf = vec![0u8; len as usize]; + read_exact(r, &mut buf)?; + let header: CarHeader = serde_ipld_dagcbor::from_slice(&buf) + .map_err(|e| Error::InvalidHeader(format!("decode: {e}")))?; + if header.version != 1 { + return Err(Error::UnsupportedVersion(header.version)); + } + if header.roots.is_empty() { + return Err(Error::NoRoots); + } + Ok(header) +} + +/// Write a CARv1 block frame (`varint(payload_len) || cid || bytes`) to `w`. +/// Each piece is written directly to the sink; no intermediate buffer. +pub fn write_frame(w: &mut W, cid: &Cid, bytes: &[u8]) -> Result<()> { + let cid_len = cid.encoded_len(); + let payload_len = (cid_len + bytes.len()) as u64; + varint::write(w, payload_len)?; + cid.write_bytes(&mut *w) + .map_err(|e| Error::InvalidCid(e.to_string()))?; + w.write_all(bytes)?; + Ok(()) +} + +/// Build a complete CARv1 frame in memory (varint || cid || bytes). Used +/// occasionally where storage isn't appropriate. +pub fn build_frame(cid: &Cid, bytes: &[u8]) -> Result> { + let mut cid_buf = Vec::with_capacity(cid.encoded_len()); + cid.write_bytes(&mut cid_buf) + .map_err(|e| Error::InvalidCid(e.to_string()))?; + let payload_len = (cid_buf.len() + bytes.len()) as u64; + let mut out = Vec::with_capacity(varint::MAX_BYTES + cid_buf.len() + bytes.len()); + varint::write(&mut out, payload_len)?; + out.extend_from_slice(&cid_buf); + out.extend_from_slice(bytes); + Ok(out) +} + +/// Read a single block frame from `r`. Returns `Ok(None)` at clean EOF. +pub fn read_frame(r: &mut R) -> Result)>> { + let payload_len = match varint::read(r)? { + Some(n) => n, + None => return Ok(None), + }; + if payload_len > MAX_BLOCK_LEN { + return Err(Error::BlockTooLarge { + actual: payload_len, + limit: MAX_BLOCK_LEN, + }); + } + let mut buf = vec![0u8; payload_len as usize]; + read_exact(r, &mut buf)?; + + // Slice out the CID prefix. + let mut cursor = std::io::Cursor::new(&buf[..]); + let cid = Cid::read_bytes(&mut cursor).map_err(|e| Error::InvalidCid(e.to_string()))?; + let cid_len = cursor.position() as usize; + if cid_len > buf.len() { + return Err(Error::BlockTooSmall { block: payload_len }); + } + let block_bytes = buf[cid_len..].to_vec(); + + // Validate codec + multihash code (atproto only uses dag-cbor / sha2-256). + if cid.codec() != DAG_CBOR_CODEC { + return Err(Error::UnsupportedCodec(cid.codec())); + } + if cid.hash().code() != SHA2_256_CODE { + return Err(Error::UnsupportedMultihash(cid.hash().code())); + } + + // Verify hash. + let computed = compute_cid_dag_cbor(&block_bytes); + if computed != cid { + return Err(Error::HashMismatch(Box::new(crate::error::HashMismatch { + declared: cid, + computed, + }))); + } + + Ok(Some((cid, block_bytes))) +} + +fn read_exact(r: &mut R, buf: &mut [u8]) -> Result<()> { + let mut filled = 0; + while filled < buf.len() { + let n = r.read(&mut buf[filled..])?; + if n == 0 { + return Err(Error::UnexpectedEof); + } + filled += n; + } + Ok(()) +} diff --git a/reference-impl/rust/star-repo/src/hash.rs b/reference-impl/rust/star-repo/src/hash.rs new file mode 100644 index 0000000..f7d5500 --- /dev/null +++ b/reference-impl/rust/star-repo/src/hash.rs @@ -0,0 +1,21 @@ +//! CID hashing for atproto blocks. +//! +//! Atproto only uses CIDv1 with the dag-cbor codec (`0x71`) and a sha2-256 +//! multihash (`0x12`). These constants and the helper that produces such a +//! CID from raw block bytes live here so they can be shared with anything +//! that needs to verify or mint atproto block CIDs. + +use ipld_core::cid::Cid; +use ipld_core::cid::multihash::Multihash; +use sha2::{Digest, Sha256}; + +pub const DAG_CBOR_CODEC: u64 = 0x71; +pub const SHA2_256_CODE: u64 = 0x12; + +/// Compute the CID (dag-cbor codec, sha2-256 multihash) for a block. +pub fn compute_cid_dag_cbor(bytes: &[u8]) -> Cid { + let digest = Sha256::digest(bytes); + let mh = Multihash::<64>::wrap(SHA2_256_CODE, digest.as_slice()) + .expect("sha2-256 digest is 32 bytes, fits in Multihash<64>"); + Cid::new_v1(DAG_CBOR_CODEC, mh) +} diff --git a/reference-impl/rust/star-repo/src/lib.rs b/reference-impl/rust/star-repo/src/lib.rs new file mode 100644 index 0000000..6c6f8b3 --- /dev/null +++ b/reference-impl/rust/star-repo/src/lib.rs @@ -0,0 +1,29 @@ +//! Helpers for working with atproto repositories: CARv1 framing, the +//! signed-commit object schema, and merkle search tree (MST) construction. +//! +//! Scope: +//! +//! - [`frame`] — CARv1 header + block frame encode/decode +//! - [`blockstore`] — load a whole CAR into a `HashMap>` +//! - [`hash`] — atproto block CIDs (dag-cbor / sha2-256) +//! - [`varint`] — multiformats unsigned varint, thinly wrapping the +//! `unsigned-varint` crate +//! - [`commit`] — atproto v3 signed commit object +//! - [`mst`] — atproto MST node schema and key-layer hash. (Higher-level +//! builder/backend pieces live in the consuming crate.) + +pub mod blockstore; +pub mod commit; +pub mod error; +pub mod frame; +pub mod hash; +pub mod mst; +pub mod varint; + +pub use blockstore::Blockstore; +pub use commit::CommitInfo; +pub use error::{CommitError, Error, MstError, Result}; +pub use frame::{ + CarHeader, MAX_BLOCK_LEN, build_frame, read_frame, read_header, write_frame, write_header, +}; +pub use hash::{DAG_CBOR_CODEC, SHA2_256_CODE, compute_cid_dag_cbor}; diff --git a/reference-impl/rust/star-repo/src/mst/key.rs b/reference-impl/rust/star-repo/src/mst/key.rs new file mode 100644 index 0000000..d61ee16 --- /dev/null +++ b/reference-impl/rust/star-repo/src/mst/key.rs @@ -0,0 +1,68 @@ +//! Atproto MST key layer. +//! +//! Per [draft-holmgren-at-repository §2.5.2][spec], the layer for a key is +//! computed as `floor(leading_zero_bits(sha256(key)) / 2)`. The 2-bit grouping +//! gives an average fanout of 4: ~3/4 of keys land at layer 0 (top two hash +//! bits not both zero), ~3/16 at layer 1, ~3/64 at layer 2, etc. +//! +//! Importantly, the layer of a key is a property of the key alone, not the +//! tree state. When building from sorted input we use this to decide which +//! level each key belongs to without any tree lookups. +//! +//! [spec]: https://www.ietf.org/archive/id/draft-holmgren-at-repository-01.txt + +use sha2::{Digest, Sha256}; + +/// Compute the MST layer for a key. +pub fn layer_of(key_bytes: &[u8]) -> u32 { + let hash = Sha256::digest(key_bytes); + let mut leading_zero_bits: u32 = 0; + for &byte in hash.iter() { + if byte == 0 { + leading_zero_bits += 8; + } else { + leading_zero_bits += byte.leading_zeros(); + break; + } + } + leading_zero_bits / 2 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn deterministic() { + assert_eq!( + layer_of(b"app.bsky.feed.post/abc"), + layer_of(b"app.bsky.feed.post/abc") + ); + } + + #[test] + fn distribution_matches_fanout_4() { + // With fanout 4 we expect ~75% of keys at layer 0. + let mut at_layer_0 = 0; + let n = 10_000; + for i in 0..n { + let s = format!("k-{i:08}"); + if layer_of(s.as_bytes()) == 0 { + at_layer_0 += 1; + } + } + let pct = (at_layer_0 as f64) / (n as f64); + assert!( + pct > 0.70 && pct < 0.80, + "expected ~0.75 of keys at layer 0, got {pct}" + ); + } + + #[test] + fn known_examples_from_spec() { + // From §2.5.2 examples — we can't replicate the exact "key1/key7/key515" + // bindings since the spec doesn't say what hash inputs they use, but + // we can sanity-check the function on synthetic hashes. + // (skip — covered indirectly by distribution test) + } +} diff --git a/reference-impl/rust/star-repo/src/mst/mod.rs b/reference-impl/rust/star-repo/src/mst/mod.rs new file mode 100644 index 0000000..1f3baab --- /dev/null +++ b/reference-impl/rust/star-repo/src/mst/mod.rs @@ -0,0 +1,7 @@ +//! Atproto MST schema and key-layer hash. + +pub mod key; +pub mod node; + +pub use key::layer_of; +pub use node::{MstEntry, MstNode, decode_node, encode_node}; diff --git a/reference-impl/rust/star-repo/src/mst/node.rs b/reference-impl/rust/star-repo/src/mst/node.rs new file mode 100644 index 0000000..42e219a --- /dev/null +++ b/reference-impl/rust/star-repo/src/mst/node.rs @@ -0,0 +1,81 @@ +//! MST node DAG-CBOR schema and encoding. +//! +//! Per [draft-holmgren-at-repository §2.5.5][spec], MST nodes have: +//! +//! ```text +//! { +//! l: Cid | null // left subtree at lower layer +//! e: [ +//! { +//! p: uint // bytes shared with previous key in this node +//! , k: bytes // suffix +//! , v: Cid // value link +//! , t: Cid | null // subtree to the right of this leaf +//! } +//! ... +//! ] +//! } +//! ``` +//! +//! Both `l` and `t` are *nullable*, not optional: they are always present in +//! the encoded map, encoded as CBOR null when there is no subtree. Omitting +//! them would produce a different content hash and break interoperability. +//! +//! Prefix compression resets at each node: the first entry has `p = 0` and +//! `k` holding the full key. Subsequent entries reference only the previous +//! entry within the *same node*. +//! +//! [spec]: https://www.ietf.org/archive/id/draft-holmgren-at-repository-01.txt + +use ipld_core::cid::Cid; +use serde::{Deserialize, Serialize}; + +use crate::error::{MstError, Result}; +use crate::hash::compute_cid_dag_cbor; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MstNode { + /// Leaf entries in this node, sorted by key. + #[serde(rename = "e")] + pub entries: Vec, + /// Leftmost subtree (keys lex-less than the first entry). Encoded as + /// `null` (not omitted) when absent. + #[serde(rename = "l", default)] + pub left: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MstEntry { + /// Suffix of this key (the bytes after the shared prefix). + #[serde(rename = "k")] + pub key_suffix: serde_bytes::ByteBuf, + /// Number of bytes shared with the previous key in this node. + #[serde(rename = "p")] + pub prefix_len: u32, + /// Subtree to the right of this leaf (keys lex-between this key and the + /// next entry's key). Encoded as `null` when absent. + #[serde(rename = "t", default)] + pub right: Option, + /// CID of the value associated with this key. + #[serde(rename = "v")] + pub value: Cid, +} + +/// Encode an MST node to canonical DAG-CBOR and compute its CID. +pub fn encode_node(node: &MstNode) -> Result<(Cid, Vec)> { + let bytes = + serde_ipld_dagcbor::to_vec(node).map_err(|e| MstError::EncodingFailed(e.to_string()))?; + let cid = compute_cid_dag_cbor(&bytes); + Ok((cid, bytes)) +} + +/// Decode an MST node from DAG-CBOR. +pub fn decode_node(bytes: &[u8]) -> Result { + serde_ipld_dagcbor::from_slice(bytes) + .map_err(|e| MstError::EncodingFailed(format!("decode: {e}")).into()) +} + +/// Length (in bytes) of the longest prefix common to `a` and `b`. +pub fn shared_prefix_len(a: &[u8], b: &[u8]) -> usize { + a.iter().zip(b.iter()).take_while(|(x, y)| x == y).count() +} diff --git a/reference-impl/rust/star-repo/src/varint.rs b/reference-impl/rust/star-repo/src/varint.rs new file mode 100644 index 0000000..0c9bfcd --- /dev/null +++ b/reference-impl/rust/star-repo/src/varint.rs @@ -0,0 +1,103 @@ +//! Unsigned varint codec — thin wrapper around `unsigned-varint`. +//! +//! The wrapper layer is small but pulls its weight: it surfaces our own +//! [`Error`] type and preserves the streaming-EOF distinction we need +//! (clean EOF before any byte is read returns `Ok(None)`; EOF mid-varint +//! is an error). + +use std::io::{Read, Write}; + +use crate::error::{Error, Result}; + +/// Maximum number of bytes a u64 varint can occupy per the multiformats +/// unsigned-varint spec. +pub const MAX_BYTES: usize = 10; + +/// Write `value` as a varint to `w`. +pub fn write(w: &mut W, value: u64) -> Result<()> { + let mut buf = unsigned_varint::encode::u64_buffer(); + let bytes = unsigned_varint::encode::u64(value, &mut buf); + w.write_all(bytes)?; + Ok(()) +} + +/// Read a varint from `r`. Returns `Ok(None)` at clean EOF (zero bytes +/// consumed); `Err(UnexpectedEof)` if EOF strikes mid-varint. +pub fn read(r: &mut R) -> Result> { + let mut probe = [0u8; 1]; + if r.read(&mut probe)? == 0 { + return Ok(None); + } + let mut chained = Read::chain(&probe[..], r); + let value = unsigned_varint::io::read_u64(&mut chained).map_err(map_read_err)?; + Ok(Some(value)) +} + +/// Read a varint from `r`; treat any EOF (clean or mid-varint) as an error. +pub fn read_required(r: &mut R) -> Result { + read(r)?.ok_or(Error::UnexpectedEof) +} + +fn map_read_err(e: unsigned_varint::io::ReadError) -> Error { + match e { + unsigned_varint::io::ReadError::Io(io_err) => { + if io_err.kind() == std::io::ErrorKind::UnexpectedEof { + Error::UnexpectedEof + } else { + Error::Io(io_err) + } + } + // Decode failures (overlong / non-minimal): collapse to InvalidVarint. + _ => Error::InvalidVarint, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + fn roundtrip(value: u64) { + let mut buf = Vec::new(); + write(&mut buf, value).unwrap(); + assert_eq!(read(&mut Cursor::new(&buf)).unwrap().unwrap(), value); + } + + #[test] + fn small_values() { + roundtrip(0); + roundtrip(127); + roundtrip(128); + roundtrip(16_383); + roundtrip(16_384); + } + + #[test] + fn max_value() { + roundtrip(u64::MAX); + } + + #[test] + fn rejects_overlong() { + let bad = [0x80u8; 11]; + assert!(matches!( + read(&mut Cursor::new(&bad[..])), + Err(Error::InvalidVarint) + )); + } + + #[test] + fn clean_eof() { + let mut cursor = Cursor::new(&[][..]); + assert!(matches!(read(&mut cursor), Ok(None))); + } + + #[test] + fn mid_varint_eof() { + let bad = [0x80u8]; + assert!(matches!( + read(&mut Cursor::new(&bad[..])), + Err(Error::UnexpectedEof) + )); + } +}