From e4ffd7d49d30b6166b929c471fe46d9311cccb78 Mon Sep 17 00:00:00 2001 From: Lewis Date: Mon, 4 May 2026 21:05:14 +0300 Subject: [PATCH] feat(search): tantivy SearchIndex Lewis: May this revision serve well! --- Cargo.lock | 761 ++++++++++++++++++++++++++++++++++++++- Cargo.toml | 4 + crates/search/Cargo.toml | 18 + crates/search/src/lib.rs | 645 +++++++++++++++++++++++++++++++++ 4 files changed, 1425 insertions(+), 3 deletions(-) create mode 100644 crates/search/Cargo.toml create mode 100644 crates/search/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 5431e73..7577dcd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -57,6 +57,15 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arc-swap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +dependencies = [ + "rustversion", +] + [[package]] name = "assert-json-diff" version = "2.0.2" @@ -79,6 +88,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atomic-polyfill" version = "1.0.3" @@ -192,6 +212,15 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +[[package]] +name = "bitpacking" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96a7139abd3d9cebf8cd6f920a389cf3dc9576172e32f4563f188cae3c3eb019" +dependencies = [ + "crunchy", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -211,6 +240,7 @@ dependencies = [ "bobbin-ingest", "bobbin-knot-proxy", "bobbin-record-lru", + "bobbin-search", "bobbin-slingshot-client", "bobbin-types", "bobbin-xrpc", @@ -241,6 +271,7 @@ name = "bobbin-ingest" version = "0.0.1" dependencies = [ "bobbin-edge-index", + "bobbin-record-lru", "bobbin-types", "futures", "jacquard-common", @@ -279,6 +310,18 @@ dependencies = [ "quick_cache", ] +[[package]] +name = "bobbin-search" +version = "0.0.1" +dependencies = [ + "bobbin-types", + "jacquard-common", + "tantivy", + "thiserror 2.0.18", + "tokio", + "tracing", +] + [[package]] name = "bobbin-slingshot-client" version = "0.0.1" @@ -323,6 +366,7 @@ dependencies = [ "bobbin-edge-index", "bobbin-knot-proxy", "bobbin-record-lru", + "bobbin-search", "bobbin-slingshot-client", "bobbin-types", "futures", @@ -421,9 +465,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] +[[package]] +name = "census" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0" + [[package]] name = "cfg-if" version = "1.0.4" @@ -588,6 +640,34 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -696,6 +776,12 @@ dependencies = [ "syn", ] +[[package]] +name = "datasketches" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745" + [[package]] name = "deadpool" version = "0.12.3" @@ -732,6 +818,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ "powerfmt", + "serde_core", ] [[package]] @@ -784,6 +871,12 @@ dependencies = [ "syn", ] +[[package]] +name = "downcast-rs" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" + [[package]] name = "ecdsa" version = "0.16.9" @@ -798,6 +891,12 @@ dependencies = [ "spki", ] +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + [[package]] name = "elliptic-curve" version = "0.13.8" @@ -845,6 +944,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "erased-serde" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + [[package]] name = "errno" version = "0.3.14" @@ -855,6 +965,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "fastdivide" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471" + [[package]] name = "fastrand" version = "2.4.1" @@ -910,6 +1026,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -919,6 +1041,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs4" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" +dependencies = [ + "rustix", + "windows-sys 0.59.0", +] + [[package]] name = "futures" version = "0.3.32" @@ -1081,11 +1213,24 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + [[package]] name = "group" version = "0.13.0" @@ -1154,7 +1299,7 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.1.5", ] [[package]] @@ -1162,6 +1307,11 @@ name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "hashbrown" @@ -1216,6 +1366,12 @@ dependencies = [ "digest", ] +[[package]] +name = "htmlescape" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" + [[package]] name = "http" version = "1.4.0" @@ -1430,6 +1586,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + [[package]] name = "ident_case" version = "1.0.1" @@ -1465,6 +1627,8 @@ checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", "hashbrown 0.17.0", + "serde", + "serde_core", ] [[package]] @@ -1503,6 +1667,15 @@ dependencies = [ "serde", ] +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -1603,6 +1776,16 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + [[package]] name = "js-sys" version = "0.3.97" @@ -1643,12 +1826,30 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "levenshtein_automata" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" + [[package]] name = "libc" version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "litemap" version = "0.8.2" @@ -1683,12 +1884,27 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "lru" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" +dependencies = [ + "hashbrown 0.16.1", +] + [[package]] name = "lru-slab" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "lz4_flex" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a" + [[package]] name = "maitake-sync" version = "0.1.2" @@ -1728,12 +1944,30 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "measure_time" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51c55d61e72fc3ab704396c5fa16f4c184db37978ae4e94ca8959693a235fc0e" +dependencies = [ + "log", +] + [[package]] name = "memchr" version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +dependencies = [ + "libc", +] + [[package]] name = "miette" version = "7.6.0" @@ -1762,6 +1996,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -1805,6 +2045,12 @@ dependencies = [ "unsigned-varint", ] +[[package]] +name = "murmurhash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b" + [[package]] name = "mycelium-bitfield" version = "0.1.5" @@ -1832,6 +2078,16 @@ dependencies = [ "web-time", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1872,12 +2128,27 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "oneshot" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" + [[package]] name = "openssl-probe" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "ordered-float" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e" +dependencies = [ + "num-traits", +] + [[package]] name = "ouroboros" version = "0.18.5" @@ -1902,6 +2173,15 @@ dependencies = [ "syn", ] +[[package]] +name = "ownedbytes" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fbd56f7631767e61784dc43f8580f403f4475bd4aaa4da003e6295e1bab4a7e" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "oxilangtag" version = "0.1.5" @@ -2045,6 +2325,12 @@ dependencies = [ "spki", ] +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + [[package]] name = "portable-atomic" version = "1.13.1" @@ -2211,6 +2497,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "rand" version = "0.8.6" @@ -2270,6 +2562,26 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -2412,6 +2724,16 @@ dependencies = [ "byteorder", ] +[[package]] +name = "rust-stemmers" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" +dependencies = [ + "serde", + "serde_derive", +] + [[package]] name = "rustc-hash" version = "2.1.2" @@ -2427,6 +2749,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustls" version = "0.23.40" @@ -2797,6 +3132,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" +[[package]] +name = "sketches-ddsketch" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05e40b6cf54d988dc1a2223531b969c9a9e30906ad90ef64890c27b4bfbb46ea" +dependencies = [ + "serde", +] + [[package]] name = "slab" version = "0.4.12" @@ -2930,6 +3274,167 @@ dependencies = [ "libc", ] +[[package]] +name = "tantivy" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edde6a10743fff00a4e1a8c9ef020bf5f3cbad301b7d2d39f2b07f123c4eac07" +dependencies = [ + "aho-corasick", + "arc-swap", + "base64", + "bitpacking", + "bon", + "byteorder", + "census", + "crc32fast", + "crossbeam-channel", + "datasketches", + "downcast-rs", + "fastdivide", + "fnv", + "fs4", + "htmlescape", + "itertools", + "levenshtein_automata", + "log", + "lru", + "lz4_flex", + "measure_time", + "memmap2", + "once_cell", + "oneshot", + "rayon", + "regex", + "rust-stemmers", + "rustc-hash", + "serde", + "serde_json", + "sketches-ddsketch", + "smallvec", + "tantivy-bitpacker", + "tantivy-columnar", + "tantivy-common", + "tantivy-fst", + "tantivy-query-grammar", + "tantivy-stacker", + "tantivy-tokenizer-api", + "tempfile", + "thiserror 2.0.18", + "time", + "typetag", + "uuid", + "winapi", +] + +[[package]] +name = "tantivy-bitpacker" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fed3d674429bcd2de5d0a6d1aa5495fed8afd9c5ecce993019caf7615f53fa4" +dependencies = [ + "bitpacking", +] + +[[package]] +name = "tantivy-columnar" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c57166f5bcfd478f370ab8445afb4678dce44801fa5ce5c451aaf8595583c5dc" +dependencies = [ + "downcast-rs", + "fastdivide", + "itertools", + "serde", + "tantivy-bitpacker", + "tantivy-common", + "tantivy-sstable", + "tantivy-stacker", +] + +[[package]] +name = "tantivy-common" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbf10915aa75da3c3b0d58b58853d2e889efbaf32d4982a4c3715dde6bba23e5" +dependencies = [ + "async-trait", + "byteorder", + "ownedbytes", + "serde", + "time", +] + +[[package]] +name = "tantivy-fst" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18" +dependencies = [ + "byteorder", + "regex-syntax", + "utf8-ranges", +] + +[[package]] +name = "tantivy-query-grammar" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfadb8526b6da90704feb293b0701a6aae62ea14983143344be2dc5ce30f1d82" +dependencies = [ + "fnv", + "nom", + "ordered-float", + "serde", + "serde_json", +] + +[[package]] +name = "tantivy-sstable" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a2cfc3ac5164cbadc28965ffb145a8f47582a60ae5897859ad8d4316596c606" +dependencies = [ + "futures-util", + "itertools", + "tantivy-bitpacker", + "tantivy-common", + "tantivy-fst", + "zstd", +] + +[[package]] +name = "tantivy-stacker" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cbb051742da9d53ca9e8fff43a9b10e319338b24e2c0e15d0372df19ffeb951" +dependencies = [ + "murmurhash32", + "tantivy-common", +] + +[[package]] +name = "tantivy-tokenizer-api" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac258c2c6390673f2685813afeeafcb8c4e0ee7de8dd3fc46838dcc37263f98" +dependencies = [ + "serde", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -2986,10 +3491,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ "deranged", + "itoa", "num-conv", "powerfmt", "serde_core", "time-core", + "time-macros", ] [[package]] @@ -2998,6 +3505,16 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" version = "0.8.3" @@ -3293,12 +3810,42 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + [[package]] name = "typenum" version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" +[[package]] +name = "typetag" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be2212c8a9b9bcfca32024de14998494cf9a5dfa59ea1b829de98bac374b86bf" +dependencies = [ + "erased-serde", + "inventory", + "once_cell", + "serde", + "typetag-impl", +] + +[[package]] +name = "typetag-impl" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27a7a9b72ba121f6f1f6c3632b85604cac41aedb5ddc70accbebb6cac83de846" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "unicode-ident" version = "1.0.24" @@ -3353,12 +3900,30 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf8-ranges" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" + [[package]] name = "utf8_iter" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "uuid" +version = "1.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" +dependencies = [ + "getrandom 0.4.2", + "js-sys", + "serde_core", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" @@ -3402,7 +3967,16 @@ version = "1.0.3+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", ] [[package]] @@ -3460,6 +4034,28 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + [[package]] name = "wasm-streams" version = "0.4.2" @@ -3473,6 +4069,18 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + [[package]] name = "web-sys" version = "0.3.97" @@ -3511,6 +4119,22 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" @@ -3520,6 +4144,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.62.2" @@ -3599,6 +4229,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.60.2" @@ -3769,12 +4408,100 @@ dependencies = [ "url", ] +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + [[package]] name = "wit-bindgen" version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck 0.5.0", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck 0.5.0", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + [[package]] name = "writeable" version = "0.6.3" @@ -3895,3 +4622,31 @@ name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index 16a4797..4e5bd58 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "crates/slingshot-client", "crates/record-lru", "crates/knot-proxy", + "crates/search", "crates/xrpc", ] @@ -24,6 +25,7 @@ bobbin-ingest = { path = "crates/ingest" } bobbin-slingshot-client = { path = "crates/slingshot-client" } bobbin-record-lru = { path = "crates/record-lru" } bobbin-knot-proxy = { path = "crates/knot-proxy" } +bobbin-search = { path = "crates/search" } bobbin-xrpc = { path = "crates/xrpc" } jacquard-common = "0.12.0-beta.2" @@ -59,6 +61,8 @@ http = "1" wiremock = "0.6" +tantivy = "0.26" + tracing = "0.1" thiserror = "2" diff --git a/crates/search/Cargo.toml b/crates/search/Cargo.toml new file mode 100644 index 0000000..3c0e9ac --- /dev/null +++ b/crates/search/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "bobbin-search" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +bobbin-types = { workspace = true } +jacquard-common = { workspace = true } + +tantivy = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["rt", "sync", "time"] } +tracing = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/crates/search/src/lib.rs b/crates/search/src/lib.rs new file mode 100644 index 0000000..e5240c5 --- /dev/null +++ b/crates/search/src/lib.rs @@ -0,0 +1,645 @@ +use std::sync::Arc; +use std::time::Duration; + +use bobbin_types::search::{SearchDoc, SearchSink}; +use jacquard_common::DefaultStr; +use jacquard_common::types::nsid::Nsid; +use jacquard_common::types::string::AtUri; +use tantivy::collector::TopDocs; +use tantivy::query::{BooleanQuery, Occur, Query, QueryParser, QueryParserError, TermQuery}; +use tantivy::schema::{ + Field, IndexRecordOption, STORED, STRING, Schema, TEXT, TextFieldIndexing, TextOptions, Value, +}; +use tantivy::{Index, IndexReader, IndexWriter, ReloadPolicy, TantivyDocument, TantivyError, Term}; +use thiserror::Error; +use tokio::sync::{mpsc, oneshot}; +use tokio::time::timeout; +use tracing::warn; + +pub const DEFAULT_WRITER_HEAP_BYTES: usize = 50_000_000; +const BATCH_SIZE: usize = 200; +const BATCH_INTERVAL: Duration = Duration::from_millis(250); +const TITLE_BOOST: f32 = 4.0; +const OFFSET_TOKEN_LEN: usize = 8; +const WRITE_QUEUE_CAPACITY: usize = 4096; + +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub struct SearchOffset(u32); + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Error)] +pub enum SearchOffsetError { + #[error("offset token must be {OFFSET_TOKEN_LEN} hex characters")] + Malformed, +} + +impl SearchOffset { + pub const fn new(value: u32) -> Self { + Self(value) + } + + pub const fn raw(self) -> u32 { + self.0 + } + + pub fn encode_token(self) -> String { + format!("{:0width$x}", self.0, width = OFFSET_TOKEN_LEN) + } + + pub fn decode_token(token: &str) -> Result { + if token.len() != OFFSET_TOKEN_LEN || !token.bytes().all(|b| b.is_ascii_hexdigit()) { + return Err(SearchOffsetError::Malformed); + } + u32::from_str_radix(token, 16) + .map(Self) + .map_err(|_| SearchOffsetError::Malformed) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SearchCursor { + Start, + At(SearchOffset), +} + +impl SearchCursor { + pub fn from_token(raw: Option<&str>) -> Result { + raw.map_or(Ok(Self::Start), |t| { + SearchOffset::decode_token(t).map(Self::At) + }) + } + + fn offset(self) -> u32 { + match self { + Self::Start => 0, + Self::At(o) => o.raw(), + } + } +} + +#[derive(Debug)] +pub struct SearchHit { + pub uri: AtUri, + pub nsid: Nsid, + pub score: f32, +} + +#[derive(Debug)] +pub struct SearchPage { + pub hits: Vec, + pub next: Option, +} + +#[derive(Debug, Error)] +pub enum SearchError { + #[error("tantivy: {0}")] + Tantivy(#[from] TantivyError), + #[error("query parse: {0}")] + Query(#[from] QueryParserError), + #[error("indexed uri is invalid: {0}")] + InvalidUri(String), + #[error("indexed nsid is invalid: {0}")] + InvalidNsid(String), + #[error("indexed document missing field: {0}")] + MissingField(&'static str), + #[error("failed to spawn search writer thread: {0}")] + ThreadSpawn(String), + #[error("search blocking task cancelled: {0}")] + Cancelled(String), +} + +#[derive(Clone, Copy)] +struct Fields { + uri: Field, + nsid: Field, + title: Field, + body: Field, +} + +enum WriteOp { + Upsert(SearchDoc), + Remove(AtUri), + Flush(oneshot::Sender<()>), +} + +pub struct SearchIndex { + inner: Arc, +} + +struct Inner { + fields: Fields, + reader: IndexReader, + parser: QueryParser, + tx: mpsc::Sender, +} + +impl SearchIndex { + pub fn new(heap_bytes: usize) -> Result { + let mut sb = Schema::builder(); + let uri = sb.add_text_field("uri", STRING | STORED); + let nsid = sb.add_text_field("nsid", STRING | STORED); + let title_indexing = TextFieldIndexing::default() + .set_tokenizer("default") + .set_index_option(IndexRecordOption::WithFreqsAndPositions); + let title_opts = TextOptions::default() + .set_indexing_options(title_indexing) + .set_stored(); + let title = sb.add_text_field("title", title_opts); + let body = sb.add_text_field("body", TEXT); + let schema = sb.build(); + let index = Index::create_in_ram(schema); + let writer: IndexWriter = index.writer(heap_bytes)?; + let reader = index + .reader_builder() + .reload_policy(ReloadPolicy::Manual) + .try_into()?; + let mut parser = QueryParser::for_index(&index, vec![title, body]); + parser.set_field_boost(title, TITLE_BOOST); + let fields = Fields { + uri, + nsid, + title, + body, + }; + let (tx, rx) = mpsc::channel(WRITE_QUEUE_CAPACITY); + let writer_reader = reader.clone(); + std::thread::Builder::new() + .name("bobbin-search-writer".into()) + .spawn(move || run_writer_thread(writer, fields, writer_reader, rx)) + .map_err(|e| SearchError::ThreadSpawn(e.to_string()))?; + Ok(Self { + inner: Arc::new(Inner { + fields, + reader, + parser, + tx, + }), + }) + } + + pub async fn flush(&self) { + let (done_tx, done_rx) = oneshot::channel(); + if self.inner.tx.send(WriteOp::Flush(done_tx)).await.is_err() { + warn!("search writer channel closed before flush"); + return; + } + if done_rx.await.is_err() { + warn!("search writer dropped before completing flush"); + } + } + + pub async fn search( + &self, + q: &str, + nsid_filter: Option<&Nsid>, + cursor: SearchCursor, + limit: u32, + ) -> Result { + let trimmed = q.trim(); + if trimmed.is_empty() { + return Ok(SearchPage { + hits: Vec::new(), + next: None, + }); + } + let inner = self.inner.clone(); + let q_owned = trimmed.to_owned(); + let nsid_owned = nsid_filter.cloned(); + let limit_usize = limit as usize; + let offset = cursor.offset() as usize; + match tokio::task::spawn_blocking(move || { + inner.search_blocking(&q_owned, nsid_owned.as_ref(), offset, limit_usize) + }) + .await + { + Ok(result) => result, + Err(e) if e.is_panic() => std::panic::resume_unwind(e.into_panic()), + Err(e) => Err(SearchError::Cancelled(e.to_string())), + } + } +} + +impl Inner { + fn search_blocking( + &self, + q: &str, + nsid_filter: Option<&Nsid>, + offset: usize, + limit: usize, + ) -> Result { + let parsed = self.parser.parse_query(q)?; + let final_query: Box = match nsid_filter { + None => parsed, + Some(n) => { + let term = Term::from_field_text(self.fields.nsid, n.as_ref()); + Box::new(BooleanQuery::new(vec![ + (Occur::Must, parsed), + ( + Occur::Must, + Box::new(TermQuery::new(term, IndexRecordOption::Basic)), + ), + ])) + } + }; + let collector = TopDocs::with_limit(limit + 1) + .and_offset(offset) + .order_by_score(); + let searcher = self.reader.searcher(); + let raw_hits: Vec<(f32, tantivy::DocAddress)> = + searcher.search(&final_query, &collector)?; + let has_more = raw_hits.len() > limit; + let page_slice = &raw_hits[..raw_hits.len().min(limit)]; + let hits = page_slice + .iter() + .map(|(score, addr)| { + let doc: TantivyDocument = searcher.doc(*addr)?; + let uri_str = stored_text(&doc, self.fields.uri, "uri")?; + let nsid_str = stored_text(&doc, self.fields.nsid, "nsid")?; + let uri = AtUri::::new_owned(uri_str.as_str()) + .map_err(|e| SearchError::InvalidUri(format!("{e}: {uri_str}")))?; + let nsid = Nsid::::new_owned(nsid_str.as_str()) + .map_err(|e| SearchError::InvalidNsid(format!("{e}: {nsid_str}")))?; + Ok::<_, SearchError>(SearchHit { + uri, + nsid, + score: *score, + }) + }) + .collect::, _>>()?; + Ok(SearchPage { + hits, + next: next_cursor(offset, limit, has_more), + }) + } +} + +fn next_cursor(offset: usize, limit: usize, has_more: bool) -> Option { + has_more + .then(|| u32::try_from(offset.saturating_add(limit)).ok()) + .flatten() + .map(SearchOffset::new) +} + +fn run_writer_thread( + writer: IndexWriter, + fields: Fields, + reader: IndexReader, + rx: mpsc::Receiver, +) { + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + { + Ok(rt) => rt, + Err(e) => { + warn!(?e, "search writer runtime build failed"); + return; + } + }; + runtime.block_on(writer_loop(writer, fields, reader, rx)); +} + +async fn writer_loop( + mut writer: IndexWriter, + fields: Fields, + reader: IndexReader, + mut rx: mpsc::Receiver, +) { + let mut pending: usize = 0; + loop { + match timeout(BATCH_INTERVAL, rx.recv()).await { + Ok(Some(WriteOp::Upsert(doc))) => { + apply_upsert(&mut writer, fields, doc); + pending += 1; + } + Ok(Some(WriteOp::Remove(uri))) => { + apply_remove(&mut writer, fields, &uri); + pending += 1; + } + Ok(Some(WriteOp::Flush(done))) => { + if pending > 0 { + commit_and_reload(&mut writer, &reader); + pending = 0; + } + let _ = done.send(()); + continue; + } + Ok(None) => break, + Err(_) => { + if pending > 0 { + commit_and_reload(&mut writer, &reader); + pending = 0; + } + continue; + } + } + if pending >= BATCH_SIZE { + commit_and_reload(&mut writer, &reader); + pending = 0; + } + } + if pending > 0 { + commit_and_reload(&mut writer, &reader); + } +} + +fn apply_upsert(writer: &mut IndexWriter, fields: Fields, doc: SearchDoc) { + let uri_term = Term::from_field_text(fields.uri, doc.uri.as_ref()); + writer.delete_term(uri_term); + let mut td = TantivyDocument::default(); + td.add_text(fields.uri, doc.uri.as_ref()); + td.add_text(fields.nsid, doc.nsid.as_ref()); + td.add_text(fields.title, &doc.title); + td.add_text(fields.body, &doc.body); + if let Err(e) = writer.add_document(td) { + warn!(?e, "search add_document failed"); + } +} + +fn apply_remove(writer: &mut IndexWriter, fields: Fields, uri: &AtUri) { + let term = Term::from_field_text(fields.uri, uri.as_ref()); + writer.delete_term(term); +} + +fn commit_and_reload(writer: &mut IndexWriter, reader: &IndexReader) { + match writer.commit() { + Ok(_) => { + if let Err(e) = reader.reload() { + warn!(?e, "search reader reload failed"); + } + } + Err(e) => warn!(?e, "search index commit failed"), + } +} + +fn stored_text( + doc: &TantivyDocument, + field: Field, + name: &'static str, +) -> Result { + doc.get_first(field) + .and_then(|v| v.as_str().map(|s| s.to_owned())) + .ok_or(SearchError::MissingField(name)) +} + +impl SearchSink for SearchIndex { + async fn upsert(&self, doc: SearchDoc) { + if self.inner.tx.send(WriteOp::Upsert(doc)).await.is_err() { + warn!("search writer channel closed; upsert dropped"); + } + } + + async fn remove(&self, uri: &AtUri) { + if self.inner.tx.send(WriteOp::Remove(uri.clone())).await.is_err() { + warn!("search writer channel closed; remove dropped"); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use bobbin_types::search::SearchDoc; + use jacquard_common::types::nsid::Nsid as NsidType; + + fn at(s: &str) -> AtUri { + AtUri::new_owned(s).unwrap() + } + + fn nsid(s: &'static str) -> NsidType { + NsidType::new_static(s).unwrap() + } + + fn doc(uri: &str, nsid_s: &'static str, title: &str, body: &str) -> SearchDoc { + SearchDoc { + uri: at(uri), + nsid: nsid(nsid_s), + title: title.to_owned(), + body: body.to_owned(), + } + } + + fn build() -> SearchIndex { + SearchIndex::new(DEFAULT_WRITER_HEAP_BYTES).unwrap() + } + + #[tokio::test] + async fn upsert_then_query_returns_matching_doc() { + let idx = build(); + idx.upsert(doc( + "at://did:plc:nel/sh.tangled.repo.issue/r1", + "sh.tangled.repo.issue", + "barnacle pagination", + "scroll resets", + )) + .await; + idx.upsert(doc( + "at://did:plc:teq/sh.tangled.repo.issue/r2", + "sh.tangled.repo.issue", + "kelp grew sideways", + "kelp", + )) + .await; + idx.flush().await; + + let page = idx + .search("barnacle", None, SearchCursor::Start, 10) + .await + .unwrap(); + assert_eq!(page.hits.len(), 1); + assert_eq!( + page.hits[0].uri.as_ref(), + "at://did:plc:nel/sh.tangled.repo.issue/r1" + ); + assert!(page.next.is_none()); + } + + #[tokio::test] + async fn query_filters_by_nsid() { + let idx = build(); + idx.upsert(doc( + "at://did:plc:nel/sh.tangled.repo.issue/r1", + "sh.tangled.repo.issue", + "anemone tide", + "", + )) + .await; + idx.upsert(doc( + "at://did:plc:teq/sh.tangled.string/k1", + "sh.tangled.string", + "anemone.md", + "anemone again", + )) + .await; + idx.flush().await; + + let only_strings = idx + .search( + "anemone", + Some(&nsid("sh.tangled.string")), + SearchCursor::Start, + 10, + ) + .await + .unwrap(); + assert_eq!(only_strings.hits.len(), 1); + assert_eq!(only_strings.hits[0].nsid.as_ref(), "sh.tangled.string"); + } + + #[tokio::test] + async fn upsert_replaces_prior_document_for_same_uri() { + let idx = build(); + let uri = "at://did:plc:nel/sh.tangled.repo.issue/r1"; + idx.upsert(doc(uri, "sh.tangled.repo.issue", "abalone", "")) + .await; + idx.upsert(doc(uri, "sh.tangled.repo.issue", "limpet", "")) + .await; + idx.flush().await; + + let abalone_hits = idx + .search("abalone", None, SearchCursor::Start, 10) + .await + .unwrap(); + assert!(abalone_hits.hits.is_empty(), "old title must be evicted"); + + let limpet_hits = idx + .search("limpet", None, SearchCursor::Start, 10) + .await + .unwrap(); + assert_eq!(limpet_hits.hits.len(), 1); + } + + #[tokio::test] + async fn remove_drops_document_from_index() { + let idx = build(); + let uri = at("at://did:plc:nel/sh.tangled.repo.issue/r1"); + idx.upsert(doc(uri.as_ref(), "sh.tangled.repo.issue", "whelk", "shell")) + .await; + idx.remove(&uri).await; + idx.flush().await; + let hits = idx + .search("whelk", None, SearchCursor::Start, 10) + .await + .unwrap(); + assert!(hits.hits.is_empty()); + } + + #[tokio::test] + async fn pagination_advances_cursor() { + let idx = build(); + let names = ["nel", "olaren", "teq", "lyna", "bailey"]; + for (i, owner) in names.iter().enumerate() { + idx.upsert(doc( + &format!("at://did:plc:{owner}/sh.tangled.repo.issue/r{i}"), + "sh.tangled.repo.issue", + "anemone", + "tides", + )) + .await; + } + idx.flush().await; + + let page1 = idx + .search("anemone", None, SearchCursor::Start, 2) + .await + .unwrap(); + assert_eq!(page1.hits.len(), 2); + let next = page1.next.expect("more pages"); + let page2 = idx + .search("anemone", None, SearchCursor::At(next), 2) + .await + .unwrap(); + assert_eq!(page2.hits.len(), 2); + let next2 = page2.next.expect("more pages"); + let page3 = idx + .search("anemone", None, SearchCursor::At(next2), 2) + .await + .unwrap(); + assert_eq!(page3.hits.len(), 1); + assert!(page3.next.is_none()); + } + + #[tokio::test] + async fn empty_query_short_circuits_to_empty_page() { + let idx = build(); + idx.upsert(doc( + "at://did:plc:nel/sh.tangled.repo.issue/r1", + "sh.tangled.repo.issue", + "abalone", + "", + )) + .await; + idx.flush().await; + let page = idx + .search(" ", None, SearchCursor::Start, 10) + .await + .unwrap(); + assert!(page.hits.is_empty()); + assert!(page.next.is_none()); + } + + #[tokio::test] + async fn flush_without_pending_ops_completes() { + let idx = build(); + idx.flush().await; + } + + #[tokio::test] + async fn writer_auto_commits_on_idle_interval() { + let idx = build(); + idx.upsert(doc( + "at://did:plc:nel/sh.tangled.repo.issue/r1", + "sh.tangled.repo.issue", + "auto", + "", + )) + .await; + tokio::time::sleep(BATCH_INTERVAL * 3).await; + let page = idx + .search("auto", None, SearchCursor::Start, 10) + .await + .unwrap(); + assert_eq!(page.hits.len(), 1); + } + + #[test] + fn next_cursor_advances_by_limit_when_more_results() { + assert_eq!(next_cursor(0, 10, true), Some(SearchOffset::new(10))); + assert_eq!(next_cursor(10, 25, true), Some(SearchOffset::new(35))); + } + + #[test] + fn next_cursor_is_none_when_no_more_results() { + assert_eq!(next_cursor(0, 10, false), None); + assert_eq!(next_cursor(99, 50, false), None); + } + + #[test] + fn next_cursor_returns_none_on_u32_overflow_rather_than_wrapping() { + let max = u32::MAX as usize; + assert_eq!(next_cursor(max, 1, true), None); + assert_eq!(next_cursor(max - 5, 10, true), None); + assert_eq!( + next_cursor(max - 10, 10, true), + Some(SearchOffset::new(u32::MAX)) + ); + } + + #[test] + fn cursor_token_round_trips() { + let off = SearchOffset::new(0xc0fe); + let token = off.encode_token(); + assert_eq!(token.len(), OFFSET_TOKEN_LEN); + assert_eq!(SearchOffset::decode_token(&token).unwrap(), off); + } + + #[test] + fn cursor_decode_rejects_malformed() { + let bad = ["", "deadbeef0", "no-hex!!", "zzzzzzzz", "1234567"]; + for s in bad { + assert!(matches!( + SearchOffset::decode_token(s), + Err(SearchOffsetError::Malformed), + )); + } + } +} -- 2.51.2