diff --git a/Cargo.lock b/Cargo.lock index fc0efaf..24e23e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -91,15 +91,6 @@ version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" -[[package]] -name = "arc-swap" -version = "1.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" -dependencies = [ - "rustversion", -] - [[package]] name = "async-broadcast" version = "0.7.2" @@ -328,15 +319,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bitpacking" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96a7139abd3d9cebf8cd6f920a389cf3dc9576172e32f4563f188cae3c3eb019" -dependencies = [ - "crunchy", -] - [[package]] name = "block-buffer" version = "0.10.4" @@ -386,31 +368,6 @@ dependencies = [ "piper", ] -[[package]] -name = "bon" -version = "3.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a602c73c7b0148ec6d12af6fd5cc7a46e2eacc8878271a999abac56eed12f561" -dependencies = [ - "bon-macros", - "rustversion", -] - -[[package]] -name = "bon-macros" -version = "3.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" -dependencies = [ - "darling", - "ident_case", - "prettyplease", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.106", -] - [[package]] name = "brotli" version = "8.0.2" @@ -539,17 +496,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65193589c6404eb80b450d618eaf9a2cafaaafd57ecce47370519ef674a7bd44" dependencies = [ "find-msvc-tools", - "jobserver", - "libc", "shlex", ] -[[package]] -name = "census" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0" - [[package]] name = "cesu8" version = "1.1.0" @@ -789,12 +738,6 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" -[[package]] -name = "crunchy" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" - [[package]] name = "crypto-common" version = "0.1.7" @@ -917,12 +860,6 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" -[[package]] -name = "datasketches" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745" - [[package]] name = "deranged" version = "0.5.8" @@ -1075,12 +1012,6 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" -[[package]] -name = "downcast-rs" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" - [[package]] name = "dpi" version = "0.1.2" @@ -1270,12 +1201,6 @@ dependencies = [ "thiserror 2.0.18", ] -[[package]] -name = "fastdivide" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471" - [[package]] name = "fastrand" version = "2.3.0" @@ -1301,7 +1226,6 @@ dependencies = [ "chrono", "dirs-next", "fast_image_resize", - "futures-util", "image", "lazy_static", "linked-hash-map", @@ -1320,7 +1244,6 @@ dependencies = [ "simd-json", "specta", "sqlx", - "tantivy", "tempfile", "time", "tokio", @@ -1454,16 +1377,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fs4" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" -dependencies = [ - "rustix 1.1.4", - "windows-sys 0.59.0", -] - [[package]] name = "futf" version = "0.1.5" @@ -2007,12 +1920,6 @@ dependencies = [ "match_token", ] -[[package]] -name = "htmlescape" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" - [[package]] name = "http" version = "1.4.0" @@ -2305,15 +2212,6 @@ dependencies = [ "cfb", ] -[[package]] -name = "inventory" -version = "0.3.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" -dependencies = [ - "rustversion", -] - [[package]] name = "io-uring" version = "0.7.10" @@ -2360,15 +2258,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "itertools" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.15" @@ -2420,16 +2309,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" -[[package]] -name = "jobserver" -version = "0.1.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" -dependencies = [ - "getrandom 0.4.2", - "libc", -] - [[package]] name = "js-sys" version = "0.3.91" @@ -2497,12 +2376,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" -[[package]] -name = "levenshtein_automata" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" - [[package]] name = "libappindicator" version = "0.9.0" @@ -2658,21 +2531,6 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" -[[package]] -name = "lru" -version = "0.16.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" -dependencies = [ - "hashbrown 0.16.1", -] - -[[package]] -name = "lz4_flex" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" - [[package]] name = "mac" version = "0.1.1" @@ -2720,30 +2578,12 @@ dependencies = [ "digest 0.11.3", ] -[[package]] -name = "measure_time" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51c55d61e72fc3ab704396c5fa16f4c184db37978ae4e94ca8959693a235fc0e" -dependencies = [ - "log", -] - [[package]] name = "memchr" version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" -[[package]] -name = "memmap2" -version = "0.9.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" -dependencies = [ - "libc", -] - [[package]] name = "memoffset" version = "0.9.1" @@ -2768,12 +2608,6 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - [[package]] name = "miniz_oxide" version = "0.8.9" @@ -2826,12 +2660,6 @@ dependencies = [ "windows-sys 0.60.2", ] -[[package]] -name = "murmurhash32" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b" - [[package]] name = "nanoid" version = "0.4.0" @@ -2961,16 +2789,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - [[package]] name = "num-conv" version = "0.2.2" @@ -3298,12 +3116,6 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" -[[package]] -name = "oneshot" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" - [[package]] name = "open" version = "5.3.3" @@ -3322,15 +3134,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" -[[package]] -name = "ordered-float" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e" -dependencies = [ - "num-traits", -] - [[package]] name = "ordered-stream" version = "0.2.0" @@ -3357,15 +3160,6 @@ dependencies = [ "windows-sys 0.61.0", ] -[[package]] -name = "ownedbytes" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fbd56f7631767e61784dc43f8580f403f4475bd4aaa4da003e6295e1bab4a7e" -dependencies = [ - "stable_deref_trait", -] - [[package]] name = "pango" version = "0.18.3" @@ -4072,16 +3866,6 @@ dependencies = [ "windows-sys 0.60.2", ] -[[package]] -name = "rust-stemmers" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" -dependencies = [ - "serde", - "serde_derive", -] - [[package]] name = "rustc-demangle" version = "0.1.26" @@ -4489,15 +4273,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" -[[package]] -name = "sketches-ddsketch" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "513c3f5f732bfd6fbb187619c2dfe9d2f25f1a2976f01d575f0fd329d565df56" -dependencies = [ - "serde", -] - [[package]] name = "slab" version = "0.4.11" @@ -4937,154 +4712,6 @@ dependencies = [ "version-compare", ] -[[package]] -name = "tantivy" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edde6a10743fff00a4e1a8c9ef020bf5f3cbad301b7d2d39f2b07f123c4eac07" -dependencies = [ - "aho-corasick", - "arc-swap", - "base64 0.22.1", - "bitpacking", - "bon", - "byteorder", - "census", - "crc32fast", - "crossbeam-channel", - "datasketches", - "downcast-rs", - "fastdivide", - "fnv", - "fs4", - "htmlescape", - "itertools", - "levenshtein_automata", - "log", - "lru", - "lz4_flex", - "measure_time", - "memmap2", - "once_cell", - "oneshot", - "rayon", - "regex", - "rust-stemmers", - "rustc-hash", - "serde", - "serde_json", - "sketches-ddsketch", - "smallvec", - "tantivy-bitpacker", - "tantivy-columnar", - "tantivy-common", - "tantivy-fst", - "tantivy-query-grammar", - "tantivy-stacker", - "tantivy-tokenizer-api", - "tempfile", - "thiserror 2.0.18", - "time", - "typetag", - "uuid", - "winapi", -] - -[[package]] -name = "tantivy-bitpacker" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fed3d674429bcd2de5d0a6d1aa5495fed8afd9c5ecce993019caf7615f53fa4" -dependencies = [ - "bitpacking", -] - -[[package]] -name = "tantivy-columnar" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c57166f5bcfd478f370ab8445afb4678dce44801fa5ce5c451aaf8595583c5dc" -dependencies = [ - "downcast-rs", - "fastdivide", - "itertools", - "serde", - "tantivy-bitpacker", - "tantivy-common", - "tantivy-sstable", - "tantivy-stacker", -] - -[[package]] -name = "tantivy-common" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbf10915aa75da3c3b0d58b58853d2e889efbaf32d4982a4c3715dde6bba23e5" -dependencies = [ - "async-trait", - "byteorder", - "ownedbytes", - "serde", - "time", -] - -[[package]] -name = "tantivy-fst" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18" -dependencies = [ - "byteorder", - "regex-syntax", - "utf8-ranges", -] - -[[package]] -name = "tantivy-query-grammar" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfadb8526b6da90704feb293b0701a6aae62ea14983143344be2dc5ce30f1d82" -dependencies = [ - "fnv", - "nom", - "ordered-float", - "serde", - "serde_json", -] - -[[package]] -name = "tantivy-sstable" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a2cfc3ac5164cbadc28965ffb145a8f47582a60ae5897859ad8d4316596c606" -dependencies = [ - "futures-util", - "itertools", - "tantivy-bitpacker", - "tantivy-common", - "tantivy-fst", - "zstd", -] - -[[package]] -name = "tantivy-stacker" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cbb051742da9d53ca9e8fff43a9b10e319338b24e2c0e15d0372df19ffeb951" -dependencies = [ - "murmurhash32", - "tantivy-common", -] - -[[package]] -name = "tantivy-tokenizer-api" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac258c2c6390673f2685813afeeafcb8c4e0ee7de8dd3fc46838dcc37263f98" -dependencies = [ - "serde", -] - [[package]] name = "tao" version = "0.34.5" @@ -5922,30 +5549,6 @@ version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" -[[package]] -name = "typetag" -version = "0.2.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c90e86058a30d42a1a928dfb4b49bb33c98c3a2b4909492e6b0881cd94798ec2" -dependencies = [ - "erased-serde", - "inventory", - "once_cell", - "serde", - "typetag-impl", -] - -[[package]] -name = "typetag-impl" -version = "0.2.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - [[package]] name = "uds_windows" version = "1.1.0" @@ -6073,12 +5676,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" -[[package]] -name = "utf8-ranges" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" - [[package]] name = "utf8_iter" version = "1.0.4" @@ -7253,34 +6850,6 @@ version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" -[[package]] -name = "zstd" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "7.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" -dependencies = [ - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" -dependencies = [ - "cc", - "pkg-config", -] - [[package]] name = "zune-core" version = "0.4.12" diff --git a/Cargo.toml b/Cargo.toml index 9643d8f..0c4efcd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,8 +59,6 @@ mimalloc = "0.1.48" specta = { version = "=2.0.0-rc.22", features = ["derive"] } tempfile = "3.27.0" sqlx = { version = "0.9.0", features = ["sqlite"] } -tantivy = "0.26.1" -futures-util = "0.3.34" [target.'cfg(not(target_os = "android"))'.dependencies] trash = "5.1" diff --git a/src-native/page.rs b/src-native/page.rs index 8df481d..9dab0b1 100644 --- a/src-native/page.rs +++ b/src-native/page.rs @@ -1,21 +1,11 @@ use crate::data::Data; use crate::db::TrackListKind; -use crate::library::Paths; +use crate::library_types::ItemId; use crate::library_types::new_item_ids_from_track_ids; -use crate::library_types::{ItemId, SpecialTrackListName}; use anyhow::Result; -use futures_util::TryStreamExt; use serde::{Deserialize, Serialize}; use specta::Type; use sqlx::Connection; -use sqlx::Row; -use std::path::Path; -use std::time::Instant; -use tantivy::collector::{Collector, TopDocs}; -use tantivy::query::{BooleanQuery, Occur, Query, QueryParser, TermQuery}; -use tantivy::tokenizer::NgramTokenizer; -use tantivy::{DocAddress, Index, IndexReader, IndexWriter, Searcher, doc}; -use tantivy::{Order, schema::*}; #[cfg_attr(feature = "napi", napi(object))] #[derive(Deserialize, Clone, Type)] @@ -65,110 +55,6 @@ fn to_sql_sort_key(sort_key: &str) -> (&'static str, bool) { } } -struct TantivyFields { - track_id: Field, - playlist_id: Field, - playlist_pos: Field, - added_at: Field, - duration_s: Field, - bpm: Field, - play_count: Field, - skip_count: Field, - year: Field, - title: Field, - title_sort: Field, - artist: Field, - artist_sort: Field, - composer: Field, - composer_sort: Field, - genre: Field, - genre_sort: Field, - comments: Field, - comments_sort: Field, - grouping: Field, - grouping_sort: Field, - album_title: Field, - album_title_sort: Field, - album_artist: Field, - album_artist_sort: Field, -} - -fn build_tantivy_schema() -> (Schema, TantivyFields) { - let mut builder = Schema::builder(); - - // todo: remove STORED, if possible - - // IDs - let track_id = builder.add_text_field("track_id", STRING | STORED); - let playlist_id = builder.add_text_field("playlist_id", STRING); - - // Numeric sorts (need FAST) - let playlist_pos = builder.add_u64_field("playlist_pos", FAST); - let added_at = builder.add_i64_field("added_at", FAST); - let duration_s = builder.add_f64_field("duration_s", FAST); - let bpm = builder.add_f64_field("bpm", FAST); - let play_count = builder.add_u64_field("play_count", FAST); - let skip_count = builder.add_u64_field("skip_count", FAST); - let year = builder.add_i64_field("year", FAST); - - // Searchable text: 1-, 2-, and 3-character inner ngrams. - let subword_options = TextOptions::default().set_indexing_options( - TextFieldIndexing::default() - .set_tokenizer("subword") - .set_index_option(IndexRecordOption::WithFreqsAndPositions), - ); - - // Searchable text (TEXT = tokenized) - let title = builder.add_text_field("title", subword_options.clone()); - let artist = builder.add_text_field("artist", subword_options.clone()); - let composer = builder.add_text_field("composer", subword_options.clone()); - let genre = builder.add_text_field("genre", subword_options.clone()); - let comments = builder.add_text_field("comments", subword_options.clone()); - let grouping = builder.add_text_field("grouping", subword_options.clone()); - let album_title = builder.add_text_field("album_title", subword_options.clone()); - let album_artist = builder.add_text_field("album_artist", subword_options); - - // Sortable text (STRING | FAST = lexicographic sort, not tokenized) - let title_sort = builder.add_text_field("title_sort", STRING | FAST); - let artist_sort = builder.add_text_field("artist_sort", STRING | FAST); - let composer_sort = builder.add_text_field("composer_sort", STRING | FAST); - let genre_sort = builder.add_text_field("genre_sort", STRING | FAST); - let comments_sort = builder.add_text_field("comments_sort", STRING | FAST); - let grouping_sort = builder.add_text_field("grouping_sort", STRING | FAST); - let album_title_sort = builder.add_text_field("album_title_sort", STRING | FAST); - let album_artist_sort = builder.add_text_field("album_artist_sort", STRING | FAST); - - let schema = builder.build(); - let fields = TantivyFields { - track_id, - playlist_id, - playlist_pos, - added_at, - duration_s, - bpm, - play_count, - skip_count, - year, - title, - title_sort, - artist, - artist_sort, - composer, - composer_sort, - genre, - genre_sort, - comments, - comments_sort, - grouping, - grouping_sort, - album_title, - album_title_sort, - album_artist, - album_artist_sort, - }; - (schema, fields) -} - #[derive(sqlx::FromRow)] struct AllTrackRow { id: String, @@ -209,379 +95,6 @@ struct PlaylistTrackRow { year: Option, } -fn add_opt_text(document: &mut TantivyDocument, field: Field, value: &Option<&str>) { - if let Some(value) = value { - document.add_text(field, value); - } -} - -fn add_opt_i64(document: &mut TantivyDocument, field: Field, value: Option) { - if let Some(value) = value { - document.add_i64(field, value); - } -} - -fn add_opt_f64(document: &mut TantivyDocument, field: Field, value: Option) { - if let Some(value) = value { - document.add_f64(field, value); - } -} - -// Builds an in-memory Tantivy index and fills it with every playlist_tracks -// row (tagged with its own playlist) plus every track (tagged with each -// "special" track list id, i.e. root). -async fn build_tantivy_index( - paths: Paths, - tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, -) -> Result<(Index, TantivyFields)> { - let (schema, fields) = build_tantivy_schema(); - let tantivy_dir = Path::new(&paths.library_dir).join("Tantivy"); - std::fs::remove_dir_all(&tantivy_dir).unwrap(); - std::fs::create_dir_all(&tantivy_dir).unwrap(); - let index = Index::create_in_dir(&tantivy_dir, schema)?; - let tokenizer = NgramTokenizer::all_ngrams(1, 3)?; - index.tokenizers().register("subword", tokenizer); - let mut writer: IndexWriter = index.writer(50_000_000)?; - - let start_time = Instant::now(); - let t = Instant::now(); - - let all_tracks: Vec<_> = sqlx::query( - "SELECT id, title, artist, composer, genre, comments, grouping, album_title, - album_artist, added_at, duration_s, bpm, play_count, skip_count, year - FROM tracks - ORDER BY added_at ASC", - ) - .fetch_all(&mut **tx) - .await?; - println!(" select all_tracks {:?}", t.elapsed()); - let t = Instant::now(); - - let mut docs = Vec::with_capacity(all_tracks.len()); - for (i, row) in all_tracks.into_iter().enumerate() { - let id: &str = row.try_get(0).unwrap(); - let title: &str = row.try_get(1).unwrap(); - let artist: &str = row.try_get(2).unwrap(); - let composer: Option<&str> = row.try_get(3).unwrap(); - let genre: Option<&str> = row.try_get(4).unwrap(); - let comments: Option<&str> = row.try_get(5).unwrap(); - let grouping: Option<&str> = row.try_get(6).unwrap(); - let album_title: Option<&str> = row.try_get(7).unwrap(); - let album_artist: Option<&str> = row.try_get(8).unwrap(); - let added_at: i64 = row.try_get(9).unwrap(); - let duration_s: f64 = row.try_get(10).unwrap(); - let bpm: Option = row.try_get(11).unwrap(); - let play_count: u64 = row.try_get(12).unwrap(); - let skip_count: u64 = row.try_get(13).unwrap(); - let year: Option = row.try_get(14).unwrap(); - - let mut doc = TantivyDocument::default(); - doc.add_text(fields.track_id, &id); - doc.add_text(fields.playlist_id, SpecialTrackListName::Root.get_id()); - doc.add_u64(fields.playlist_pos, i.try_into().unwrap()); - - // Search fields - doc.add_text(fields.title, &title); - doc.add_text(fields.artist, &artist); - add_opt_text(&mut doc, fields.composer, &composer); - add_opt_text(&mut doc, fields.genre, &genre); - add_opt_text(&mut doc, fields.comments, &comments); - add_opt_text(&mut doc, fields.grouping, &grouping); - add_opt_text(&mut doc, fields.album_title, &album_title); - add_opt_text(&mut doc, fields.album_artist, &album_artist); - - // Sort fields (mirror the text values) - doc.add_text(fields.title_sort, &title); - doc.add_text(fields.artist_sort, &artist); - add_opt_text(&mut doc, fields.composer_sort, &composer); - add_opt_text(&mut doc, fields.genre_sort, &genre); - add_opt_text(&mut doc, fields.comments_sort, &comments); - add_opt_text(&mut doc, fields.grouping_sort, &grouping); - add_opt_text(&mut doc, fields.album_title_sort, &album_title); - add_opt_text(&mut doc, fields.album_artist_sort, &album_artist); - - // Numeric sorts - doc.add_i64(fields.added_at, added_at); - doc.add_f64(fields.duration_s, duration_s); - add_opt_f64(&mut doc, fields.bpm, bpm); - doc.add_u64(fields.play_count, play_count); - doc.add_u64(fields.skip_count, skip_count); - add_opt_i64(&mut doc, fields.year, year); - - docs.push(doc); - } - println!(" create docs {:?}", t.elapsed()); - let t = Instant::now(); - - for doc in docs { - writer.add_document(doc)?; - } - println!(" add docs {:?}", t.elapsed()); - let t = Instant::now(); - - let mut playlist_tracks = sqlx::query( - "SELECT - playlist_tracks.track_list_id, - playlist_tracks.item_pos, - tracks.id, - tracks.title, - tracks.artist, - tracks.composer, - tracks.genre, - tracks.comments, - tracks.grouping, - tracks.album_title, - tracks.album_artist, - tracks.added_at, - tracks.duration_s, - tracks.bpm, - tracks.play_count, - tracks.skip_count, - tracks.year - FROM playlist_tracks - JOIN tracks ON tracks.id = playlist_tracks.track_id", - ) - .fetch(&mut **tx); - println!(" select playlist_tracks {:?}", t.elapsed()); - - while let Some(row) = playlist_tracks.try_next().await? { - let track_list_id: &str = row.try_get(0).unwrap(); - let item_pos: u64 = row.try_get(1).unwrap(); - let id: &str = row.try_get(2).unwrap(); - let title: &str = row.try_get(3).unwrap(); - let artist: &str = row.try_get(4).unwrap(); - let composer: Option<&str> = row.try_get(5).unwrap(); - let genre: Option<&str> = row.try_get(6).unwrap(); - let comments: Option<&str> = row.try_get(7).unwrap(); - let grouping: Option<&str> = row.try_get(8).unwrap(); - let album_title: Option<&str> = row.try_get(9).unwrap(); - let album_artist: Option<&str> = row.try_get(10).unwrap(); - let added_at: i64 = row.try_get(11).unwrap(); - let duration_s: f64 = row.try_get(12).unwrap(); - let bpm: Option = row.try_get(13).unwrap(); - let play_count: u64 = row.try_get(14).unwrap(); - let skip_count: u64 = row.try_get(15).unwrap(); - let year: Option = row.try_get(16).unwrap(); - - let mut doc = TantivyDocument::default(); - doc.add_text(fields.track_id, &id); - doc.add_text(fields.playlist_id, &track_list_id); - doc.add_u64(fields.playlist_pos, item_pos); - - // Search fields - doc.add_text(fields.title, &title); - doc.add_text(fields.artist, &artist); - add_opt_text(&mut doc, fields.composer, &composer); - add_opt_text(&mut doc, fields.genre, &genre); - add_opt_text(&mut doc, fields.comments, &comments); - add_opt_text(&mut doc, fields.grouping, &grouping); - add_opt_text(&mut doc, fields.album_title, &album_title); - add_opt_text(&mut doc, fields.album_artist, &album_artist); - - // Sort fields (mirror the text values) - doc.add_text(fields.title_sort, &title); - doc.add_text(fields.artist_sort, &artist); - add_opt_text(&mut doc, fields.composer_sort, &composer); - add_opt_text(&mut doc, fields.genre_sort, &genre); - add_opt_text(&mut doc, fields.comments_sort, &comments); - add_opt_text(&mut doc, fields.grouping_sort, &grouping); - add_opt_text(&mut doc, fields.album_title_sort, &album_title); - add_opt_text(&mut doc, fields.album_artist_sort, &album_artist); - - // Numeric sorts - doc.add_i64(fields.added_at, added_at); - doc.add_f64(fields.duration_s, duration_s); - add_opt_f64(&mut doc, fields.bpm, bpm); - doc.add_u64(fields.play_count, play_count); - doc.add_u64(fields.skip_count, skip_count); - add_opt_i64(&mut doc, fields.year, year); - - writer.add_document(doc)?; - } - - println!(" select all tracks {:?}", start_time.elapsed()); - - let start = Instant::now(); - writer.commit()?; - println!(" commit {:?}", start.elapsed()); - Ok((index, fields)) -} - -/// Pull track IDs from any TopDocs result regardless of its sort/score type. -fn extract_track_ids( - searcher: &Searcher, - results: Vec<(T, DocAddress)>, - fields: &TantivyFields, -) -> Result> { - let start_time = std::time::Instant::now(); - let mut ids = Vec::with_capacity(results.len()); - for (_, doc_address) in results { - let doc: TantivyDocument = searcher.doc(doc_address)?; - if let Some(id) = doc.get_first(fields.track_id).and_then(|v| v.as_str()) { - ids.push(id.to_string()); - } - } - println!("extract_track_ids took {:?}", start_time.elapsed()); - Ok(ids) -} - -/// Generic helper: build the collector, search, extract IDs. -fn do_search( - searcher: &Searcher, - query: &dyn Query, - fields: &TantivyFields, - collector: impl Collector>, -) -> Result> { - let start_time = std::time::Instant::now(); - let results = searcher.search(query, &collector)?; - println!("search took {:?}", start_time.elapsed()); - extract_track_ids(searcher, results, fields) -} - -fn search_tantivy( - index: &Index, - reader: &IndexReader, - fields: &TantivyFields, - playlist_id: &str, - filter_query: &str, - sort_key: &str, - sort_desc: bool, -) -> Result> { - let start_time = std::time::Instant::now(); - let searcher = reader.searcher(); - - let playlist_term = Term::from_field_text(fields.playlist_id, playlist_id); - let playlist_query = TermQuery::new(playlist_term, IndexRecordOption::Basic); - - let query: Box = if filter_query.is_empty() { - Box::new(playlist_query) - } else { - let text_fields = vec![ - fields.title, - fields.artist, - fields.composer, - fields.genre, - fields.comments, - fields.grouping, - fields.album_title, - fields.album_artist, - ]; - let mut parser = QueryParser::for_index(index, text_fields); - parser.set_conjunction_by_default(); - - let text_query = parser.parse_query(filter_query)?; - - Box::new(BooleanQuery::new(vec![ - (Occur::Must, Box::new(playlist_query)), - (Occur::Must, text_query), - ])) - }; - - let order = if sort_desc { Order::Desc } else { Order::Asc }; - let limit = searcher.num_docs().try_into().unwrap(); - - let top_docs = TopDocs::with_limit(limit); - println!("prepare search took {:?}", start_time.elapsed()); - let results = match sort_key { - "index" => do_search( - &searcher, - &query, - &fields, - top_docs.order_by_u64_field("playlist_pos", order), - ), - "dateAdded" => do_search( - &searcher, - &query, - &fields, - top_docs.order_by_fast_field::("added_at", order), - ), - "duration" => do_search( - &searcher, - &query, - &fields, - top_docs.order_by_fast_field::("duration_s", order), - ), - "bpm" => do_search( - &searcher, - &query, - &fields, - top_docs.order_by_fast_field::("bpm", order), - ), - "playCount" => do_search( - &searcher, - &query, - &fields, - top_docs.order_by_u64_field("play_count", order), - ), - "skipCount" => do_search( - &searcher, - &query, - &fields, - top_docs.order_by_u64_field("skip_count", order), - ), - "year" => do_search( - &searcher, - &query, - &fields, - top_docs.order_by_fast_field::("year", order), - ), - "name" => do_search( - &searcher, - &query, - &fields, - top_docs.order_by_string_fast_field("title_sort", order), - ), - "artist" => do_search( - &searcher, - &query, - &fields, - top_docs.order_by_string_fast_field("artist_sort", order), - ), - "albumName" => do_search( - &searcher, - &query, - &fields, - top_docs.order_by_string_fast_field("album_title_sort", order), - ), - "albumArtist" => do_search( - &searcher, - &query, - &fields, - top_docs.order_by_string_fast_field("album_artist_sort", order), - ), - "composer" => do_search( - &searcher, - &query, - &fields, - top_docs.order_by_string_fast_field("composer_sort", order), - ), - "genre" => do_search( - &searcher, - &query, - &fields, - top_docs.order_by_string_fast_field("genre_sort", order), - ), - "comments" => do_search( - &searcher, - &query, - &fields, - top_docs.order_by_string_fast_field("comments_sort", order), - ), - "grouping" => do_search( - &searcher, - &query, - &fields, - top_docs.order_by_string_fast_field("grouping_sort", order), - ), - - _ => panic!("Unsupported sort key {sort_key}"), - }; - - let track_ids = results?; - - Ok(track_ids) -} - #[cfg(feature = "napi")] #[cfg_attr(feature = "napi", napi(js_name = "get_tracks_page"))] #[allow(dead_code)] @@ -594,12 +107,6 @@ pub async fn get_tracks_page(options: TracksPageOptions) -> Result { let paths = data.paths.clone(); let mut tx = data.db.begin().await?; - // todo: just for testing - let start_time = std::time::Instant::now(); - let (tantivy_index, tantivy_fields) = build_tantivy_index(paths, &mut tx).await?; - let tantivy_reader = tantivy_index.reader()?; - println!("built index took {:?}", start_time.elapsed(),); - let start_time = std::time::Instant::now(); let track_list: TrackListPage = sqlx::query_as( "SELECT kind, name, description @@ -611,22 +118,12 @@ pub async fn get_tracks_page(options: TracksPageOptions) -> Result { .await?; let filter_query = options.filter_query.trim(); - println!("select track_lists took {:?}", start_time.elapsed(),); - - let start_timex = std::time::Instant::now(); - let track_ids = search_tantivy( - &tantivy_index, - &tantivy_reader, - &tantivy_fields, - &options.playlist_id, - filter_query, - &options.sort_key, - options.sort_desc, - )?; - println!("search_tantivity() took {:?}", start_timex.elapsed(),); tx.commit().await?; + let track_ids = vec![]; + + // todo: remove let item_ids = new_item_ids_from_track_ids(&track_ids); println!(