From 40d64afff7a92d0bd2e10043af8b9cfbdb41ed60 Mon Sep 17 00:00:00 2001 From: Kasper Date: Wed, 2 Sep 2026 09:27:03 +0200 Subject: [PATCH] wip sqlite ngram search --- ferrum-addon/addon.d.ts | 11 +++ src-native/filter.rs | 139 +++++++++++++++++++++++++++- src-native/migrations/1_start.sql | 41 +++++++++ src-native/page.rs | 148 +++++++++++++++++++++++++++--- 4 files changed, 323 insertions(+), 16 deletions(-) diff --git a/ferrum-addon/addon.d.ts b/ferrum-addon/addon.d.ts index 91517db..f2d20ac 100644 --- a/ferrum-addon/addon.d.ts +++ b/ferrum-addon/addon.d.ts @@ -122,6 +122,17 @@ export declare function move_tracks(playlistId: string, itemIds: Array, export declare function new_playlist(name: string, description: string, isFolder: boolean, parentId: string): void +export declare const enum NgramField { + Title = 0, + Artist = 1, + Album = 2, + AlbumArtist = 3, + Composer = 6, + Comments = 4, + Genre = 5, + Group = 7 +} + export interface Paths { pathSeparator: string libraryDir: string diff --git a/src-native/filter.rs b/src-native/filter.rs index 48c32e4..5f63002 100644 --- a/src-native/filter.rs +++ b/src-native/filter.rs @@ -1,7 +1,12 @@ -use crate::library_types::{ItemId, Library, TRACK_ID_MAP}; +use crate::{ + data::Data, + library_types::{ItemId, Library, TRACK_ID_MAP}, +}; +use anyhow::{Context, Result}; use rayon::prelude::*; use serde::Deserialize; use specta::Type; +use sqlx::Connection; use std::str::Chars; use std::time::Instant; use unicode_normalization::{Recompositions, UnicodeNormalization}; @@ -234,6 +239,24 @@ pub enum Field { Bpm, } +#[derive(Clone, Debug, Deserialize, Type)] +#[cfg_attr(feature = "napi", napi)] +pub enum NgramField { + Title = 0, + Artist = 1, + Album = 2, + AlbumArtist = 3, + Composer = 6, + Comments = 4, + Genre = 5, + Group = 7, +} +impl NgramField { + pub fn as_id(&self) -> u8 { + self.clone() as u8 + } +} + #[derive(Default, Clone, Debug, Deserialize, Type)] #[cfg_attr(feature = "napi", napi(object))] pub struct FilterTerm { @@ -241,11 +264,123 @@ pub struct FilterTerm { pub literal: String, } impl FilterTerm { - fn is_whitespace(&self) -> bool { + pub fn is_whitespace(&self) -> bool { self.field.is_none() && self.literal == "" } } +#[derive(sqlx::FromRow)] +struct QueuedTrack { + queue_id: i64, + track_id: i64, + title: String, + artist: String, + album_title: Option, + album_artist: Option, + composer: Option, + comments: Option, + genre: Option, + grouping: Option, +} + +pub async fn insert_queued_track_ngrams() -> Result<()> { + let start_time = Instant::now(); + + let mut data = Data::get_async().await; + // todo: maybe use tx for x individual tracks at a time + let mut tx = data.db.begin().await?; + + // todo: bulk insert + + // todo: add all fields + let tracks: Vec = sqlx::query_as( + " + SELECT + q.id AS queue_id, + t.id AS track_id, + t.title, + t.artist, + t.album_title, + t.album_artist, + t.comments, + t.genre, + t.composer, + t.grouping + FROM search_queue q + JOIN tracks t ON t.id = q.track_id + ORDER BY q.id ASC + ", + ) + .fetch_all(&mut *tx) + .await + .context("Failed to fetch search_queue tracks")?; + + let last_queue_id = match tracks.last() { + Some(track) => track.queue_id, + None => { + tx.commit().await?; + return Ok(()); + } + }; + + for track in &tracks { + sqlx::query("DELETE FROM search_ngrams WHERE track_id = ?") + .bind(&track.track_id) + .execute(&mut *tx) + .await?; + + let fields = [ + (NgramField::Title, Some(track.title.as_str())), + (NgramField::Artist, Some(track.artist.as_str())), + (NgramField::Album, track.album_title.as_deref()), + (NgramField::AlbumArtist, track.album_artist.as_deref()), + (NgramField::Comments, track.comments.as_deref()), + (NgramField::Genre, track.genre.as_deref()), + (NgramField::Composer, track.composer.as_deref()), + (NgramField::Group, track.grouping.as_deref()), + ]; + + for (field, text) in fields { + let Some(text) = text else { + continue; + }; + + // todo: add normalise ngrams too using .to_lowercase().nfc() + + let chars: Vec = text.to_lowercase().chars().collect(); + + for n in 1..=3 { + for window in chars.windows(n) { + // todo: deduplicate rows before inserting + let ngram: String = window.iter().collect(); + + sqlx::query( + "INSERT OR IGNORE INTO search_ngrams + (track_id, field, ngram, is_normalised) + VALUES (?, ?, ?, FALSE)", + ) + .bind(&track.track_id) + .bind(field.as_id()) + .bind(ngram) + .execute(&mut *tx) + .await?; + } + } + } + } + + sqlx::query("DELETE FROM search_queue WHERE id <= ?") + .bind(last_queue_id) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + println!("Indexing took {}ms", start_time.elapsed().as_millis()); + + Ok(()) +} + pub fn filter(mut item_ids: Vec, terms: Vec, library: &Library) -> Vec { let now = Instant::now(); let terms: Vec<_> = terms diff --git a/src-native/migrations/1_start.sql b/src-native/migrations/1_start.sql index e36b497..bc62daf 100644 --- a/src-native/migrations/1_start.sql +++ b/src-native/migrations/1_start.sql @@ -40,6 +40,47 @@ create table tracks ( volume INTEGER NULL -- from -100 to 100 ); +CREATE TRIGGER search_queue_after_track_insert AFTER INSERT ON tracks BEGIN + INSERT OR IGNORE INTO search_queue (track_id) VALUES (NEW.id); +END; + +CREATE TRIGGER search_queue_after_track_update AFTER UPDATE OF + title, + artist, + album_title, + album_artist, + comments, + genre, + composer, + grouping +ON tracks BEGIN + INSERT OR IGNORE INTO search_queue (track_id) VALUES (NEW.id); +END; + +CREATE TRIGGER search_queue_after_track_delete AFTER DELETE ON tracks BEGIN + DELETE FROM search_ngrams WHERE track_id = OLD.id; + DELETE FROM search_queue WHERE track_id = OLD.id; +END; + +CREATE TABLE search_ngrams ( + ngram TEXT NOT NULL, + -- field IDs are defined by the Fields enum in filter.rs + field INTEGER NOT NULL, + is_normalised BOOLEAN NOT NULL, + track_id INTEGER NOT NULL REFERENCES tracks(id), + -- `ngram` first, because we always filter for it. + -- if we do filter for `is_normalised`, we're doing an exact search. exact searches often return few results anyway (for example when searching é), so we can put `field` first. + PRIMARY KEY (ngram, field, is_normalised, track_id) +) WITHOUT ROWID; + +-- needed to purge a track's ngrams cheaply on reindex/delete +CREATE INDEX search_ngrams_by_track ON search_ngrams (track_id, field); + +CREATE TABLE search_queue ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + track_id INTEGER NOT NULL UNIQUE REFERENCES tracks(id) +); + CREATE TABLE plays ( date INTEGER NOT NULL, track_id INTEGER NOT NULL REFERENCES tracks(id), diff --git a/src-native/page.rs b/src-native/page.rs index 8715721..6c38191 100644 --- a/src-native/page.rs +++ b/src-native/page.rs @@ -1,13 +1,11 @@ use crate::data::Data; use crate::db::TrackListKind; -use crate::filter::{FilterTerm, filter}; -use crate::library_types::new_item_ids_from_track_ids; -use crate::library_types::{ItemId, Library, TrackList}; -use crate::sort::sort; -use anyhow::Result; +use crate::filter::{FilterTerm, insert_queued_track_ngrams}; +use crate::library_types::{ItemId, new_item_ids_from_track_ids}; +use anyhow::{Context, Result}; use serde::{Deserialize, Serialize}; use specta::Type; -use sqlx::Connection; +use sqlx::{AssertSqlSafe, Connection}; #[cfg_attr(feature = "napi", napi(object))] #[derive(Deserialize, Clone, Type)] @@ -104,11 +102,37 @@ pub async fn get_tracks_page_js(options: TracksPageOptions) -> Result, field: i32, literal: &str) { +// sql.push_str( +// " AND EXISTS ( +// SELECT 1 +// FROM search_ngrams sn +// WHERE sn.track_id = t.id +// AND sn.field = ? +// AND sn.ngram = ? +// )", +// ); + +// args.push(FilterArg::Integer(field as i64)); +// args.push(FilterArg::Text(literal.to_owned())); +// } + pub async fn get_tracks_page(options: TracksPageOptions) -> Result { + { + insert_queued_track_ngrams().await?; + } + let mut data = Data::get_async().await; let mut tx = data.db.begin().await?; let start_time = std::time::Instant::now(); + let track_list: TrackListPage = sqlx::query_as( "SELECT kind, name, description FROM track_lists @@ -118,28 +142,124 @@ pub async fn get_tracks_page(options: TracksPageOptions) -> Result { .fetch_one(&mut *tx) .await?; - // todo: sort, filter + let sql = String::from( + " + WITH term1 AS ( + SELECT track_id + FROM search_ngrams + WHERE ngram IN ('dev', 'evo', 'vot', 'oti', 'tio', 'ion') + AND field IN (0, 1, 2) + AND is_normalised = 0 + GROUP BY track_id, field + HAVING COUNT(DISTINCT ngram) = 6 + ), + term2 AS ( + SELECT track_id + FROM search_ngrams + WHERE ngram IN ('tri', 'ris', 'ist', 'sta', 'tam') + AND field IN (0, 1, 2) + AND is_normalised = 0 + GROUP BY track_id, field + HAVING COUNT(DISTINCT ngram) = 5 + ) + SELECT track_id FROM term1 + INTERSECT + SELECT track_id FROM term2; + ", + ); + // let mut sql = String::from( + // "SELECT pt.track_id + // FROM playlist_tracks pt + // JOIN tracks t ON t.id = pt.track_id + // WHERE pt.track_list_id = ?", + // ); + // let mut where_clauses = Vec::new(); + let mut args = sqlx::sqlite::SqliteArguments::default(); - tx.commit().await?; + // for term in options.filter_terms.iter().filter(|t| !t.is_whitespace()) { + // match term.field { + // // Some(Field::Title) => add_text_filter(&mut sql, &mut args, 0, &term.literal), + // // Some(Field::Artist) => add_text_filter(&mut sql, &mut args, 1, &term.literal), + // // Some(Field::Album) => add_text_filter(&mut sql, &mut args, 2, &term.literal), + // // Some(Field::AlbumArtist) => add_text_filter(&mut sql, &mut args, 3, &term.literal), + // // Some(Field::Comments) => add_text_filter(&mut sql, &mut args, 4, &term.literal), + // // Some(Field::Genre) => add_text_filter(&mut sql, &mut args, 5, &term.literal), + // // Some(Field::Composer) => add_text_filter(&mut sql, &mut args, 6, &term.literal), + // // Some(Field::Group) => add_text_filter(&mut sql, &mut args, 7, &term.literal), + // None => { + // where_clauses.push( + // "EXISTS ( + // SELECT 1 + // FROM search_ngrams sn + // WHERE sn.track_id = t.id + // AND sn.field BETWEEN 0 AND 7 + // AND sn.ngram = ? + // )", + // ); + // args.add(&term.literal); + // } + // _ => todo!(), + // } + // } + + // sql.push_str(" ORDER BY "); + + // if options.group_album_tracks { + // sql.push_str( + // "t.album_artist COLLATE NOCASE, + // t.album_title COLLATE NOCASE, + // t.disc_num, + // t.track_num, + // ", + // ); + // } - let track_ids = vec![]; + // let sort_column = match options.sort_key.as_str() { + // "title" => "t.title", + // "artist" => "t.artist", + // "album" => "t.album_title", + // "album_artist" => "t.album_artist", + // "comments" => "t.comments", + // "genre" => "t.genre", + // "composer" => "t.composer", + // "group" => "t.grouping", + // _ => "pt.item_pos", + // }; - // todo: remove - let item_ids = new_item_ids_from_track_ids(&track_ids); + // sql.push_str(sort_column); + // sql.push_str(if options.sort_desc { " DESC" } else { " ASC" }); + + // // Stable ordering. + // sql.push_str(", pt.item_pos ASC"); + + let track_ids: Vec = sqlx::query_scalar_with(AssertSqlSafe(sql), args) + .fetch_all(&mut *tx) + .await + .context("Failed to select page track_ids")?; println!( "get_tracks_page took {:?}, {} results", start_time.elapsed(), track_ids.len() ); - let tracks_page = TracksPage { + + let text_ids: Vec = sqlx::query_scalar( + "SELECT text_id FROM tracks WHERE id IN (SELECT value FROM json_each(?))", + ) + .bind(serde_json::to_string(&track_ids)?) + .fetch_all(&mut *tx) + .await?; + let item_ids = new_item_ids_from_track_ids(&text_ids); + + tx.commit().await?; + + Ok(TracksPage { playlist_kind: track_list.kind.to_string(), playlist_name: track_list.name, playlist_description: track_list.description, playlist_length: track_ids.len().try_into().unwrap(), item_ids, - }; - Ok(tracks_page) + }) } #[cfg(test)] -- 2.51.2