From 99cd8ba6b88cf451e6bb76194ea9c5ee844e5016 Mon Sep 17 00:00:00 2001 From: Owais Jamil Date: Thu, 9 Apr 2026 07:35:33 -0500 Subject: [PATCH] feat: embedding service with WordPiece tokenizer and post text extractor --- docs/tasks/phase-7.md | 24 +- lib/core/embedding/embedding_service.dart | 186 ++++++++++++++++ lib/core/embedding/word_piece_tokenizer.dart | 147 +++++++++++++ .../search/data/post_text_extractor.dart | 62 ++++++ .../embedding/embedding_service_test.dart | 156 +++++++++++++ .../embedding/word_piece_tokenizer_test.dart | 171 +++++++++++++++ .../search/data/post_text_extractor_test.dart | 207 ++++++++++++++++++ 7 files changed, 942 insertions(+), 11 deletions(-) create mode 100644 lib/core/embedding/embedding_service.dart create mode 100644 lib/core/embedding/word_piece_tokenizer.dart create mode 100644 lib/features/search/data/post_text_extractor.dart create mode 100644 test/core/embedding/embedding_service_test.dart create mode 100644 test/core/embedding/word_piece_tokenizer_test.dart create mode 100644 test/features/search/data/post_text_extractor_test.dart diff --git a/docs/tasks/phase-7.md b/docs/tasks/phase-7.md index 828337f..c2c1907 100644 --- a/docs/tasks/phase-7.md +++ b/docs/tasks/phase-7.md @@ -12,22 +12,22 @@ updated: 2026-04-09 #### ObjectBox Setup - [x] Add `objectbox`, `objectbox_flutter_libs` to `pubspec.yaml`; add `objectbox_generator` to dev deps -- [ ] `EmbeddedPost` entity - `postUri` (unique), `accountDid`, `source` (saved/liked), `indexedText`, `embedding` (384D float vector, HNSW cosine index), `embeddedAt` -- [ ] Run `build_runner` to generate `objectbox.g.dart` and `objectbox-model.json` -- [ ] `ObjectBoxStore` singleton - `openStore()` at app startup (after Drift init), expose via `RepositoryProvider` -- [ ] `EmbeddingRepository` - CRUD operations on `EmbeddedPost`: `upsert`, `deleteByUri`, `queryByAccount`, `countByAccount` +- [x] `EmbeddedPost` entity - `postUri` (unique), `accountDid`, `source` (saved/liked), `indexedText`, `embedding` (384D float vector, HNSW cosine index), `embeddedAt` +- [x] Run `build_runner` to generate `objectbox.g.dart` and `objectbox-model.json` +- [x] `ObjectBoxStore` singleton - `openStore()` at app startup (after Drift init), expose via `RepositoryProvider` +- [x] `EmbeddingRepository` - CRUD operations on `EmbeddedPost`: `upsert`, `deleteByUri`, `queryByAccount`, `countByAccount` #### TFLite Embedding Service - [x] Add `tflite_flutter` to `pubspec.yaml` - [x] Bundle `minilm_l6_v2_int8.tflite` and `vocab.txt` as Flutter assets -- [ ] `WordPieceTokenizer` - load vocab, tokenize text, pad/truncate to 256 tokens, return `List` -- [ ] `EmbeddingService` - long-lived background `Isolate` with `ReceivePort`/`SendPort` message passing -- [ ] `EmbeddingService.initialize()` - spawn isolate, load TFLite model + tokenizer in isolate -- [ ] `EmbeddingService.embed(String text)` - send text to isolate, receive `Float32List[384]`, L2-normalize -- [ ] `EmbeddingService.isAvailable` - flag gating UI entry points, false if model fails to load -- [ ] `EmbeddingService.dispose()` - close isolate and interpreter -- [ ] `PostTextExtractor` - concatenate post text + image alt texts + link card title/description into a single searchable string +- [x] `WordPieceTokenizer` - load vocab, tokenize text, pad/truncate to 256 tokens, return `List` +- [x] `EmbeddingService` - long-lived background `Isolate` with `ReceivePort`/`SendPort` message passing +- [x] `EmbeddingService.initialize()` - spawn isolate, load TFLite model + tokenizer in isolate +- [x] `EmbeddingService.embed(String text)` - send text to isolate, receive `Float32List[384]`, L2-normalize +- [x] `EmbeddingService.isAvailable` - flag gating UI entry points, false if model fails to load +- [x] `EmbeddingService.dispose()` - close isolate and interpreter +- [x] `PostTextExtractor` - concatenate post text + image alt texts + link card title/description into a single searchable string #### Liked Posts Sync @@ -37,6 +37,8 @@ updated: 2026-04-09 - [ ] `LikedPostsRepository.getLikedPosts(accountDid, {limit, offset})` - paginated query - [ ] `LikedPostsRepository.removeLike(accountDid, postUri)` - delete entry - [ ] Eviction: drop oldest entries when count exceeds 1000 per account +- [ ] Documentation update: move development information from README.md to a top-level DEVELOPMENT.md. + Should be updated to reflect new architecture and patterns. #### Indexing Pipeline diff --git a/lib/core/embedding/embedding_service.dart b/lib/core/embedding/embedding_service.dart new file mode 100644 index 0000000..9cb8f9b --- /dev/null +++ b/lib/core/embedding/embedding_service.dart @@ -0,0 +1,186 @@ +import 'dart:async'; +import 'dart:isolate'; +import 'dart:math' show sqrt; + +import 'package:flutter/foundation.dart'; +import 'package:flutter/services.dart'; +import 'package:lazurite/core/embedding/word_piece_tokenizer.dart'; +import 'package:tflite_flutter/tflite_flutter.dart'; + +final class _SetupData { + const _SetupData({required this.sendPort, required this.rootIsolateToken}); + final SendPort sendPort; + final RootIsolateToken rootIsolateToken; +} + +/// A request sent to the isolate: (text, replyPort) or null to dispose. +typedef _EmbedRequest = (String text, SendPort replyPort); + +/// L2-normalize [vector], returning a new [Float32List]. +/// +/// If the norm is near zero the original vector is returned unchanged. +@visibleForTesting +Float32List l2Normalize(Float32List vector) { + var norm = 0.0; + for (var i = 0; i < vector.length; i++) { + norm += vector[i] * vector[i]; + } + norm = sqrt(norm); + if (norm < 1e-10) return vector; + final out = Float32List(vector.length); + for (var i = 0; i < vector.length; i++) { + out[i] = vector[i] / norm; + } + return out; +} + +Future _isolateEntry(_SetupData setup) async { + BackgroundIsolateBinaryMessenger.ensureInitialized(setup.rootIsolateToken); + + final receivePort = ReceivePort(); + + setup.sendPort.send(receivePort.sendPort); + + Interpreter? interpreter; + WordPieceTokenizer? tokenizer; + + try { + interpreter = await Interpreter.fromAsset('all-MiniLM-L6-v2-quant.tflite'); + final vocabText = await rootBundle.loadString('assets/vocab.txt'); + tokenizer = WordPieceTokenizer.fromString(vocabText); + setup.sendPort.send(true); + } catch (_) { + setup.sendPort.send(false); + receivePort.close(); + return; + } + + await for (final message in receivePort) { + if (message == null) break; + final (text, replyPort) = message as _EmbedRequest; + try { + final result = _runInference(interpreter, tokenizer, text); + replyPort.send(result); + } catch (_) { + replyPort.send(null); + } + } + + interpreter.close(); + receivePort.close(); +} + +Float32List _runInference(Interpreter interpreter, WordPieceTokenizer tokenizer, String text) { + final tokenIds = tokenizer.tokenize(text); + const seqLen = WordPieceTokenizer.maxTokens; + + final inputIds = [tokenIds]; + final attentionMask = [tokenIds.map((id) => id != 0 ? 1 : 0).toList()]; + final tokenTypeIds = [List.filled(seqLen, 0)]; + + final outputBuffer = [List.filled(384, 0.0)]; + interpreter.runForMultipleInputs([inputIds, attentionMask, tokenTypeIds], {0: outputBuffer}); + + return l2Normalize(Float32List.fromList(outputBuffer[0])); +} + +/// On-device text embedding service backed by a long-lived background [Isolate]. +/// +/// Start with [initialize], shut down with [dispose]. Check [isAvailable] +/// before calling [embed]; the flag is false when the model fails to load or +/// when the service has not yet been initialised. +class EmbeddingService { + /// Creates a real embedding service backed by TFLite + Isolate. + EmbeddingService() : _mockEmbedFn = null; + + /// Creates a test double that bypasses the Isolate and TFLite entirely. + /// + /// [embedFn] is called synchronously (from the caller's perspective) on every + /// [embed] invocation. [initialize] immediately sets [isAvailable] to true. + @visibleForTesting + EmbeddingService.forTesting(Future Function(String text) embedFn) : _mockEmbedFn = embedFn; + + final Future Function(String text)? _mockEmbedFn; + + bool _isAvailable = false; + Isolate? _isolate; + SendPort? _isolateSendPort; + ReceivePort? _setupPort; + + /// Whether the service is ready to produce embeddings. + /// + /// False until [initialize] completes successfully, and false again after + /// [dispose] is called or if the model failed to load. + bool get isAvailable => _isAvailable; + + /// Initialise the service. + /// + /// For the real implementation this spawns a background [Isolate], loads the + /// TFLite model, and builds the [WordPieceTokenizer]. For the test double it + /// is a no-op that marks the service as available. + /// + /// Safe to call multiple times; subsequent calls are no-ops. + Future initialize() async { + if (_isAvailable) return; + if (_mockEmbedFn != null) { + _isAvailable = true; + return; + } + + final setupPort = ReceivePort(); + _setupPort = setupPort; + final messages = StreamIterator(setupPort); + + try { + _isolate = await Isolate.spawn( + _isolateEntry, + _SetupData(sendPort: setupPort.sendPort, rootIsolateToken: RootIsolateToken.instance!), + debugName: 'EmbeddingIsolate', + ); + + await messages.moveNext(); + _isolateSendPort = messages.current as SendPort; + + await messages.moveNext(); + _isAvailable = messages.current as bool; + } finally { + await messages.cancel(); + setupPort.close(); + _setupPort = null; + } + } + + /// Embed [text] and return an L2-normalised [Float32List] of length 384. + /// + /// Throws [StateError] if the service is not available. + Future embed(String text) async { + if (!_isAvailable) { + throw StateError('EmbeddingService is not available. Call initialize() first.'); + } + + if (_mockEmbedFn != null) { + return _mockEmbedFn(text); + } + + final responsePort = ReceivePort(); + _isolateSendPort!.send((text, responsePort.sendPort)); + final result = await responsePort.first; + responsePort.close(); + + if (result == null) throw StateError('Embedding inference failed for text: "$text"'); + return result as Float32List; + } + + /// Shut down the background isolate and mark the service as unavailable. + /// + /// Safe to call before [initialize] or after [dispose]. + void dispose() { + _isolateSendPort?.send(null); + _isolate?.kill(priority: Isolate.immediate); + _setupPort?.close(); + _isAvailable = false; + _isolate = null; + _isolateSendPort = null; + _setupPort = null; + } +} diff --git a/lib/core/embedding/word_piece_tokenizer.dart b/lib/core/embedding/word_piece_tokenizer.dart new file mode 100644 index 0000000..339201d --- /dev/null +++ b/lib/core/embedding/word_piece_tokenizer.dart @@ -0,0 +1,147 @@ +import 'package:characters/characters.dart'; + +/// BERT-style WordPiece tokenizer compatible with all-MiniLM-L6-v2. +/// +/// Converts text to token IDs using the standard BERT uncased vocabulary. +/// The returned list always has exactly [maxTokens] elements. +/// +/// Token ID conventions (BERT-base-uncased): +/// [PAD] = 0 +/// [UNK] = 100 +/// [CLS] = 101 +/// [SEP] = 102 +class WordPieceTokenizer { + WordPieceTokenizer._(this._vocab); + + /// Constructs a tokenizer from the raw contents of a vocab file. + /// + /// Each line is one token; its line number (0-indexed) is its ID. + factory WordPieceTokenizer.fromString(String vocabText) { + final vocab = {}; + var index = 0; + for (final line in vocabText.split('\n')) { + final token = line.trimRight(); + if (token.isNotEmpty) { + vocab[token] = index; + } + index++; + } + return WordPieceTokenizer._(vocab); + } + static const int padId = 0; + static const int unkId = 100; + static const int clsId = 101; + static const int sepId = 102; + static const int maxTokens = 256; + + final Map _vocab; + + /// Tokenize [text] into a list of token IDs padded/truncated to [maxTokens]. + /// + /// Layout: `[CLS] token_ids... [SEP] [PAD]...` + List tokenize(String text) { + final cleaned = _cleanText(text.toLowerCase()); + final basicTokens = _basicTokenize(cleaned); + + final ids = [clsId]; + for (final word in basicTokens) { + final pieces = _wordPiece(word); + + if (ids.length + pieces.length >= maxTokens) { + ids.addAll(pieces.take(maxTokens - ids.length - 1)); + break; + } + ids.addAll(pieces); + } + ids.add(sepId); + + while (ids.length < maxTokens) { + ids.add(padId); + } + + return ids; + } + + /// Remove control characters and normalize whitespace. + String _cleanText(String text) { + final buf = StringBuffer(); + for (final char in text.characters) { + final cp = char.codeUnitAt(0); + if (cp == 0 || cp == 0xFFFD || _isControlChar(cp)) continue; + buf.write(_isWhitespace(cp) ? ' ' : char); + } + return buf.toString(); + } + + /// Split on whitespace and punctuation to produce basic tokens. + List _basicTokenize(String text) { + final tokens = []; + final buf = StringBuffer(); + for (final char in text.characters) { + final cp = char.codeUnitAt(0); + if (_isWhitespace(cp)) { + if (buf.isNotEmpty) { + tokens.add(buf.toString()); + buf.clear(); + } + } else if (_isPunctuation(cp)) { + if (buf.isNotEmpty) { + tokens.add(buf.toString()); + buf.clear(); + } + tokens.add(char); + } else { + buf.write(char); + } + } + if (buf.isNotEmpty) tokens.add(buf.toString()); + return tokens; + } + + /// WordPiece sub-word tokenization for a single [word]. + /// + /// Returns `[unkId]` if no valid segmentation exists. + List _wordPiece(String word) { + if (word.isEmpty) return []; + if (_vocab.containsKey(word)) return [_vocab[word]!]; + + final result = []; + var start = 0; + + while (start < word.length) { + var end = word.length; + int? foundId; + int? foundLen; + + while (start < end) { + final sub = start == 0 ? word.substring(0, end) : '##${word.substring(start, end)}'; + if (_vocab.containsKey(sub)) { + foundId = _vocab[sub]!; + foundLen = end - start; + break; + } + end--; + } + + if (foundId == null) return [unkId]; + + result.add(foundId); + start += foundLen!; + } + + return result; + } + + bool _isWhitespace(int cp) => cp == 0x20 || cp == 0x09 || cp == 0x0A || cp == 0x0D; + + bool _isControlChar(int cp) => (cp < 0x20 && !_isWhitespace(cp)) || (cp >= 0x7F && cp <= 0x9F); + + bool _isPunctuation(int cp) => + (cp >= 33 && cp <= 47) || + (cp >= 58 && cp <= 64) || + (cp >= 91 && cp <= 96) || + (cp >= 123 && cp <= 126) || + (cp >= 0x2000 && cp <= 0x206F) || + (cp >= 0x2E00 && cp <= 0x2E7F) || + (cp >= 0x3000 && cp <= 0x303F); +} diff --git a/lib/features/search/data/post_text_extractor.dart b/lib/features/search/data/post_text_extractor.dart new file mode 100644 index 0000000..019b6eb --- /dev/null +++ b/lib/features/search/data/post_text_extractor.dart @@ -0,0 +1,62 @@ +import 'package:bluesky/app_bsky_embed_recordwithmedia.dart'; +import 'package:bluesky/app_bsky_feed_defs.dart'; +import 'package:bluesky/app_bsky_feed_post.dart'; + +/// Extracts a single searchable string from a [PostView] for embedding. +/// +/// Concatenates (in order, separated by spaces): +/// 1. Post body text +/// 2. Alt-text from every image in an images embed +/// 3. Title + description from an external link-card embed +/// +/// Returns an empty string if no text can be extracted. +class PostTextExtractor { + const PostTextExtractor(); + + String extract(PostView post) { + final parts = []; + + final recordText = _recordText(post.record); + if (recordText.isNotEmpty) parts.add(recordText); + + final embed = post.embed; + if (embed != null) { + if (embed.isEmbedImagesView) { + for (final image in embed.embedImagesView!.images) { + final alt = image.alt.trim(); + if (alt.isNotEmpty) parts.add(alt); + } + } else if (embed.isEmbedExternalView) { + final external = embed.embedExternalView!.external; + final title = external.title.trim(); + if (title.isNotEmpty) parts.add(title); + final desc = external.description.trim(); + if (desc.isNotEmpty) parts.add(desc); + } else if (embed.isEmbedRecordWithMediaView) { + final media = embed.embedRecordWithMediaView!.media; + if (media.isEmbedImagesView) { + for (final image in media.embedImagesView!.images) { + final alt = image.alt.trim(); + if (alt.isNotEmpty) parts.add(alt); + } + } else if (media.isEmbedExternalView) { + final external = media.embedExternalView!.external; + final title = external.title.trim(); + if (title.isNotEmpty) parts.add(title); + final desc = external.description.trim(); + if (desc.isNotEmpty) parts.add(desc); + } + } + } + + return parts.join(' '); + } + + String _recordText(Map record) { + try { + return FeedPostRecord.fromJson(record).text.trim(); + } catch (_) { + return ''; + } + } +} diff --git a/test/core/embedding/embedding_service_test.dart b/test/core/embedding/embedding_service_test.dart new file mode 100644 index 0000000..721db16 --- /dev/null +++ b/test/core/embedding/embedding_service_test.dart @@ -0,0 +1,156 @@ +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:lazurite/core/embedding/embedding_service.dart'; + +void main() { + group('l2Normalize', () { + test('unit vector is unchanged', () { + final v = Float32List.fromList([1.0, 0.0, 0.0]); + final result = l2Normalize(v); + expect(result[0], closeTo(1.0, 1e-6)); + expect(result[1], closeTo(0.0, 1e-6)); + expect(result[2], closeTo(0.0, 1e-6)); + }); + + test('scales vector to unit length', () { + final v = Float32List.fromList([3.0, 4.0]); + final result = l2Normalize(v); + // norm = 5; normalised = [0.6, 0.8] + expect(result[0], closeTo(0.6, 1e-6)); + expect(result[1], closeTo(0.8, 1e-6)); + }); + + test('result has norm ≈ 1', () { + final v = Float32List.fromList(List.generate(384, (i) => (i + 1).toDouble())); + final result = l2Normalize(v); + var norm = 0.0; + for (final x in result) { + norm += x * x; + } + expect(norm, closeTo(1.0, 1e-5)); + }); + + test('near-zero vector is returned unchanged (no division by zero)', () { + final v = Float32List.fromList([0.0, 0.0, 0.0]); + final result = l2Normalize(v); + expect(result, equals(v)); + }); + + test('returns a new list, does not mutate input', () { + final v = Float32List.fromList([3.0, 4.0]); + l2Normalize(v); + expect(v[0], equals(3.0)); + expect(v[1], equals(4.0)); + }); + }); + + group('EmbeddingService', () { + group('initial state', () { + test('isAvailable is false before initialize', () { + final service = EmbeddingService.forTesting((_) async => Float32List(384)); + expect(service.isAvailable, isFalse); + }); + }); + + group('initialize / dispose', () { + test('isAvailable is true after initialize with mock backend', () async { + final service = EmbeddingService.forTesting((_) async => Float32List(384)); + await service.initialize(); + expect(service.isAvailable, isTrue); + }); + + test('initialize is idempotent', () async { + var calls = 0; + final service = EmbeddingService.forTesting((_) async { + calls++; + return Float32List(384); + }); + await service.initialize(); + await service.initialize(); // second call should be a no-op + expect(service.isAvailable, isTrue); + // Idempotency check: embed once to confirm service still works. + await service.embed('test'); + expect(calls, equals(1)); // embed was called once, not initialize twice + }); + + test('dispose resets isAvailable to false', () async { + final service = EmbeddingService.forTesting((_) async => Float32List(384)); + await service.initialize(); + service.dispose(); + expect(service.isAvailable, isFalse); + }); + + test('dispose before initialize does not throw', () { + final service = EmbeddingService.forTesting((_) async => Float32List(384)); + expect(() => service.dispose(), returnsNormally); + }); + + test('dispose can be called multiple times safely', () async { + final service = EmbeddingService.forTesting((_) async => Float32List(384)); + await service.initialize(); + service.dispose(); + expect(() => service.dispose(), returnsNormally); + }); + }); + + group('embed', () { + test('throws StateError when not initialized', () async { + final service = EmbeddingService.forTesting((_) async => Float32List(384)); + expect(() => service.embed('hello'), throwsStateError); + }); + + test('throws StateError after dispose', () async { + final service = EmbeddingService.forTesting((_) async => Float32List(384)); + await service.initialize(); + service.dispose(); + expect(() => service.embed('hello'), throwsStateError); + }); + + test('returns the value produced by the mock backend', () async { + final expected = Float32List.fromList(List.generate(384, (i) => i.toDouble())); + final service = EmbeddingService.forTesting((_) async => expected); + await service.initialize(); + + final result = await service.embed('some text'); + expect(result, equals(expected)); + }); + + test('result has length 384', () async { + final service = EmbeddingService.forTesting((_) async => Float32List(384)); + await service.initialize(); + + final result = await service.embed('hello world'); + expect(result.length, equals(384)); + }); + + test('forwards the exact text to the backend', () async { + String? received; + final service = EmbeddingService.forTesting((text) async { + received = text; + return Float32List(384); + }); + await service.initialize(); + + await service.embed('the quick brown fox'); + expect(received, equals('the quick brown fox')); + }); + + test('multiple concurrent embeds each receive correct results', () async { + var callCount = 0; + final service = EmbeddingService.forTesting((text) async { + callCount++; + final v = Float32List(384); + v[0] = callCount.toDouble(); + return v; + }); + await service.initialize(); + + final results = await Future.wait([service.embed('a'), service.embed('b'), service.embed('c')]); + + expect(results.length, equals(3)); + expect(results.every((r) => r.length == 384), isTrue); + }); + }); + }); +} diff --git a/test/core/embedding/word_piece_tokenizer_test.dart b/test/core/embedding/word_piece_tokenizer_test.dart new file mode 100644 index 0000000..47f647d --- /dev/null +++ b/test/core/embedding/word_piece_tokenizer_test.dart @@ -0,0 +1,171 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:lazurite/core/embedding/word_piece_tokenizer.dart'; + +/// Minimal synthetic vocabulary for fast, isolated tests. +/// Each line is one token; line number (0-indexed) is its ID. +/// +/// Mapping: ID +/// [PAD] (padId) = 0 +/// ##a = 1 +/// ##b = 2 +/// ##c = 3 +/// ##bc = 4 +/// ##un = 5 +/// ab = 6 +/// abc = 7 +/// hello = 8 +/// world = 9 +/// [UNK] (unkId = 100) → not in the 10-token vocab, so words with no +/// sub-token match will return [UNK]=100 from +/// the full-vocab constant, but for a synthetic +/// vocab the 100 will be the literal constant. +/// +/// We embed [PAD] × 90 filler entries between index 10 and 99 so that +/// unkId=100, clsId=101, sepId=102 fall at the expected positions. +String _buildVocab() { + final lines = []; + lines.add('[PAD]'); + lines.add('##a'); + lines.add('##b'); + lines.add('##c'); + lines.add('##bc'); + lines.add('##un'); + lines.add('ab'); + lines.add('abc'); + lines.add('hello'); + lines.add('world'); + for (var i = 10; i < 100; i++) { + lines.add('[unused_${i}_]'); + } + + lines.add('[UNK]'); + lines.add('[CLS]'); + lines.add('[SEP]'); + return lines.join('\n'); +} + +final _vocab = _buildVocab(); + +void main() { + late WordPieceTokenizer tokenizer; + + setUp(() { + tokenizer = WordPieceTokenizer.fromString(_vocab); + }); + + group('WordPieceTokenizer', () { + group('structure', () { + test('always returns exactly maxTokens (256) elements', () { + final result = tokenizer.tokenize('hello'); + expect(result.length, equals(WordPieceTokenizer.maxTokens)); + }); + + test('first token is always CLS (101)', () { + expect(tokenizer.tokenize('hello').first, equals(WordPieceTokenizer.clsId)); + expect(tokenizer.tokenize('').first, equals(WordPieceTokenizer.clsId)); + }); + + test('empty string → [CLS, SEP, PAD, PAD, ...]', () { + final result = tokenizer.tokenize(''); + expect(result[0], equals(WordPieceTokenizer.clsId)); + expect(result[1], equals(WordPieceTokenizer.sepId)); + expect(result.sublist(2), everyElement(equals(WordPieceTokenizer.padId))); + }); + + test('padding fills remaining indices after [SEP] with PAD (0)', () { + final result = tokenizer.tokenize('hello'); + expect(result.sublist(3), everyElement(equals(WordPieceTokenizer.padId))); + }); + }); + + group('whole-word tokens', () { + test('known word → its vocab ID', () { + final result = tokenizer.tokenize('hello'); + expect(result[1], equals(8)); + expect(result[2], equals(WordPieceTokenizer.sepId)); + }); + + test('two known words → two IDs between CLS and SEP', () { + final result = tokenizer.tokenize('hello world'); + expect(result[0], equals(WordPieceTokenizer.clsId)); + expect(result[1], equals(8)); + expect(result[2], equals(9)); + expect(result[3], equals(WordPieceTokenizer.sepId)); + }); + }); + + group('wordpiece sub-word splitting', () { + test('word not in vocab but has valid subword decomposition', () { + final result = tokenizer.tokenize('abc'); + expect(result[1], equals(7)); + }); + + test('sub-word fallback: ab + ##c → [6, 3]', () { + final result = tokenizer.tokenize('abbc'); + expect(result[1], equals(6)); // "ab" + expect(result[2], equals(4)); // "##bc" + expect(result[3], equals(WordPieceTokenizer.sepId)); + }); + + test('word with no valid sub-token decomposition → UNK', () { + final result = tokenizer.tokenize('xyz'); + expect(result[1], equals(WordPieceTokenizer.unkId)); + expect(result[2], equals(WordPieceTokenizer.sepId)); + }); + }); + + group('truncation', () { + test('very long text is truncated to maxTokens with SEP preserved', () { + final longText = ('hello ' * 300).trim(); + final result = tokenizer.tokenize(longText); + expect(result.length, equals(WordPieceTokenizer.maxTokens)); + + final lastNonPad = result.lastIndexWhere((id) => id != WordPieceTokenizer.padId); + expect(result[lastNonPad], equals(WordPieceTokenizer.sepId)); + }); + + test('text of exactly maxTokens - 2 content tokens fits without truncation', () { + final words = List.filled(254, 'hello'); + final result = tokenizer.tokenize(words.join(' ')); + expect(result[0], equals(WordPieceTokenizer.clsId)); + expect(result[255], equals(WordPieceTokenizer.sepId)); + expect(result.contains(WordPieceTokenizer.padId), isFalse); + }); + }); + + group('case', () { + test('input is case-folded to lowercase before tokenisation', () { + final lower = tokenizer.tokenize('hello'); + final upper = tokenizer.tokenize('HELLO'); + expect(lower, equals(upper)); + }); + }); + + group('punctuation', () { + test('punctuation is split as individual tokens', () { + final result = tokenizer.tokenize('hello,world'); + expect(result[0], equals(WordPieceTokenizer.clsId)); + expect(result[1], equals(8)); + expect(result[2], equals(WordPieceTokenizer.unkId)); + expect(result[3], equals(9)); + expect(result[4], equals(WordPieceTokenizer.sepId)); + }); + }); + + group('fromString factory', () { + test('handles Windows-style CRLF line endings', () { + final crlfVocab = _buildVocab().replaceAll('\n', '\r\n'); + final crlfTokenizer = WordPieceTokenizer.fromString(crlfVocab); + final result = crlfTokenizer.tokenize('hello'); + expect(result[1], equals(8)); + }); + + test('skips blank trailing line if present', () { + final withTrailing = '${_buildVocab()}\n'; + final t = WordPieceTokenizer.fromString(withTrailing); + final result = t.tokenize('hello'); + expect(result[1], equals(8)); + }); + }); + }); +} diff --git a/test/features/search/data/post_text_extractor_test.dart b/test/features/search/data/post_text_extractor_test.dart new file mode 100644 index 0000000..537a7b4 --- /dev/null +++ b/test/features/search/data/post_text_extractor_test.dart @@ -0,0 +1,207 @@ +import 'package:atproto_core/atproto_core.dart'; +import 'package:bluesky/app_bsky_actor_defs.dart'; +import 'package:bluesky/app_bsky_embed_external.dart'; +import 'package:bluesky/app_bsky_embed_images.dart'; +import 'package:bluesky/app_bsky_embed_record.dart'; +import 'package:bluesky/app_bsky_embed_recordwithmedia.dart'; +import 'package:bluesky/app_bsky_feed_defs.dart'; +import 'package:bluesky/app_bsky_feed_post.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:lazurite/features/search/data/post_text_extractor.dart'; + +const _author = ProfileViewBasic(did: 'did:plc:test', handle: 'test.bsky.social'); +final _uri = AtUri.parse('at://did:plc:test/app.bsky.feed.post/xyz'); + +PostView _post({String text = '', UPostViewEmbed? embed}) { + final record = FeedPostRecord(text: text, createdAt: DateTime.utc(2026, 1, 1)); + return PostView( + uri: _uri, + cid: 'cid-test', + author: _author, + record: record.toJson(), + indexedAt: DateTime.utc(2026, 1, 1), + embed: embed, + ); +} + +UPostViewEmbed _imagesEmbed(List altTexts) { + final images = altTexts + .map( + (alt) => EmbedImagesViewImage( + thumb: 'https://example.com/thumb.jpg', + fullsize: 'https://example.com/full.jpg', + alt: alt, + aspectRatio: null, + ), + ) + .toList(); + return UPostViewEmbed.embedImagesView(data: EmbedImagesView(images: images)); +} + +UPostViewEmbed _externalEmbed({ + required String title, + required String description, + String uri = 'https://example.com', +}) { + return UPostViewEmbed.embedExternalView( + data: EmbedExternalView( + external: EmbedExternalViewExternal(uri: uri, title: title, description: description), + ), + ); +} + +UPostViewEmbed _recordWithImagesEmbed(String postText, List altTexts) { + return UPostViewEmbed.embedRecordWithMediaView( + data: EmbedRecordWithMediaView( + record: const EmbedRecordView(record: UEmbedRecordViewRecord.unknown(data: {})), + media: UEmbedRecordWithMediaViewMedia.embedImagesView( + data: EmbedImagesView( + images: altTexts + .map( + (alt) => EmbedImagesViewImage( + thumb: 'https://example.com/thumb.jpg', + fullsize: 'https://example.com/full.jpg', + alt: alt, + aspectRatio: null, + ), + ) + .toList(), + ), + ), + ), + ); +} + +UPostViewEmbed _recordWithExternalEmbed(String postText, {required String title, required String description}) { + return UPostViewEmbed.embedRecordWithMediaView( + data: EmbedRecordWithMediaView( + record: const EmbedRecordView(record: UEmbedRecordViewRecord.unknown(data: {})), + media: UEmbedRecordWithMediaViewMedia.embedExternalView( + data: EmbedExternalView( + external: EmbedExternalViewExternal(uri: 'https://example.com', title: title, description: description), + ), + ), + ), + ); +} + +void main() { + late PostTextExtractor extractor; + + setUp(() { + extractor = const PostTextExtractor(); + }); + + group('PostTextExtractor', () { + group('text-only posts', () { + test('returns the post body text', () { + final post = _post(text: 'Hello world'); + expect(extractor.extract(post), equals('Hello world')); + }); + + test('trims surrounding whitespace from post text', () { + final post = _post(text: ' trimmed '); + expect(extractor.extract(post), equals('trimmed')); + }); + + test('returns empty string for a post with no text and no embed', () { + final post = _post(text: ''); + expect(extractor.extract(post), equals('')); + }); + }); + + group('image embeds', () { + test('appends alt texts to post text', () { + final post = _post(text: 'Check this out', embed: _imagesEmbed(['a cat', 'a dog'])); + expect(extractor.extract(post), equals('Check this out a cat a dog')); + }); + + test('skips images with empty alt text', () { + final post = _post(text: 'Photo', embed: _imagesEmbed(['', 'nice view', ''])); + expect(extractor.extract(post), equals('Photo nice view')); + }); + + test('handles all-blank alt texts gracefully', () { + final post = _post(text: 'Silent', embed: _imagesEmbed(['', ' '])); + expect(extractor.extract(post), equals('Silent')); + }); + + test('returns only alt texts when post text is empty', () { + final post = _post(embed: _imagesEmbed(['sunset photo'])); + expect(extractor.extract(post), equals('sunset photo')); + }); + }); + + group('external link-card embeds', () { + test('appends title and description to post text', () { + final post = _post( + text: 'Read this', + embed: _externalEmbed(title: 'Great Article', description: 'Very informative'), + ); + expect(extractor.extract(post), equals('Read this Great Article Very informative')); + }); + + test('omits empty title', () { + final post = _post( + text: 'Link', + embed: _externalEmbed(title: '', description: 'A description'), + ); + expect(extractor.extract(post), equals('Link A description')); + }); + + test('omits empty description', () { + final post = _post( + text: 'Link', + embed: _externalEmbed(title: 'Title', description: ''), + ); + expect(extractor.extract(post), equals('Link Title')); + }); + + test('returns only title+description when post text is empty', () { + final post = _post( + embed: _externalEmbed(title: 'My Title', description: 'My Desc'), + ); + expect(extractor.extract(post), equals('My Title My Desc')); + }); + }); + + group('record-with-media embeds (images)', () { + test('appends image alt texts from media component', () { + final post = _post(text: 'With quote', embed: _recordWithImagesEmbed('', ['alt one', 'alt two'])); + expect(extractor.extract(post), equals('With quote alt one alt two')); + }); + + test('skips empty alt texts in record-with-media', () { + final post = _post(text: 'Post', embed: _recordWithImagesEmbed('', ['', 'valid'])); + expect(extractor.extract(post), equals('Post valid')); + }); + }); + + group('record-with-media embeds (external)', () { + test('appends title and description from media external', () { + final post = _post( + text: 'Quoting with link', + embed: _recordWithExternalEmbed('', title: 'Card Title', description: 'Card Desc'), + ); + expect(extractor.extract(post), equals('Quoting with link Card Title Card Desc')); + }); + }); + + group('combinations', () { + test('multiple image alt texts are space-separated between them', () { + final post = _post(embed: _imagesEmbed(['first', 'second', 'third'])); + expect(extractor.extract(post), equals('first second third')); + }); + + test('produces a single space-joined string with no leading or trailing space', () { + final post = _post( + text: 'Body', + embed: _externalEmbed(title: 'T', description: 'D'), + ); + final result = extractor.extract(post); + expect(result.startsWith(' '), isFalse); + expect(result.endsWith(' '), isFalse); + }); + }); + }); +} -- 2.51.2