diff --git a/owilix/plugins/ngram/__init__.py b/owilix/plugins/ngram/__init__.py index 5ceab29..ccfb93a 100644 --- a/owilix/plugins/ngram/__init__.py +++ b/owilix/plugins/ngram/__init__.py @@ -4,6 +4,7 @@ Plugin module for creating n-gram filters, n-gram analysis and n-gram querying. Usage: owilix plugin owilix.plugins.ngram.BloomNGrams create (all:latest/collectionName=main) (**/language=eng/metadata_*.parquet) tokenizer=mistralai/Mixtral-8x7B-v0.1 +owilix plugin owilix.plugins.ngram.BloomNGrams query (all:latest/collectionName=main) ./owilix/plugins/ngram/query-test.jsonl (**/language=eng/*.bloom) The bloom filter takes a sub-word tokenizer and creates character tokenizer n-grams, stored in a bloom filter (rbloom). The bloom filter is constructed for a group of files. @@ -22,11 +23,12 @@ Examples using the mixtral tokenizer: | 785858 | 9585058384 | 0.001242 | 2772.066 | 1142.628 | 632796133.446 | 2.4260 | | 1291984 | 9585058384 | 0.001375 | 4564.114 | 1142.628 | 940008906.542 | 3.9944 | | 2342451 | 47925291888 | 0.001562 | 8282.486 | 5713.140 | 1499167985.877 | 1.4497 | +| 9085789 | 47925291888 | 0.00019 | 32163,812 | 5713.140 | 4572710230.4186 | 5.629 | ## Efficiency: - +- currently around 305 rows/second on a powerful desktop machine """ diff --git a/owilix/plugins/ngram/bloom_ngram.py b/owilix/plugins/ngram/bloom_ngram.py index 8f309de..33ff889 100644 --- a/owilix/plugins/ngram/bloom_ngram.py +++ b/owilix/plugins/ngram/bloom_ngram.py @@ -22,6 +22,9 @@ transformers_logging.set_verbosity_error() from owilix.cmd.base import BaseCommand, SubCommand, currentItemProgress +def _hash(s: str) -> int: + # Use xxhash's XXH3_64bits to hash the string and return a 64-bit integer + return xxhash.xxh3_64(s).intdigest() class BloomNGrams(BaseCommand): @@ -181,9 +184,7 @@ class NGramCreator: None """ # Lets count in hll. - def _hash(s: str) -> int: - # Use xxhash's XXH3_64bits to hash the string and return a 64-bit integer - return xxhash.xxh3_64(s).intdigest() + bloom_filter = Bloom(self.bloom_filter_size, self.bloom_error_rate, _hash) total_record_count = 0 @@ -383,7 +384,7 @@ class BloomFilterProcessor: """ results = {} for ngram in ngrams: - results[ngram] = bloom_filter.check(ngram) + results[ngram] = ngram in bloom_filter return results def summarize_results(self, text, ngrams, bloom_results): @@ -429,7 +430,7 @@ class BloomFilterProcessor: ngram_max = summary_data['ngram_max'] # Load Bloom filter - bloom_filter = Bloom.load(bloom_file) + bloom_filter = Bloom.load(bloom_file,_hash ) for item in self.data: text_id = item['id'] diff --git a/owilix/plugins/ngram/query-test.jsonl b/owilix/plugins/ngram/query-test.jsonl new file mode 100644 index 0000000..ca85c35 --- /dev/null +++ b/owilix/plugins/ngram/query-test.jsonl @@ -0,0 +1,2 @@ +{"id": 1, "text": "This website contains cookies. Cookies are healthy"} +{"id": 2, "text": "We also have privacy policies. Navigate through the menu items."} \ No newline at end of file