amorfo77/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2026-08-21 02:03:30 +00:00
Code Issues Projects Releases Wiki Activity

feat(search): add whoosh-compat FieldRegistry construction

Browse Source
This commit is contained in:
Trenton Holmes
2026-08-19 13:36:53 -07:00
committed by stumpylog
parent b30c7f42cb
commit 2c6f89bab1
4 changed files with 190 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/documents/search/_tokenizer.py
+2 -2
View File
@@ -71,7 +71,7 @@ def register_tokenizers(index: tantivy.Index, language: str | None) -> None:
use fast=True and Tantivy requires fast-field tokenizers to exist
even for documents that omit those fields.
"""
index.register_tokenizer("paperless_text", _paperless_text(language))
index.register_tokenizer("paperless_text", paperless_text_analyzer(language))
index.register_tokenizer("simple_analyzer", _simple_analyzer())
index.register_tokenizer("bigram_analyzer", _bigram_analyzer())
index.register_tokenizer("simple_search_analyzer", _simple_search_analyzer())
@@ -79,7 +79,7 @@ def register_tokenizers(index: tantivy.Index, language: str | None) -> None:
index.register_fast_field_tokenizer("simple_analyzer", _simple_analyzer())
def _paperless_text(language: str | None) -> tantivy.TextAnalyzer:
def paperless_text_analyzer(language: str | None) -> tantivy.TextAnalyzer:
"""Main full-text tokenizer for content, title, etc: simple -> remove_long(129) -> lowercase -> ascii_fold [-> stemmer]"""
builder = (
tantivy.TextAnalyzerBuilder(tantivy.Tokenizer.simple())