mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-11-21 13:02:40 +00:00
Feature: match CJK terms through their bigram fields in place (#14156)
QUERY-mode searches blended a separate bigram clause in at the top of the query, built from the parsed AST's free-text tokens. Because it sat beside the exact clause rather than inside the query, nothing around a CJK term constrained its bigram match: an exclusion that was one OR branch's own condition could never reach it, so "(東京 AND NOT secret) OR bill" still returned the secret document. Widen each CJK leaf where it sits instead, through emit()'s rewrite_leaf hook, so every AND, NOT, REQUIRE, boost and field restriction around the leaf applies to its bigram match too. Negated leaves are widened on purpose, so "NOT X" excludes exactly what "X" matches.
This commit is contained in:
1 parent
420bf503e8
commit
762e8cf4d1
16 files changed
+2019
-252
No files matched your search
@@ -6,13 +6,17 @@ import pytest
|
||||
|
||||
from documents.search._backend import TantivyBackend
|
||||
from documents.search._backend import reset_backend
|
||||
from documents.tests.factories import DocumentFactory
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Generator
|
||||
from pathlib import Path
|
||||
|
||||
from pytest_django.fixtures import Settings
|
||||
|
||||
from documents.models import Document
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def index_dir(tmp_path: Path, settings: Settings) -> Path:
|
||||
@@ -31,3 +35,40 @@ def backend() -> Generator[TantivyBackend, None, None]:
|
||||
finally:
|
||||
b.close()
|
||||
reset_backend()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def long_cjk_run() -> str:
|
||||
"""A CJK run too long for the content analyzer to keep.
|
||||
|
||||
48 characters, 144 UTF-8 bytes: one token longer than remove_long's
|
||||
129-byte limit, so the content side analyzes to nothing and only the
|
||||
bigram field can match it.
|
||||
"""
|
||||
return "東京都の公共文書について" * 4
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def index_document(backend: TantivyBackend) -> Callable[..., Document]:
|
||||
"""Build a Document with DocumentFactory and add it to the index.
|
||||
|
||||
The factory supplies a unique checksum, so a test only passes the
|
||||
fields its assertion actually depends on.
|
||||
"""
|
||||
|
||||
def _index(**kwargs: object) -> Document:
|
||||
doc = DocumentFactory(**kwargs)
|
||||
backend.add_or_update(doc)
|
||||
return doc
|
||||
|
||||
return _index
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def matched_ids(backend: TantivyBackend) -> Callable[[str], set[int]]:
|
||||
"""Run a query as no particular user and return the matching pks."""
|
||||
|
||||
def _matched_ids(query: str) -> set[int]:
|
||||
return set(backend.search_ids(query, user=None))
|
||||
|
||||
return _matched_ids
|
||||
Reference in new issue
Block a user