mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-08-28 05:33:24 +00:00
stumpylogandClaude Opus 5
4e2d71513a
_build_cjk_query scanned the raw query string for CJK runs, so a CJK term
the user negated ('invoice NOT 漢字') or restricted to one field
('title:漢字', 'notes:漢字') came straight back as a top-level Should
clause over every bigram field. The fuzzy clause already collects its
words from the parsed tree for exactly this reason; the CJK clause a few
lines below did not.
Collect the CJK runs from whoosh_compat's free_text_tokens over
result.ast instead, one default field at a time so the tokens keep their
field attribution: a bare term (already copied onto every default field
by the parser) still searches every bigram field, while title:東京
reaches bigram_title alone, and a term on a non-default field
contributes nothing. Fields sharing identical CJK text share one parse.
The raw-string builder stays for the simple TEXT/TITLE modes, whose
input is plain text with no query grammar to respect, as does
extract_cjk_text, which the indexing side calls per bigram field.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
113 lines
3.6 KiB
Python
113 lines
3.6 KiB
Python
"""The CJK bigram clause blended into QUERY-mode searches.
|
|
|
|
The clause exists so CJK runs are matchable at all (the default analyzers
|
|
keep a whitespace-free CJK run as one indivisible token), but it must not
|
|
widen the query beyond what the user asked for: a CJK term the query
|
|
excludes, or restricts to one field, must not come back through it.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import TYPE_CHECKING
|
|
|
|
import pytest
|
|
|
|
from documents.models import Document
|
|
|
|
if TYPE_CHECKING:
|
|
from documents.search._backend import TantivyBackend
|
|
|
|
pytestmark = [pytest.mark.search, pytest.mark.django_db]
|
|
|
|
|
|
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
|
|
return set(backend.search_ids(query, user=None))
|
|
|
|
|
|
def _index(backend: TantivyBackend, **kwargs: object) -> Document:
|
|
doc = Document.objects.create(**kwargs)
|
|
backend.add_or_update(doc)
|
|
return doc
|
|
|
|
|
|
class TestCjkClauseFollowsTheParsedQuery:
|
|
def test_negated_cjk_term_is_excluded(self, backend: TantivyBackend) -> None:
|
|
"""'invoice NOT 漢字' must not return the document containing 漢字."""
|
|
with_cjk = _index(
|
|
backend,
|
|
title="Invoice A",
|
|
content="invoice total 漢字",
|
|
checksum="cjk-neg-1",
|
|
)
|
|
without_cjk = _index(
|
|
backend,
|
|
title="Invoice B",
|
|
content="invoice total only",
|
|
checksum="cjk-neg-2",
|
|
)
|
|
|
|
assert _matched_ids(backend, "invoice") == {with_cjk.pk, without_cjk.pk}
|
|
assert _matched_ids(backend, "invoice NOT 漢字") == {without_cjk.pk}
|
|
|
|
def test_fielded_cjk_term_searches_only_that_field(
|
|
self,
|
|
backend: TantivyBackend,
|
|
) -> None:
|
|
"""'title:東京' must not match a document whose 東京 is in the content."""
|
|
content_only = _index(
|
|
backend,
|
|
title="Tokyo report",
|
|
content="東京都の人口は約1400万人です",
|
|
checksum="cjk-field-1",
|
|
)
|
|
titled = _index(
|
|
backend,
|
|
title="東京都の報告書",
|
|
content="an english summary",
|
|
checksum="cjk-field-2",
|
|
)
|
|
|
|
assert _matched_ids(backend, "東京") == {content_only.pk, titled.pk}
|
|
assert _matched_ids(backend, "title:東京") == {titled.pk}
|
|
|
|
def test_cjk_on_a_non_default_field_builds_no_clause(
|
|
self,
|
|
backend: TantivyBackend,
|
|
) -> None:
|
|
"""A CJK term restricted to a field outside the default search fields
|
|
has nothing to contribute to the bigram clause: 'notes:東京' must not
|
|
fall back to matching 東京 in the content."""
|
|
_index(
|
|
backend,
|
|
title="Tokyo report",
|
|
content="東京都の人口は約1400万人です",
|
|
checksum="cjk-notes-1",
|
|
)
|
|
|
|
assert _matched_ids(backend, "notes:東京") == set()
|
|
|
|
def test_bare_cjk_term_still_matches_every_default_field(
|
|
self,
|
|
backend: TantivyBackend,
|
|
) -> None:
|
|
"""The clause's reason for existing: an unfielded CJK run matches
|
|
wherever it is indexed, and does so alongside a latin term."""
|
|
in_content = _index(
|
|
backend,
|
|
title="report",
|
|
content="本文に重要な情報",
|
|
checksum="cjk-bare-1",
|
|
)
|
|
in_title = _index(
|
|
backend,
|
|
title="重要な報告書",
|
|
content="english only",
|
|
checksum="cjk-bare-2",
|
|
)
|
|
|
|
assert _matched_ids(backend, "重要") == {in_content.pk, in_title.pk}
|
|
assert _matched_ids(backend, "重要 OR report") == {
|
|
in_content.pk,
|
|
in_title.pk,
|
|
}
|