diff --git a/src/documents/search/_query.py b/src/documents/search/_query.py index f62d451c8..87c84d242 100644 --- a/src/documents/search/_query.py +++ b/src/documents/search/_query.py @@ -190,12 +190,28 @@ def _build_ast_cjk_query( # tantivy's own query parser, and the raw query text the clause collects # routinely carries characters that parser reads as grammar (a colon, a # bracket, a quote, a leading -). Each token is cut into its word runs and -# only those are kept, so nothing injectable can reach the parser. Cutting -# rather than dropping the whole token is what keeps ordinary hyphenated, -# dotted and quoted input ("COVID-19", "hello@example.com", "tax reports") -# contributing to the clause at all. +# only those are kept, so no field syntax, pattern, range or grouping can +# reach the parser. Cutting rather than dropping the whole token is what +# keeps ordinary hyphenated, dotted and quoted input ("COVID-19", +# "hello@example.com", "tax reports") contributing to the clause at all. _WORD_RUN_RE = regex.compile(r"\w+") +# The one piece of tantivy grammar that survives the cut: its boolean +# keywords are themselves word runs. Only these exact spellings are +# grammar there ("And"/"and" are ordinary terms), so lowercasing exactly +# these turns them back into the ordinary terms the field analyzer used to +# make of them, before the clause switched to raw text. Left alone, a +# quoted phrase would silently restructure the clause ("tax AND reports" +# becoming a conjunction) or fail to parse and drop it entirely +# ("tax AND", or "IN" anywhere). +# +# Only these words are touched: tantivy lowercases query terms with the +# field's own analyzer, and doing it ourselves first is not always the +# same operation (Python folds a final sigma to a different letter than +# tantivy does, and turns Turkish 'İ' into a sequence tantivy then splits +# in two), which would search for terms the index does not contain. +_TANTIVY_KEYWORDS: Final[frozenset[str]] = frozenset({"AND", "OR", "NOT", "IN"}) + def _try_parse_fuzzy_query( index: tantivy.Index, @@ -223,7 +239,10 @@ def _try_parse_fuzzy_query( default fields rather than just the one the user named. That is recall-only widening on a secondary 0.1-boosted clause the score threshold already disciplines, accepted in exchange for never feeding - field syntax to tantivy's parser. + field syntax to tantivy's parser. What the word string guarantees is + exactly that: no field prefix, pattern, range, grouping or quoting + survives, and the boolean keywords that do survive (they are word + runs) are lowercased into ordinary terms; see _TANTIVY_KEYWORDS. The words are the query's RAW text, not the analyzer's output (``analyzed=False``), because ``index.parse_query`` analyzes whatever @@ -246,7 +265,11 @@ def _try_parse_fuzzy_query( analyzed=False, ) words = list( - dict.fromkeys(word for token in tokens for word in _WORD_RUN_RE.findall(token)), + dict.fromkeys( + word.lower() if word in _TANTIVY_KEYWORDS else word + for token in tokens + for word in _WORD_RUN_RE.findall(token) + ), ) if not words: return None @@ -468,7 +491,7 @@ def parse_user_query( # plain Shoulds beside the exact clause they re-admit exactly the # documents the query excluded. Restate the exclusions once, above the # whole blend. Redundant against the exact clause, which already - # carries them, but idempotently so, and cheaper than stripping them. + # carries them, but idempotently so. negations = _negation_clauses(index, result.ast, registry) if not negations: return _any_of(clauses) diff --git a/src/documents/tests/search/test_cjk_clause.py b/src/documents/tests/search/test_cjk_clause.py index c8ee659df..ebd907fe9 100644 --- a/src/documents/tests/search/test_cjk_clause.py +++ b/src/documents/tests/search/test_cjk_clause.py @@ -15,6 +15,8 @@ import pytest from documents.models import Document if TYPE_CHECKING: + from pytest_django.fixtures import SettingsWrapper + from documents.search._backend import TantivyBackend pytestmark = [pytest.mark.search, pytest.mark.django_db] @@ -49,11 +51,29 @@ class TestCjkClauseFollowsTheParsedQuery: assert _matched_ids(backend, "invoice") == {with_cjk.pk, without_cjk.pk} assert _matched_ids(backend, "invoice NOT 漢字") == {without_cjk.pk} + @pytest.mark.parametrize( + ("threshold", "expected"), + [ + pytest.param(None, {"titled"}, id="fuzzy_off"), + pytest.param(0.0, {"titled", "content_only"}, id="fuzzy_on"), + ], + ) def test_fielded_cjk_term_searches_only_that_field( self, backend: TantivyBackend, + settings: SettingsWrapper, + threshold: float | None, + expected: set[str], ) -> None: - """'title:東京' must not match a document whose 東京 is in the content.""" + """'title:東京' must not match a document whose 東京 is in the content. + + The CJK clause honours the field. The fuzzy clause, when enabled, + does not: it contributes every free-text term UNFIELDED by design + (see _try_parse_fuzzy_query), so it brings the content-only + document back on its own 0.1-boosted terms. That is the documented + trade-off, pinned here so it stays deliberate. + """ + settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = threshold content_only = _index( backend, title="Tokyo report", @@ -66,9 +86,10 @@ class TestCjkClauseFollowsTheParsedQuery: content="an english summary", checksum="cjk-field-2", ) + pks = {"titled": titled.pk, "content_only": content_only.pk} - assert _matched_ids(backend, "東京") == {content_only.pk, titled.pk} - assert _matched_ids(backend, "title:東京") == {titled.pk} + assert _matched_ids(backend, "東京") == set(pks.values()) + assert _matched_ids(backend, "title:東京") == {pks[label] for label in expected} def test_cjk_on_a_non_default_field_builds_no_clause( self, diff --git a/src/documents/tests/search/test_fuzzy_tokenization.py b/src/documents/tests/search/test_fuzzy_tokenization.py index 335c6992b..3048228a2 100644 --- a/src/documents/tests/search/test_fuzzy_tokenization.py +++ b/src/documents/tests/search/test_fuzzy_tokenization.py @@ -105,3 +105,70 @@ class TestFuzzyClauseWords: ) assert _matched_ids(backend, '"tax reports"') == {near_miss.pk} + + +class TestBooleanKeywordsInRawText: + """Tantivy's boolean keywords are word runs, so they survive the cut + into words and its own parser reads them as grammar. Raw query text + reaches that parser with its case intact, so a quoted phrase can carry + them in.""" + + @pytest.fixture + def corpus(self, backend: TantivyBackend) -> dict[str, int]: + both = _index( + backend, + title="A", + content="taxation reportage weekly", + checksum="fuzz-kw-1", + ) + tax_only = _index( + backend, + title="B", + content="taxation only here", + checksum="fuzz-kw-2", + ) + report_only = _index( + backend, + title="C", + content="reportage only here", + checksum="fuzz-kw-3", + ) + return { + "both": both.pk, + "tax_only": tax_only.pk, + "report_only": report_only.pk, + } + + @pytest.mark.parametrize( + "query", + [ + pytest.param('"tax AND reports"', id="and"), + pytest.param('"tax OR reports"', id="or"), + pytest.param('"tax NOT reports"', id="not"), + pytest.param('"tax IN reports"', id="in"), + ], + ) + def test_a_keyword_inside_a_phrase_stays_an_ordinary_word( + self, + backend: TantivyBackend, + corpus: dict[str, int], + query: str, + ) -> None: + """The phrase asks for three words, so the clause must stay the + disjunction it is for '"tax reports"': AND must not turn it into a + conjunction, NOT must not give it its own exclusion, IN must not + fail the parse.""" + assert _matched_ids(backend, '"tax reports"') == set(corpus.values()) + assert _matched_ids(backend, query) == set(corpus.values()) + + def test_a_trailing_keyword_does_not_drop_the_clause( + self, + backend: TantivyBackend, + corpus: dict[str, int], + ) -> None: + """'tax AND' is a syntax error to tantivy's parser, which would + cost the whole query its fuzzy clause.""" + assert _matched_ids(backend, '"tax AND"') == { + corpus["both"], + corpus["tax_only"], + } diff --git a/src/documents/tests/search/test_query_negation.py b/src/documents/tests/search/test_query_negation.py index 0db1d5f1f..8b3cd0b4c 100644 --- a/src/documents/tests/search/test_query_negation.py +++ b/src/documents/tests/search/test_query_negation.py @@ -109,3 +109,45 @@ class TestNegationConstrainsEveryClause: secret_invoice.pk, unrelated.pk, } + + def test_a_negation_under_or_does_not_constrain_the_cjk_clause( + self, + backend: TantivyBackend, + ) -> None: + """The limit of the hoist, pinned deliberately. + + An exclusion that is one branch's own condition cannot be restated + above the blend without dropping documents the other branch + matches, so it is left where it is and the CJK clause stays + unconstrained by it. That shows through here in a way it does not + for latin text: the exact clause cannot match a CJK run at all, so + the CJK clause is the only thing matching the tokyo documents, and + the secret one comes with it. + """ + secret = _index( + backend, + title="Tokyo A", + content="東京都の秘密です secret", + checksum="neg-or-cjk-1", + ) + public = _index( + backend, + title="Tokyo B", + content="東京都の報告書です public", + checksum="neg-or-cjk-2", + ) + bill = _index( + backend, + title="Bill", + content="bill payment received", + checksum="neg-or-cjk-3", + ) + + assert _matched_ids(backend, "(東京 AND NOT secret) OR bill") == { + bill.pk, + public.pk, + secret.pk, + } + # The same exclusion in conjunctive position is hoisted, and does + # constrain the CJK clause. + assert _matched_ids(backend, "東京 AND NOT secret") == {public.pk}