fix(search): neutralize tantivy's boolean keywords in the fuzzy words

The fuzzy clause's word string is cut to \w+ runs so no query grammar
reaches index.parse_query, but tantivy's boolean keywords are themselves
word runs. Under analyzed=True the field analyzer lowercased them into
ordinary terms before they got that far; now that the words are raw
query text, an uppercase keyword out of a quoted phrase arrives as
grammar: '"tax AND reports"' quietly made the clause a conjunction,
'"tax NOT reports"' gave it its own exclusion, and '"tax AND"' (or IN
anywhere) failed the parse and cost the query its fuzzy clause outright.

Lowercase exactly AND/OR/NOT/IN, which is what the analyzer used to do
and is the only spelling tantivy reads as grammar ("And" is a term).
Nothing else is touched: tantivy already lowercases query terms with the
field's analyzer, and doing it ourselves first is not the same operation
for every input (Python folds a final sigma differently, and turns 'İ'
into a sequence tantivy then splits in two), which would search for
terms the index does not contain.

Also pins two behaviours that were reasoned about but untested: the
fielded-CJK test now runs with the fuzzy clause on as well, where the
clause's documented unfielded contribution does bring the other document
back, and the negation tests pin the CJK over-admission for an exclusion
under an Or, which cannot be hoisted without dropping the other branch's
documents.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

src/documents/search/_query.py

@@ -190,12 +190,28 @@ def _build_ast_cjk_query(
 # tantivy's own query parser, and the raw query text the clause collects
 # routinely carries characters that parser reads as grammar (a colon, a
 # bracket, a quote, a leading -). Each token is cut into its word runs and
-# only those are kept, so nothing injectable can reach the parser. Cutting
-# rather than dropping the whole token is what keeps ordinary hyphenated,
-# dotted and quoted input ("COVID-19", "hello@example.com", "tax reports")
-# contributing to the clause at all.
+# only those are kept, so no field syntax, pattern, range or grouping can
+# reach the parser. Cutting rather than dropping the whole token is what
+# keeps ordinary hyphenated, dotted and quoted input ("COVID-19",
+# "hello@example.com", "tax reports") contributing to the clause at all.
 _WORD_RUN_RE = regex.compile(r"\w+")
 
+# The one piece of tantivy grammar that survives the cut: its boolean
+# keywords are themselves word runs. Only these exact spellings are
+# grammar there ("And"/"and" are ordinary terms), so lowercasing exactly
+# these turns them back into the ordinary terms the field analyzer used to
+# make of them, before the clause switched to raw text. Left alone, a
+# quoted phrase would silently restructure the clause ("tax AND reports"
+# becoming a conjunction) or fail to parse and drop it entirely
+# ("tax AND", or "IN" anywhere).
+#
+# Only these words are touched: tantivy lowercases query terms with the
+# field's own analyzer, and doing it ourselves first is not always the
+# same operation (Python folds a final sigma to a different letter than
+# tantivy does, and turns Turkish 'İ' into a sequence tantivy then splits
+# in two), which would search for terms the index does not contain.
+_TANTIVY_KEYWORDS: Final[frozenset[str]] = frozenset({"AND", "OR", "NOT", "IN"})
+
 
 def _try_parse_fuzzy_query(
     index: tantivy.Index,
@@ -223,7 +239,10 @@ def _try_parse_fuzzy_query(
     default fields rather than just the one the user named. That is
     recall-only widening on a secondary 0.1-boosted clause the score
     threshold already disciplines, accepted in exchange for never feeding
-    field syntax to tantivy's parser.
+    field syntax to tantivy's parser. What the word string guarantees is
+    exactly that: no field prefix, pattern, range, grouping or quoting
+    survives, and the boolean keywords that do survive (they are word
+    runs) are lowercased into ordinary terms; see _TANTIVY_KEYWORDS.
 
     The words are the query's RAW text, not the analyzer's output
     (``analyzed=False``), because ``index.parse_query`` analyzes whatever
@@ -246,7 +265,11 @@ def _try_parse_fuzzy_query(
         analyzed=False,
     )
     words = list(
-        dict.fromkeys(word for token in tokens for word in _WORD_RUN_RE.findall(token)),
+        dict.fromkeys(
+            word.lower() if word in _TANTIVY_KEYWORDS else word
+            for token in tokens
+            for word in _WORD_RUN_RE.findall(token)
+        ),
     )
     if not words:
         return None
@@ -468,7 +491,7 @@ def parse_user_query(
     # plain Shoulds beside the exact clause they re-admit exactly the
     # documents the query excluded. Restate the exclusions once, above the
     # whole blend. Redundant against the exact clause, which already
-    # carries them, but idempotently so, and cheaper than stripping them.
+    # carries them, but idempotently so.
     negations = _negation_clauses(index, result.ast, registry)
     if not negations:
         return _any_of(clauses)

src/documents/tests/search/test_cjk_clause.py

@@ -15,6 +15,8 @@ import pytest
 from documents.models import Document
 
 if TYPE_CHECKING:
+    from pytest_django.fixtures import SettingsWrapper
+
     from documents.search._backend import TantivyBackend
 
 pytestmark = [pytest.mark.search, pytest.mark.django_db]
@@ -49,11 +51,29 @@ class TestCjkClauseFollowsTheParsedQuery:
         assert _matched_ids(backend, "invoice") == {with_cjk.pk, without_cjk.pk}
         assert _matched_ids(backend, "invoice NOT 漢字") == {without_cjk.pk}
 
+    @pytest.mark.parametrize(
+        ("threshold", "expected"),
+        [
+            pytest.param(None, {"titled"}, id="fuzzy_off"),
+            pytest.param(0.0, {"titled", "content_only"}, id="fuzzy_on"),
+        ],
+    )
     def test_fielded_cjk_term_searches_only_that_field(
         self,
         backend: TantivyBackend,
+        settings: SettingsWrapper,
+        threshold: float | None,
+        expected: set[str],
     ) -> None:
-        """'title:東京' must not match a document whose 東京 is in the content."""
+        """'title:東京' must not match a document whose 東京 is in the content.
+
+        The CJK clause honours the field. The fuzzy clause, when enabled,
+        does not: it contributes every free-text term UNFIELDED by design
+        (see _try_parse_fuzzy_query), so it brings the content-only
+        document back on its own 0.1-boosted terms. That is the documented
+        trade-off, pinned here so it stays deliberate.
+        """
+        settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = threshold
         content_only = _index(
             backend,
             title="Tokyo report",
@@ -66,9 +86,10 @@ class TestCjkClauseFollowsTheParsedQuery:
             content="an english summary",
             checksum="cjk-field-2",
         )
+        pks = {"titled": titled.pk, "content_only": content_only.pk}
 
-        assert _matched_ids(backend, "東京") == {content_only.pk, titled.pk}
-        assert _matched_ids(backend, "title:東京") == {titled.pk}
+        assert _matched_ids(backend, "東京") == set(pks.values())
+        assert _matched_ids(backend, "title:東京") == {pks[label] for label in expected}
 
     def test_cjk_on_a_non_default_field_builds_no_clause(
         self,

src/documents/tests/search/test_fuzzy_tokenization.py

@@ -105,3 +105,70 @@ class TestFuzzyClauseWords:
         )
 
         assert _matched_ids(backend, '"tax reports"') == {near_miss.pk}
+
+
+class TestBooleanKeywordsInRawText:
+    """Tantivy's boolean keywords are word runs, so they survive the cut
+    into words and its own parser reads them as grammar. Raw query text
+    reaches that parser with its case intact, so a quoted phrase can carry
+    them in."""
+
+    @pytest.fixture
+    def corpus(self, backend: TantivyBackend) -> dict[str, int]:
+        both = _index(
+            backend,
+            title="A",
+            content="taxation reportage weekly",
+            checksum="fuzz-kw-1",
+        )
+        tax_only = _index(
+            backend,
+            title="B",
+            content="taxation only here",
+            checksum="fuzz-kw-2",
+        )
+        report_only = _index(
+            backend,
+            title="C",
+            content="reportage only here",
+            checksum="fuzz-kw-3",
+        )
+        return {
+            "both": both.pk,
+            "tax_only": tax_only.pk,
+            "report_only": report_only.pk,
+        }
+
+    @pytest.mark.parametrize(
+        "query",
+        [
+            pytest.param('"tax AND reports"', id="and"),
+            pytest.param('"tax OR reports"', id="or"),
+            pytest.param('"tax NOT reports"', id="not"),
+            pytest.param('"tax IN reports"', id="in"),
+        ],
+    )
+    def test_a_keyword_inside_a_phrase_stays_an_ordinary_word(
+        self,
+        backend: TantivyBackend,
+        corpus: dict[str, int],
+        query: str,
+    ) -> None:
+        """The phrase asks for three words, so the clause must stay the
+        disjunction it is for '"tax reports"': AND must not turn it into a
+        conjunction, NOT must not give it its own exclusion, IN must not
+        fail the parse."""
+        assert _matched_ids(backend, '"tax reports"') == set(corpus.values())
+        assert _matched_ids(backend, query) == set(corpus.values())
+
+    def test_a_trailing_keyword_does_not_drop_the_clause(
+        self,
+        backend: TantivyBackend,
+        corpus: dict[str, int],
+    ) -> None:
+        """'tax AND' is a syntax error to tantivy's parser, which would
+        cost the whole query its fuzzy clause."""
+        assert _matched_ids(backend, '"tax AND"') == {
+            corpus["both"],
+            corpus["tax_only"],
+        }

src/documents/tests/search/test_query_negation.py

@@ -109,3 +109,45 @@ class TestNegationConstrainsEveryClause:
             secret_invoice.pk,
             unrelated.pk,
         }
+
+    def test_a_negation_under_or_does_not_constrain_the_cjk_clause(
+        self,
+        backend: TantivyBackend,
+    ) -> None:
+        """The limit of the hoist, pinned deliberately.
+
+        An exclusion that is one branch's own condition cannot be restated
+        above the blend without dropping documents the other branch
+        matches, so it is left where it is and the CJK clause stays
+        unconstrained by it. That shows through here in a way it does not
+        for latin text: the exact clause cannot match a CJK run at all, so
+        the CJK clause is the only thing matching the tokyo documents, and
+        the secret one comes with it.
+        """
+        secret = _index(
+            backend,
+            title="Tokyo A",
+            content="東京都の秘密です secret",
+            checksum="neg-or-cjk-1",
+        )
+        public = _index(
+            backend,
+            title="Tokyo B",
+            content="東京都の報告書です public",
+            checksum="neg-or-cjk-2",
+        )
+        bill = _index(
+            backend,
+            title="Bill",
+            content="bill payment received",
+            checksum="neg-or-cjk-3",
+        )
+
+        assert _matched_ids(backend, "(東京 AND NOT secret) OR bill") == {
+            bill.pk,
+            public.pk,
+            secret.pk,
+        }
+        # The same exclusion in conjunctive position is hoisted, and does
+        # constrain the CJK clause.
+        assert _matched_ids(backend, "東京 AND NOT secret") == {public.pk}