fix(search): stop analyzing the fuzzy clause's words twice

_try_parse_fuzzy_query collected free_text_tokens with the default
analyzed=True and handed the analyzer's output back to
index.parse_query, which analyzes it again. Analysis is not idempotent:
'universities' stems to 'univers', and re-stemming that yields 'univ', a
term the index does not contain. prefix=True hid the mistake as
over-broad matching rather than as no matches at all, which is why no
test caught it: searching 'universities' also returned documents whose
only relevant word was 'univalent' or 'unicycle'.

Collect the raw text instead. Raw text has not been tokenized, so the
whole-token \w+ filter that keeps tantivy query grammar out of the
re-parse would now reject ordinary input outright: 'COVID-19',
'hello@example.com' and the phrase "tax reports" each arrive as a single
token containing punctuation, and a query made only of such terms would
lose its fuzzy clause entirely. Cut each token into its word runs and
keep those, which recovers the terms and keeps the guarantee the filter
exists for: only word characters ever reach the parser.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

src/documents/search/_query.py

@@ -186,12 +186,15 @@ def _build_ast_cjk_query(
     return _any_of(clauses) if clauses else None
 
 
-# A joined fuzzy word string must stay plain words: any token that could
-# read as tantivy query grammar (a colon, bracket, quote, operator...) is
-# dropped rather than escaped. Today's default-field analyzers only emit
-# word characters, so this never fires; it guards a future field whose
-# analyzer passes punctuation through (an identity/keyword analyzer).
-_WORD_TOKEN_RE = regex.compile(r"\w+")
+# A joined fuzzy word string must stay plain words: it goes back through
+# tantivy's own query parser, and the raw query text the clause collects
+# routinely carries characters that parser reads as grammar (a colon, a
+# bracket, a quote, a leading -). Each token is cut into its word runs and
+# only those are kept, so nothing injectable can reach the parser. Cutting
+# rather than dropping the whole token is what keeps ordinary hyphenated,
+# dotted and quoted input ("COVID-19", "hello@example.com", "tax reports")
+# contributing to the clause at all.
+_WORD_RUN_RE = regex.compile(r"\w+")
 
 
 def _try_parse_fuzzy_query(
@@ -222,13 +225,29 @@ def _try_parse_fuzzy_query(
     threshold already disciplines, accepted in exchange for never feeding
     field syntax to tantivy's parser.
 
+    The words are the query's RAW text, not the analyzer's output
+    (``analyzed=False``), because ``index.parse_query`` analyzes whatever
+    it is given and analysis is not idempotent: ``universities`` stems to
+    ``univers``, and handing that back stems it again to ``univ``, a term
+    the index does not contain. ``prefix=True`` hid this as over-broad
+    matching (``univ`` also prefixes ``unicycle``) rather than as no
+    matches at all. Raw text is untokenized, which is why it is cut into
+    word runs above rather than taken whole.
+
     The ValueError guard stays as insurance (the word string is plain
     tokens, so tantivy accepting it is expected, not assumed): on a parse
     failure the fuzzy clause is skipped and the exact/CJK clauses stand,
     rather than the whole query failing.
     """
-    tokens = wc.free_text_tokens(ast, registry=registry, fields=_DEFAULT_SEARCH_FIELDS)
-    words = [t for t in tokens if _WORD_TOKEN_RE.fullmatch(t)]
+    tokens = wc.free_text_tokens(
+        ast,
+        registry=registry,
+        fields=_DEFAULT_SEARCH_FIELDS,
+        analyzed=False,
+    )
+    words = list(
+        dict.fromkeys(word for token in tokens for word in _WORD_RUN_RE.findall(token)),
+    )
     if not words:
         return None
     fuzzy_text = " ".join(words)

src/documents/tests/search/test_fuzzy_tokenization.py

@@ -0,0 +1,107 @@
+"""The words the fuzzy blend clause hands back to tantivy's parser.
+
+The clause re-parses a word string through tantivy, which analyzes it
+again, so the words must be the query's raw text rather than the analyzed
+text (analysis is not idempotent), and must still be split into plain
+words so that hyphenated, dotted and quoted terms keep contributing.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+from documents.models import Document
+
+if TYPE_CHECKING:
+    from pytest_django.fixtures import SettingsWrapper
+
+    from documents.search._backend import TantivyBackend
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+
+def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
+    return set(backend.search_ids(query, user=None))
+
+
+def _index(backend: TantivyBackend, **kwargs: object) -> Document:
+    doc = Document.objects.create(**kwargs)
+    backend.add_or_update(doc)
+    return doc
+
+
+@pytest.fixture(autouse=True)
+def fuzzy_enabled(settings: SettingsWrapper) -> None:
+    """Enable the fuzzy blend clause. The threshold doubles as a minimum
+    score filter, so it is set to 0.0: every hit passes and the test sees
+    the clause's matching behaviour, not the filter's."""
+    settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.0
+
+
+class TestFuzzyClauseWords:
+    def test_a_stemmed_word_is_not_stemmed_a_second_time(
+        self,
+        backend: TantivyBackend,
+    ) -> None:
+        """'universities' stems to 'univers'; feeding that back to tantivy
+        stems it again to 'univ', whose fuzzy prefix reaches unrelated
+        words. The clause must stay wide enough for a typo and no wider."""
+        wanted = _index(
+            backend,
+            title="A",
+            content="universities of europe",
+            checksum="fuzz-stem-1",
+        )
+        typo = _index(
+            backend,
+            title="B",
+            content="universties of europe",
+            checksum="fuzz-stem-2",
+        )
+        _index(
+            backend,
+            title="C",
+            content="univalent chemical bonds",
+            checksum="fuzz-stem-3",
+        )
+        _index(
+            backend,
+            title="D",
+            content="unicycle repair manual",
+            checksum="fuzz-stem-4",
+        )
+
+        assert _matched_ids(backend, "universities") == {wanted.pk, typo.pk}
+
+    def test_a_hyphenated_term_still_reaches_the_clause(
+        self,
+        backend: TantivyBackend,
+    ) -> None:
+        """'COVID-19' is one raw token: unless it is split into words, it
+        carries characters the re-parse would read as grammar, is dropped,
+        and the whole query loses its fuzzy clause."""
+        misspelled = _index(
+            backend,
+            title="A",
+            content="covidx testing results",
+            checksum="fuzz-hyphen-1",
+        )
+
+        assert _matched_ids(backend, "COVID-19") == {misspelled.pk}
+
+    def test_a_phrase_still_reaches_the_clause(
+        self,
+        backend: TantivyBackend,
+    ) -> None:
+        """A phrase is one raw token carrying a space, and is the whole
+        query's only free text here."""
+        near_miss = _index(
+            backend,
+            title="A",
+            content="taxation reportage weekly",
+            checksum="fuzz-phrase-1",
+        )
+
+        assert _matched_ids(backend, '"tax reports"') == {near_miss.pk}