diff --git a/src/documents/search/_query.py b/src/documents/search/_query.py index e0617731c..f62d451c8 100644 --- a/src/documents/search/_query.py +++ b/src/documents/search/_query.py @@ -186,12 +186,15 @@ def _build_ast_cjk_query( return _any_of(clauses) if clauses else None -# A joined fuzzy word string must stay plain words: any token that could -# read as tantivy query grammar (a colon, bracket, quote, operator...) is -# dropped rather than escaped. Today's default-field analyzers only emit -# word characters, so this never fires; it guards a future field whose -# analyzer passes punctuation through (an identity/keyword analyzer). -_WORD_TOKEN_RE = regex.compile(r"\w+") +# A joined fuzzy word string must stay plain words: it goes back through +# tantivy's own query parser, and the raw query text the clause collects +# routinely carries characters that parser reads as grammar (a colon, a +# bracket, a quote, a leading -). Each token is cut into its word runs and +# only those are kept, so nothing injectable can reach the parser. Cutting +# rather than dropping the whole token is what keeps ordinary hyphenated, +# dotted and quoted input ("COVID-19", "hello@example.com", "tax reports") +# contributing to the clause at all. +_WORD_RUN_RE = regex.compile(r"\w+") def _try_parse_fuzzy_query( @@ -222,13 +225,29 @@ def _try_parse_fuzzy_query( threshold already disciplines, accepted in exchange for never feeding field syntax to tantivy's parser. + The words are the query's RAW text, not the analyzer's output + (``analyzed=False``), because ``index.parse_query`` analyzes whatever + it is given and analysis is not idempotent: ``universities`` stems to + ``univers``, and handing that back stems it again to ``univ``, a term + the index does not contain. ``prefix=True`` hid this as over-broad + matching (``univ`` also prefixes ``unicycle``) rather than as no + matches at all. Raw text is untokenized, which is why it is cut into + word runs above rather than taken whole. + The ValueError guard stays as insurance (the word string is plain tokens, so tantivy accepting it is expected, not assumed): on a parse failure the fuzzy clause is skipped and the exact/CJK clauses stand, rather than the whole query failing. """ - tokens = wc.free_text_tokens(ast, registry=registry, fields=_DEFAULT_SEARCH_FIELDS) - words = [t for t in tokens if _WORD_TOKEN_RE.fullmatch(t)] + tokens = wc.free_text_tokens( + ast, + registry=registry, + fields=_DEFAULT_SEARCH_FIELDS, + analyzed=False, + ) + words = list( + dict.fromkeys(word for token in tokens for word in _WORD_RUN_RE.findall(token)), + ) if not words: return None fuzzy_text = " ".join(words) diff --git a/src/documents/tests/search/test_fuzzy_tokenization.py b/src/documents/tests/search/test_fuzzy_tokenization.py new file mode 100644 index 000000000..335c6992b --- /dev/null +++ b/src/documents/tests/search/test_fuzzy_tokenization.py @@ -0,0 +1,107 @@ +"""The words the fuzzy blend clause hands back to tantivy's parser. + +The clause re-parses a word string through tantivy, which analyzes it +again, so the words must be the query's raw text rather than the analyzed +text (analysis is not idempotent), and must still be split into plain +words so that hyphenated, dotted and quoted terms keep contributing. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from documents.models import Document + +if TYPE_CHECKING: + from pytest_django.fixtures import SettingsWrapper + + from documents.search._backend import TantivyBackend + +pytestmark = [pytest.mark.search, pytest.mark.django_db] + + +def _matched_ids(backend: TantivyBackend, query: str) -> set[int]: + return set(backend.search_ids(query, user=None)) + + +def _index(backend: TantivyBackend, **kwargs: object) -> Document: + doc = Document.objects.create(**kwargs) + backend.add_or_update(doc) + return doc + + +@pytest.fixture(autouse=True) +def fuzzy_enabled(settings: SettingsWrapper) -> None: + """Enable the fuzzy blend clause. The threshold doubles as a minimum + score filter, so it is set to 0.0: every hit passes and the test sees + the clause's matching behaviour, not the filter's.""" + settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.0 + + +class TestFuzzyClauseWords: + def test_a_stemmed_word_is_not_stemmed_a_second_time( + self, + backend: TantivyBackend, + ) -> None: + """'universities' stems to 'univers'; feeding that back to tantivy + stems it again to 'univ', whose fuzzy prefix reaches unrelated + words. The clause must stay wide enough for a typo and no wider.""" + wanted = _index( + backend, + title="A", + content="universities of europe", + checksum="fuzz-stem-1", + ) + typo = _index( + backend, + title="B", + content="universties of europe", + checksum="fuzz-stem-2", + ) + _index( + backend, + title="C", + content="univalent chemical bonds", + checksum="fuzz-stem-3", + ) + _index( + backend, + title="D", + content="unicycle repair manual", + checksum="fuzz-stem-4", + ) + + assert _matched_ids(backend, "universities") == {wanted.pk, typo.pk} + + def test_a_hyphenated_term_still_reaches_the_clause( + self, + backend: TantivyBackend, + ) -> None: + """'COVID-19' is one raw token: unless it is split into words, it + carries characters the re-parse would read as grammar, is dropped, + and the whole query loses its fuzzy clause.""" + misspelled = _index( + backend, + title="A", + content="covidx testing results", + checksum="fuzz-hyphen-1", + ) + + assert _matched_ids(backend, "COVID-19") == {misspelled.pk} + + def test_a_phrase_still_reaches_the_clause( + self, + backend: TantivyBackend, + ) -> None: + """A phrase is one raw token carrying a space, and is the whole + query's only free text here.""" + near_miss = _index( + backend, + title="A", + content="taxation reportage weekly", + checksum="fuzz-phrase-1", + ) + + assert _matched_ids(backend, '"tax reports"') == {near_miss.pk}