paperless-ngx/src/documents/tests/search/test_fuzzy_alternative.py

"""_fuzzy_alternative, the near-match side of the widening hook.

A leaf's words each become a Fuzzy leaf on the leaf's own field, so a
typo in one word of a term still finds the document while everything
around the term keeps constraining it. A Term's words are OR'd for
per-word recall; a Phrase's are AND-ed, because quoting must not widen a
search. CJK words get no fuzzy side at all.
"""

from __future__ import annotations

import pytest
import whoosh_compat as wc
import whoosh_compat.ast as wc_ast

from documents.search._query import _fuzzy_alternative
from documents.tests.search._ast_helpers import CONTENT
from documents.tests.search._ast_helpers import TITLE
from documents.tests.search._ast_helpers import content

pytestmark = pytest.mark.search


# 130 characters, past the analyzer's 129-byte remove_long limit (128 is
# kept, 129 is dropped), so the index never holds it and neither side
# should search for it.
_TOO_LONG = "x" * 130


def _fuzzy(field: wc.FieldRef, text: str) -> wc_ast.Fuzzy:
    return wc_ast.Fuzzy(field=field, text=text, distance=1, prefix=True)


class TestTheAlternative:
    def test_a_single_word_becomes_one_fuzzy_leaf(self) -> None:
        """
        GIVEN:
            - A one-word term
        WHEN:
            - The fuzzy alternative is built
        THEN:
            - It is a single Fuzzy leaf on the same field, distance 1 and
              prefix matching, which is what the old clause used
        """
        assert _fuzzy_alternative(content("invoice")) == _fuzzy(CONTENT, "invoice")

    def test_the_leafs_own_field_is_used(self) -> None:
        """
        GIVEN:
            - A term fielded on title
        WHEN:
            - The fuzzy alternative is built
        THEN:
            - The Fuzzy leaf is on title, not spread across the default
              fields. This is the fielding fix: today's clause searches a
              fielded word everywhere
        """
        assert _fuzzy_alternative(wc_ast.Term(field=TITLE, text="invoice")) == _fuzzy(
            TITLE,
            "invoice",
        )

    def test_several_words_are_ored(self) -> None:
        """
        GIVEN:
            - A hyphenated term the analyzer splits into two words
        WHEN:
            - The fuzzy alternative is built
        THEN:
            - The words are OR'd. AND would be stricter than the exact
              side, which is OR'd for an unfielded term, and would lose
              today's per-word recall inside a term
        """
        assert _fuzzy_alternative(content("COVID-19")) == wc_ast.Or(
            children=(_fuzzy(CONTENT, "covid"), _fuzzy(CONTENT, "19")),
        )

    def test_words_are_split_where_the_index_splits_them(self) -> None:
        """
        GIVEN:
            - A term the analyzer lowercases and folds
        WHEN:
            - The fuzzy alternative is built
        THEN:
            - The Fuzzy text is the analyzer's output, not the raw text,
              so it is in the same shape as the index terms
        """
        assert _fuzzy_alternative(content("Éclair")) == _fuzzy(CONTENT, "eclair")

    def test_the_splitter_does_not_stem(self) -> None:
        """
        GIVEN:
            - A word the stemmer would change
        WHEN:
            - The fuzzy alternative is built
        THEN:
            - The word is unstemmed. Fuzzy text goes through the field's
              pattern_normalizer, which stems it once; stemming here too
              would search for a term the index does not hold
        """
        assert _fuzzy_alternative(content("universities")) == _fuzzy(
            CONTENT,
            "universities",
        )

    @pytest.mark.parametrize(
        "text",
        [
            pytest.param("x", id="one_character"),
            pytest.param("__", id="only_dropped_characters"),
            pytest.param(_TOO_LONG, id="past_remove_long"),
            pytest.param("", id="empty"),
            pytest.param("   ", id="whitespace"),
        ],
    )
    def test_text_with_no_usable_word_gets_no_alternative(self, text: str) -> None:
        """
        GIVEN:
            - Text with no word the index could hold: a single character,
              characters the tokenizer drops, a word past remove_long, or
              nothing at all
        WHEN:
            - The fuzzy alternative is built
        THEN:
            - None, so the leaf is left alone. Returning an alternative
              here would collapse Or(leaf, fuzzy) to a required clause
              that can never match, making a query return LESS with fuzzy
              on than off
        """
        assert _fuzzy_alternative(content(text)) is None

    def test_a_one_character_word_is_dropped_from_a_longer_term(self) -> None:
        """
        GIVEN:
            - A term mixing a one-character word with a real one
        WHEN:
            - The fuzzy alternative is built
        THEN:
            - Only the real word survives. A one-character prefix fuzzy
              term matches every term in the field
        """
        assert _fuzzy_alternative(content("h52.1")) == _fuzzy(CONTENT, "h52")

    def test_the_leaf_span_is_copied(self) -> None:
        """
        GIVEN:
            - A leaf with a source span, whose words both survive the
              one-character filter so there is an Or to inspect
        WHEN:
            - The fuzzy alternative is built
        THEN:
            - Every node it builds carries that span, so an emit-time
              diagnostic still points into the query text
        """
        leaf = wc_ast.Term(field=CONTENT, text="ab-cd", startchar=4, endchar=9)

        alternative = _fuzzy_alternative(leaf)

        assert isinstance(alternative, wc_ast.Or)
        spans = {
            (node.startchar, node.endchar)
            for node in (alternative, *alternative.children)
        }
        assert spans == {(4, 9)}


class TestPhrasesAreNotWidened:
    def test_a_multi_word_phrase_requires_every_word(self) -> None:
        """
        GIVEN:
            - A quoted two-word phrase
        WHEN:
            - Its fuzzy alternative is built
        THEN:
            - The words are And-combined, where a Term's are Or-combined
              (test_a_hyphenated_term_matches_on_one_word pins that). The
              parser's default group is And, so an Or here would let the
              quoted phrase match strictly more than the same two words
              unquoted
        """
        leaf = wc_ast.Phrase(field=CONTENT, text="tax report")

        assert _fuzzy_alternative(leaf) == wc_ast.And(
            children=(_fuzzy(CONTENT, "tax"), _fuzzy(CONTENT, "report")),
        )


class TestCjkGetsNoFuzzySide:
    @pytest.mark.parametrize(
        "text",
        [
            pytest.param("東京", id="cjk_only"),
            pytest.param("東京都の報告書", id="longer_run"),
            pytest.param("서울", id="hangul"),
        ],
    )
    def test_a_cjk_word_is_skipped(self, text: str) -> None:
        """
        GIVEN:
            - A CJK term
        WHEN:
            - Its fuzzy alternative is built
        THEN:
            - There is none. The content analyzer keeps the run as one
              token, so a prefix Fuzzy over it would match any run within
              one edit of its start, which is the "東京都 matches 京都"
              failure the bigram fields exist to avoid
        """
        assert _fuzzy_alternative(content(text)) is None

    def test_latin_beside_cjk_still_gets_its_fuzzy_side(self) -> None:
        """
        GIVEN:
            - A term mixing a CJK run and a latin word
        WHEN:
            - Its fuzzy alternative is built
        THEN:
            - Only the latin word is fuzzed. Skipping CJK words must not
              cost the latin half its near-match
        """
        leaf = content("東京 report")

        assert _fuzzy_alternative(leaf) == _fuzzy(CONTENT, "report")