mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-08-26 12:43:19 +00:00
stumpylogandClaude Opus 5
e311c84139
Index terms are stemmed but query patterns were not, so invoice* matched nothing while invoic* worked. v2's index was unstemmed (whoosh TEXT() defaults to StandardAnalyzer), so this regressed against both baselines, not just dev. Uses the typed run's stem unless the stem is longer than the run, since a stem can be longer than a partial prefix and a shorter prefix only widens recall. Patterns spanning the stem boundary (produ*name) still cannot match a stemmed index, so usage.md loses that example rather than advertising a broken one. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
129 lines
4.2 KiB
Python
129 lines
4.2 KiB
Python
"""Wildcard patterns must match a stemmed index.
|
|
|
|
Query patterns are normalized but were not stemmed, while index terms are
|
|
stemmed, so the natural spelling of a prefix search matched nothing:
|
|
``invoice*`` found no document although ``invoic*`` did. v2's index was
|
|
UNSTEMMED (whoosh ``TEXT()`` defaults to ``StandardAnalyzer``), so this
|
|
regressed against both baselines.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import TYPE_CHECKING
|
|
|
|
import pytest
|
|
|
|
from documents.models import Document
|
|
from documents.search._registry import _make_pattern_normalizer
|
|
|
|
if TYPE_CHECKING:
|
|
from documents.search._backend import TantivyBackend
|
|
|
|
pytestmark = [pytest.mark.search, pytest.mark.django_db]
|
|
|
|
CONTENT = (
|
|
"invoice total due for electricity from both companies, "
|
|
"payments made to the university library"
|
|
)
|
|
|
|
|
|
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
|
|
return set(backend.search_ids(query, user=None))
|
|
|
|
|
|
@pytest.fixture
|
|
def indexed_doc(backend: TantivyBackend) -> Document:
|
|
doc = Document.objects.create(
|
|
title="Invoice 2020 productname",
|
|
content=CONTENT,
|
|
checksum="pattern-stemming-1",
|
|
archive_serial_number=900,
|
|
)
|
|
backend.add_or_update(doc)
|
|
return doc
|
|
|
|
|
|
class TestPrefixStemming:
|
|
@pytest.mark.parametrize(
|
|
"query",
|
|
[
|
|
"invoice*",
|
|
"electricity*",
|
|
"companies*",
|
|
"payments*",
|
|
"library*",
|
|
"title:Invoice*",
|
|
],
|
|
)
|
|
def test_full_word_prefix_matches_its_stem(
|
|
self,
|
|
backend: TantivyBackend,
|
|
indexed_doc: Document,
|
|
query: str,
|
|
) -> None:
|
|
assert _matched_ids(backend, query) == {indexed_doc.id}
|
|
|
|
@pytest.mark.parametrize("query", ["invoic*", "electr*", "payment*"])
|
|
def test_already_stemmed_prefix_still_matches(
|
|
self,
|
|
backend: TantivyBackend,
|
|
indexed_doc: Document,
|
|
query: str,
|
|
) -> None:
|
|
assert _matched_ids(backend, query) == {indexed_doc.id}
|
|
|
|
@pytest.mark.parametrize("query", ["univers*", "librar*"])
|
|
def test_partial_prefix_is_not_lengthened_by_its_stem(
|
|
self,
|
|
backend: TantivyBackend,
|
|
indexed_doc: Document,
|
|
query: str,
|
|
) -> None:
|
|
"""A partial prefix keeps matching: "librar" stems to "librari", which
|
|
is longer than what was typed, so the typed run is kept. Using the
|
|
shorter of the two widens recall rather than failing closed."""
|
|
assert _matched_ids(backend, query) == {indexed_doc.id}
|
|
|
|
def test_pattern_past_the_stem_boundary_is_documented_not_fixed(
|
|
self,
|
|
backend: TantivyBackend,
|
|
indexed_doc: Document,
|
|
) -> None:
|
|
"""produ*name cannot match a stemmed index ("productname" is indexed as
|
|
"productnam"); usage.md must not advertise it. Pinned so the limitation
|
|
is deliberate, not accidental."""
|
|
assert _matched_ids(backend, "produ*name") == set()
|
|
|
|
|
|
class TestPatternNormalizer:
|
|
@pytest.mark.parametrize(
|
|
("text", "expected"),
|
|
[
|
|
("Invoice", "invoic"),
|
|
("companies", "compani"),
|
|
# y -> i: same length as typed, and the index only holds the stem
|
|
("library", "librari"),
|
|
("invoic", "invoic"),
|
|
("Universit", "universit"),
|
|
("Café", "cafe"),
|
|
],
|
|
)
|
|
def test_shorter_of_the_typed_run_and_its_stem(
|
|
self,
|
|
text: str,
|
|
expected: str,
|
|
) -> None:
|
|
assert _make_pattern_normalizer("en")(text) == expected
|
|
|
|
def test_run_that_yields_no_token_falls_back_to_the_typed_run(self) -> None:
|
|
"""A run past the remove_long limit analyzes to zero tokens, so there is
|
|
no stem to substitute and the folded run is used as typed."""
|
|
over_long = "invoices" * 20
|
|
assert _make_pattern_normalizer("en")(over_long) == over_long
|
|
|
|
@pytest.mark.parametrize("language", [None, "klingon"])
|
|
def test_unstemmed_language_folds_only(self, language: str | None) -> None:
|
|
"""With no stemmer configured, or one this build has no stemmer for, the
|
|
index holds surface forms and the pattern must keep them too."""
|
|
assert _make_pattern_normalizer(language)("Invoices") == "invoices"
|