amorfo77/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2026-09-21 17:08:33 +00:00
Code Issues Projects Releases Wiki Activity

Chore: Move the search tests onto the shared indexing fixtures (#14159)

Browse Source
This commit is contained in:
Trenton H
2026-09-17 14:55:13 -07:00
committed by GitHub
parent 20d309a413
commit fdd32b73bb
24 changed files with 410 additions and 606 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/documents/tests/conftest.py
+12
View File
@@ -182,3 +182,15 @@ def faker_session_locale():
@pytest.fixture(scope="session", autouse=True)
def faker_seed():
return 12345
@pytest.fixture
def indexed_document(_search_index: None) -> "Document":
"""One searchable document, for tests about what the search endpoint
returns rather than about what it finds.
"""
from documents.search import get_backend
doc = DocumentFactory.create(title="quarterly invoice", content="acme corp")
get_backend().add_or_update(doc)
return doc
src/documents/tests/search/_ast_helpers.py
+30
View File
@@ -0,0 +1,30 @@
"""Field references and leaf constructors shared by the unit-level tests.
These are pure whoosh-compat AST values with no state and no database
behind them, so they stay plain module-level functions rather than
fixtures. ``NOTES`` is the reason this module exists at all: notes is a
JSON field, so addressing its text means naming a subpath, and spelling
``wc.FieldRef("notes", "note")`` out once per test file invites two of
them to disagree.
"""
from __future__ import annotations
import whoosh_compat as wc
import whoosh_compat.ast as wc_ast
CONTENT = wc.FieldRef("content")
TITLE = wc.FieldRef("title")
NOTES = wc.FieldRef("notes", "note")
BIGRAM_CONTENT = wc.FieldRef("bigram_content")
BIGRAM_TITLE = wc.FieldRef("bigram_title")
def content(text: str) -> wc_ast.Term:
"""A Term on the content field, the default leaf these tests widen."""
return wc_ast.Term(field=CONTENT, text=text)
def bigram(text: str) -> wc_ast.Term:
"""A Term on the bigram side of content."""
return wc_ast.Term(field=BIGRAM_CONTENT, text=text)
src/documents/tests/search/conftest.py
+23
View File
@@ -3,9 +3,12 @@ from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
import tantivy
from documents.search._backend import TantivyBackend
from documents.search._backend import reset_backend
from documents.search._schema import build_schema
from documents.search._tokenizer import register_tokenizers
from documents.tests.factories import DocumentFactory
if TYPE_CHECKING:
@@ -72,3 +75,23 @@ def matched_ids(backend: TantivyBackend) -> Callable[[str], set[int]]:
return set(backend.search_ids(query, user=None))
return _matched_ids
@pytest.fixture(scope="module")
def query_index() -> tantivy.Index:
"""An in-memory, unstemmed index for the parse-only tests.
These never index a document, so one index per module is shared
read-only across that module's tests.
"""
idx = tantivy.Index(build_schema(), path=None)
register_tokenizers(idx, "")
return idx
@pytest.fixture
def fuzzy_enabled(settings: Settings) -> None:
"""Enable the fuzzy blend clause. The threshold doubles as a minimum
score filter, so it is set to 0.0: every hit passes and the test sees
the clause's matching behaviour, not the filter's."""
settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.0
src/documents/tests/search/test_acceptance.py
+60 -92
View File
@@ -18,13 +18,16 @@ from django.contrib.auth.models import User
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import Note
from documents.models import StoragePath
from documents.search._query import parse_user_query
from documents.tests.factories import DocumentFactory
if TYPE_CHECKING:
from collections.abc import Callable
from documents.models import Document
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
@@ -32,49 +35,28 @@ pytestmark = [pytest.mark.search, pytest.mark.django_db]
FROZEN_NOW = datetime(2026, 6, 15, 12, 0, tzinfo=UTC)
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
return set(backend.search_ids(query, user=None))
def _index(backend: TantivyBackend, **kwargs: object) -> Document:
"""Create a Document and index it in one step, for the common case
where nothing needs to happen between the two (no related Note/
CustomFieldInstance to attach first)."""
doc = Document.objects.create(**kwargs)
backend.add_or_update(doc)
return doc
@pytest.fixture
def indexed_documents(backend: TantivyBackend) -> dict[str, int]:
def indexed_documents(index_document: Callable[..., Document]) -> dict[str, int]:
"""Index a small fixture set, return {label: doc_id} for corpus queries."""
docs = {
"invoice_2020": _index(
backend,
"invoice_2020": index_document(
title="Invoice 2020",
content="invoice total due",
checksum="acc-invoice-2020",
archive_serial_number=100,
),
"invoice_2021": _index(
backend,
"invoice_2021": index_document(
title="Invoice 2021",
content="invoice total due",
checksum="acc-invoice-2021",
archive_serial_number=101,
),
"invoice_2023": _index(
backend,
"invoice_2023": index_document(
title="Invoice 2023",
content="invoice total due",
checksum="acc-invoice-2023",
archive_serial_number=102,
),
"receipt_2022": _index(
backend,
"receipt_2022": index_document(
title="Receipt 2022",
content="receipt total due",
checksum="acc-receipt-2022",
archive_serial_number=103,
),
}
@@ -87,7 +69,7 @@ class TestIssue13568BracketWildcard:
def test_bracket_class_wildcard_matches_only_in_range_years(
self,
backend: TantivyBackend,
matched_ids: Callable[[str], set[int]],
indexed_documents: dict[str, int],
) -> None:
"""
@@ -105,7 +87,7 @@ class TestIssue13568BracketWildcard:
- Only the 2020 and 2021 documents match, proving the bracket
character class survived (issue #13568's original bug)
"""
matched = _matched_ids(backend, "title:202[0-1]*")
matched = matched_ids("title:202[0-1]*")
expected = {
indexed_documents["invoice_2020"],
indexed_documents["invoice_2021"],
@@ -121,6 +103,7 @@ class TestFieldBoosts:
def test_title_boost_ranks_title_match_above_content_only_match(
self,
backend: TantivyBackend,
index_document: Callable[..., Document],
) -> None:
"""
GIVEN:
@@ -132,17 +115,13 @@ class TestFieldBoosts:
- The title match ranks first, proving our title field boost
actually affects ranking
"""
title_match = _index(
backend,
title_match = index_document(
title="urgent",
content="nothing else relevant",
checksum="acc-boost-title",
)
_index(
backend,
index_document(
title="nothing",
content="urgent matter here",
checksum="acc-boost-content",
)
query = parse_user_query(backend._index, "urgent", UTC)
searcher = backend._index.searcher()
@@ -157,6 +136,8 @@ class TestJsonSubpaths:
def test_notes_user_matches_document_with_that_note_author(
self,
backend: TantivyBackend,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
@@ -168,20 +149,20 @@ class TestJsonSubpaths:
- Only the document with alice's note matches
"""
alice = User.objects.create_user(username="alice")
doc_with_note = Document.objects.create(
doc_with_note = DocumentFactory(
title="Has note",
content="x",
checksum="acc-note-with",
)
Note.objects.create(document=doc_with_note, user=alice, note="reminder")
backend.add_or_update(doc_with_note)
_index(backend, title="No note", content="x", checksum="acc-note-without")
matched = _matched_ids(backend, "notes.user:alice")
index_document(title="No note", content="x")
matched = matched_ids("notes.user:alice")
assert matched == {doc_with_note.pk}
def test_custom_fields_name_and_value_combine(
self,
backend: TantivyBackend,
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
@@ -203,10 +184,9 @@ class TestJsonSubpaths:
name="Other Field",
data_type=CustomField.FieldDataType.STRING,
)
matching = Document.objects.create(
matching = DocumentFactory(
title="Matching",
content="x",
checksum="acc-cf-matching",
)
CustomFieldInstance.objects.create(
document=matching,
@@ -214,10 +194,9 @@ class TestJsonSubpaths:
value_text="policy",
)
backend.add_or_update(matching)
non_matching = Document.objects.create(
non_matching = DocumentFactory(
title="Non-matching",
content="x",
checksum="acc-cf-nonmatching",
)
CustomFieldInstance.objects.create(
document=non_matching,
@@ -225,8 +204,7 @@ class TestJsonSubpaths:
value_text="policy",
)
backend.add_or_update(non_matching)
matched = _matched_ids(
backend,
matched = matched_ids(
'custom_fields.name:"Contract Number" custom_fields.value:policy',
)
assert matched == {matching.pk}
@@ -240,7 +218,7 @@ class TestUnregisteredIdFieldFoldsToLiteralText:
def test_tag_id_query_matches_nothing(
self,
backend: TantivyBackend,
matched_ids: Callable[[str], set[int]],
indexed_documents: dict[str, int],
) -> None:
"""
@@ -254,7 +232,7 @@ class TestUnregisteredIdFieldFoldsToLiteralText:
- It folds to a literal text search and matches nothing,
rather than erroring
"""
matched = _matched_ids(backend, "tag_id:5")
matched = matched_ids("tag_id:5")
assert matched == set()
@@ -268,7 +246,8 @@ class TestFuzzyBlendSurvivesWhooshGrammar:
def test_typo_fuzzy_matches_alongside_date_keyword(
self,
backend: TantivyBackend,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
settings,
) -> None:
"""
@@ -287,26 +266,25 @@ class TestFuzzyBlendSurvivesWhooshGrammar:
"""
settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.5
with time_machine.travel(FROZEN_NOW, tick=False):
doc = _index(
backend,
doc = index_document(
title="Receipt March",
content="receipt total due",
checksum="fuzzy-blend-1",
archive_serial_number=900,
)
# Sanity: the exact spelling matches through the exact clause.
assert doc.pk in _matched_ids(backend, "added:today receipt")
assert doc.pk in matched_ids("added:today receipt")
# The regression: the misspelling (one transposition) only
# matches via the fuzzy clause, and "added:today" is
# whoosh-only grammar tantivy's parser rejects, so raw-string
# fuzzy parsing skips the clause entirely and this returns
# nothing. The typo is deliberate; keep codespell away from it.
typo_query = "added:today reciept" # codespell:ignore reciept
assert doc.pk in _matched_ids(backend, typo_query)
assert doc.pk in matched_ids(typo_query)
def test_negated_words_do_not_fuzzy_match(
self,
backend: TantivyBackend,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
settings,
) -> None:
"""
@@ -329,14 +307,12 @@ class TestFuzzyBlendSurvivesWhooshGrammar:
"""
settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.5
with time_machine.travel(FROZEN_NOW, tick=False):
_index(
backend,
index_document(
title="Receipt Archive",
content="receipt archived stack",
checksum="fuzzy-blend-2",
archive_serial_number=901,
)
assert _matched_ids(backend, "added:today total NOT receipt") == set()
assert matched_ids("added:today total NOT receipt") == set()
class TestUnquotedDateKeywordPhrases:
@@ -346,21 +322,20 @@ class TestUnquotedDateKeywordPhrases:
spelling keeps working now that paperless no longer pre-quotes it."""
@pytest.fixture
def period_documents(self, backend: TantivyBackend) -> dict[str, int]:
def period_documents(
self,
index_document: Callable[..., Document],
) -> dict[str, int]:
with time_machine.travel(FROZEN_NOW, tick=False):
in_may = _index(
backend,
in_may = index_document(
title="May Doc",
content="statement",
checksum="kw-may",
archive_serial_number=910,
added=datetime(2026, 5, 20, 12, 0, tzinfo=UTC),
)
in_june = _index(
backend,
in_june = index_document(
title="June Doc",
content="statement",
checksum="kw-june",
archive_serial_number=911,
added=datetime(2026, 6, 10, 12, 0, tzinfo=UTC),
)
@@ -376,7 +351,7 @@ class TestUnquotedDateKeywordPhrases:
)
def test_unquoted_matches_the_same_documents_as_quoted(
self,
backend: TantivyBackend,
matched_ids: Callable[[str], set[int]],
period_documents: dict[str, int],
query: str,
) -> None:
@@ -393,7 +368,7 @@ class TestUnquotedDateKeywordPhrases:
whoosh-compat's own grammar to accept it unquoted natively
"""
with time_machine.travel(FROZEN_NOW, tick=False):
assert _matched_ids(backend, query) == {period_documents["in_may"]}
assert matched_ids(query) == {period_documents["in_may"]}
@pytest.mark.parametrize(
"query",
@@ -409,7 +384,7 @@ class TestUnquotedDateKeywordPhrases:
)
def test_every_phrase_and_date_field_parses_without_error(
self,
backend: TantivyBackend,
matched_ids: Callable[[str], set[int]],
period_documents: dict[str, int],
query: str,
) -> None:
@@ -426,11 +401,12 @@ class TestUnquotedDateKeywordPhrases:
are whoosh-compat's own and are pinned in its own suite
"""
with time_machine.travel(FROZEN_NOW, tick=False):
_matched_ids(backend, query)
matched_ids(query)
def test_text_field_keyword_words_are_ordinary_text(
self,
backend: TantivyBackend,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
period_documents: dict[str, int],
) -> None:
"""
@@ -447,14 +423,12 @@ class TestUnquotedDateKeywordPhrases:
documents do not match
"""
with time_machine.travel(FROZEN_NOW, tick=False):
wordy = _index(
backend,
wordy = index_document(
title="Notes from the previous month",
content="meeting notes",
checksum="kw-text",
archive_serial_number=912,
)
assert _matched_ids(backend, "title:previous month") == {wordy.pk}
assert matched_ids("title:previous month") == {wordy.pk}
class TestFieldAliases:
@@ -464,7 +438,8 @@ class TestFieldAliases:
def test_type_alias_and_canonical_name_match_the_same_document(
self,
backend: TantivyBackend,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
@@ -487,25 +462,22 @@ class TestFieldAliases:
real index
"""
invoice_type = DocumentType.objects.create(name="invoice")
typed = _index(
backend,
typed = index_document(
title="First",
content="quarterly statement",
checksum="alias-type-1",
document_type=invoice_type,
)
_index(
backend,
index_document(
title="Second",
content="invoice mentioned in body",
checksum="alias-type-2",
)
assert _matched_ids(backend, "type:invoice") == {typed.pk}
assert _matched_ids(backend, "document_type:invoice") == {typed.pk}
assert matched_ids("type:invoice") == {typed.pk}
assert matched_ids("document_type:invoice") == {typed.pk}
def test_path_alias_and_canonical_name_match_the_same_document(
self,
backend: TantivyBackend,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
@@ -523,18 +495,14 @@ class TestFieldAliases:
real index
"""
archive = StoragePath.objects.create(name="archive", path="archive/{title}")
stored = _index(
backend,
stored = index_document(
title="Stored",
content="quarterly statement",
checksum="alias-path-1",
storage_path=archive,
)
_index(
backend,
index_document(
title="Loose",
content="archive mentioned in body",
checksum="alias-path-2",
)
assert _matched_ids(backend, "path:archive") == {stored.pk}
assert _matched_ids(backend, "storage_path:archive") == {stored.pk}
assert matched_ids("path:archive") == {stored.pk}
assert matched_ids("storage_path:archive") == {stored.pk}
src/documents/tests/search/test_cjk_emit_registry.py
+8 -10
View File
@@ -10,19 +10,17 @@ from __future__ import annotations
import pytest
import tantivy
import whoosh_compat as wc
import whoosh_compat.ast as wc_ast
from documents.search._query import _CJK_BIGRAM_FIELDS
from documents.search._query import _get_emit_field_registry
from documents.search._schema import field_descriptors
from documents.search._tokenizer import register_tokenizers
from documents.tests.search._ast_helpers import BIGRAM_CONTENT
from documents.tests.search._ast_helpers import CONTENT
pytestmark = pytest.mark.search
_BIGRAM_CONTENT = wc.FieldRef("bigram_content")
_CONTENT = wc.FieldRef("content")
class TestEmitFieldRegistry:
def test_a_multi_bigram_run_requires_every_bigram_even_under_an_or(
@@ -41,8 +39,8 @@ class TestEmitFieldRegistry:
"""
tree = wc_ast.Or(
children=(
wc_ast.Term(field=_BIGRAM_CONTENT, text="東京都"),
wc_ast.Term(field=_CONTENT, text="report"),
wc_ast.Term(field=BIGRAM_CONTENT, text="東京都"),
wc_ast.Term(field=CONTENT, text="report"),
),
)
@@ -52,11 +50,11 @@ class TestEmitFieldRegistry:
children=(
wc_ast.And(
children=(
wc_ast.Term(field=_BIGRAM_CONTENT, text="東京"),
wc_ast.Term(field=_BIGRAM_CONTENT, text="京都"),
wc_ast.Term(field=BIGRAM_CONTENT, text="東京"),
wc_ast.Term(field=BIGRAM_CONTENT, text="京都"),
),
),
wc_ast.Term(field=_CONTENT, text="report"),
wc_ast.Term(field=CONTENT, text="report"),
),
)
@@ -95,7 +93,7 @@ class TestEmitFieldRegistry:
writer.commit()
index.reload()
resolved = _get_emit_field_registry(None).resolve(_BIGRAM_CONTENT)
resolved = _get_emit_field_registry(None).resolve(BIGRAM_CONTENT)
assert resolved is not None
tokens = resolved.spec.analyzer(text)
src/documents/tests/search/test_cjk_leaf_rewrite.py
+58 -65
View File
@@ -19,26 +19,19 @@ from documents.search._query import _FIELD_BOOSTS
from documents.search._query import _get_emit_field_registry
from documents.search._query import _widen_leaf
from documents.search._registry import get_field_registry
from documents.tests.search._ast_helpers import BIGRAM_CONTENT
from documents.tests.search._ast_helpers import BIGRAM_TITLE
from documents.tests.search._ast_helpers import CONTENT
from documents.tests.search._ast_helpers import NOTES
from documents.tests.search._ast_helpers import TITLE
from documents.tests.search._ast_helpers import bigram
from documents.tests.search._ast_helpers import content
if TYPE_CHECKING:
from collections.abc import Callable
pytestmark = pytest.mark.search
_CONTENT = wc.FieldRef("content")
_TITLE = wc.FieldRef("title")
_NOTES = wc.FieldRef("notes", "note")
_BIGRAM_CONTENT = wc.FieldRef("bigram_content")
_BIGRAM_TITLE = wc.FieldRef("bigram_title")
def _content(text: str) -> wc_ast.Term:
return wc_ast.Term(field=_CONTENT, text=text)
def _bigram(text: str) -> wc_ast.Term:
return wc_ast.Term(field=_BIGRAM_CONTENT, text=text)
def _widened(original: wc_ast.Node, bigram_side: wc_ast.Node) -> wc_ast.Or:
return wc_ast.Or(children=(original, bigram_side))
@@ -61,9 +54,9 @@ class TestTheHook:
@pytest.mark.parametrize(
"leaf",
[
pytest.param(_content("invoice"), id="latin_term"),
pytest.param(content("invoice"), id="latin_term"),
pytest.param(
wc_ast.Term(field=_NOTES, text="東京"),
wc_ast.Term(field=NOTES, text="東京"),
id="non_default_field",
),
pytest.param(wc_ast.Term(field=None, text="東京"), id="unfielded"),
@@ -96,7 +89,7 @@ class TestTheHook:
- Neither cjk_terms nor latin_terms gains a piece, and the
leaf is returned unchanged rather than becoming an empty Or
"""
leaf = _content("⺀")
leaf = content("⺀")
assert _widen_cjk_leaf(leaf) is leaf
@@ -111,7 +104,7 @@ class TestTheHook:
copy: analyze() only keeps the leaf's own enclosing-group
analysis for that exact object
"""
leaf = _content("東京")
leaf = content("東京")
widened = _widen_cjk_leaf(leaf)
@@ -128,11 +121,11 @@ class TestTheHook:
- The bigram side targets bigram_title, the leaf's own field's
companion, so a fielded term stays fielded
"""
leaf = wc_ast.Term(field=_TITLE, text="東京")
leaf = wc_ast.Term(field=TITLE, text="東京")
assert _widen_cjk_leaf(leaf) == _widened(
leaf,
wc_ast.Term(field=_BIGRAM_TITLE, text="東京"),
wc_ast.Term(field=BIGRAM_TITLE, text="東京"),
)
def test_each_cjk_run_gets_its_own_bigram_leaf(self) -> None:
@@ -151,11 +144,11 @@ class TestTheHook:
any one of its runs, which is what the separate bigram
clause did before this change
"""
leaf = _content("東京・大阪")
leaf = content("東京・大阪")
assert _widen_cjk_leaf(leaf) == _widened(
leaf,
wc_ast.Or(children=(_bigram("東京"), _bigram("大阪"))),
wc_ast.Or(children=(bigram("東京"), bigram("大阪"))),
)
def test_each_run_keeps_its_own_bigrams_required(self) -> None:
@@ -172,11 +165,11 @@ class TestTheHook:
it does not inherit from the enclosing group. Without this,
東京都 would match a document holding only 京都
"""
assert _analyze(_content("東京都・大阪府")) == wc_ast.Or(
assert _analyze(content("東京都・大阪府")) == wc_ast.Or(
children=(
wc_ast.And(children=(_content("東京都"), _content("大阪府"))),
wc_ast.And(children=(_bigram("東京"), _bigram("京都"))),
wc_ast.And(children=(_bigram("大阪"), _bigram("阪府"))),
wc_ast.And(children=(content("東京都"), content("大阪府"))),
wc_ast.And(children=(bigram("東京"), bigram("京都"))),
wc_ast.And(children=(bigram("大阪"), bigram("阪府"))),
),
)
@@ -191,9 +184,9 @@ class TestTheHook:
every token at position 0, so a positional Phrase against it
could never match
"""
leaf = wc_ast.Phrase(field=_CONTENT, text="東京都")
leaf = wc_ast.Phrase(field=CONTENT, text="東京都")
assert _widen_cjk_leaf(leaf) == _widened(leaf, _bigram("東京都"))
assert _widen_cjk_leaf(leaf) == _widened(leaf, bigram("東京都"))
def test_a_multi_word_phrase_requires_every_run(self) -> None:
"""
@@ -208,27 +201,27 @@ class TestTheHook:
the quoted phrase match strictly more than the same two
words unquoted, and quoting must not widen a search
"""
leaf = wc_ast.Phrase(field=_CONTENT, text="東京都 大阪府")
leaf = wc_ast.Phrase(field=CONTENT, text="東京都 大阪府")
assert _widen_cjk_leaf(leaf) == _widened(
leaf,
wc_ast.And(children=(_bigram("東京都"), _bigram("大阪府"))),
wc_ast.And(children=(bigram("東京都"), bigram("大阪府"))),
)
@pytest.mark.parametrize(
("leaf", "alternative"),
[
pytest.param(
_content("東京-report"),
wc_ast.And(children=(_bigram("東京"), _content("report"))),
content("東京-report"),
wc_ast.And(children=(bigram("東京"), content("report"))),
id="separated_term",
),
pytest.param(
wc_ast.Phrase(field=_CONTENT, text="東京 report"),
wc_ast.And(children=(_bigram("東京"), _content("report"))),
wc_ast.Phrase(field=CONTENT, text="東京 report"),
wc_ast.And(children=(bigram("東京"), content("report"))),
id="phrase",
),
pytest.param(_content("東京report"), _bigram("東京"), id="glued"),
pytest.param(content("東京report"), bigram("東京"), id="glued"),
],
)
def test_latin_is_required_only_where_the_analyzer_splits_it_off(
@@ -265,9 +258,9 @@ class TestTheHook:
in And (a single piece collapses to itself), even though
the CJK side contributed nothing
"""
leaf = _content("⺀report")
leaf = content("⺀report")
assert _widen_cjk_leaf(leaf) == _widened(leaf, _content("report"))
assert _widen_cjk_leaf(leaf) == _widened(leaf, content("report"))
def test_new_nodes_carry_the_leaf_span(self) -> None:
"""
@@ -280,7 +273,7 @@ class TestTheHook:
span, so an emit-time diagnostic still points into the
query text
"""
leaf = wc_ast.Term(field=_CONTENT, text="東京・大阪", startchar=3, endchar=8)
leaf = wc_ast.Term(field=CONTENT, text="東京・大阪", startchar=3, endchar=8)
widened = _widen_cjk_leaf(leaf)
@@ -298,9 +291,9 @@ class TestAnalyzedTree:
@pytest.mark.parametrize(
"node",
[
pytest.param(wc_ast.Prefix(field=_CONTENT, text="東京"), id="cjk_prefix"),
pytest.param(wc_ast.Prefix(field=CONTENT, text="東京"), id="cjk_prefix"),
pytest.param(
wc_ast.Wildcard(field=_CONTENT, pattern="東*"),
wc_ast.Wildcard(field=CONTENT, pattern="東*"),
id="cjk_wildcard",
),
],
@@ -340,13 +333,13 @@ class TestAnalyzedTree:
still negated under NOT. A NOT left holding nothing would
turn into "match everything" instead
"""
resolved = _get_emit_field_registry(None).resolve(_BIGRAM_CONTENT)
resolved = _get_emit_field_registry(None).resolve(BIGRAM_CONTENT)
assert resolved is not None
bigrams = resolved.spec.analyzer(long_cjk_run)
assert _analyze(build(_content(long_cjk_run))) == build(
assert _analyze(build(content(long_cjk_run))) == build(
wc_ast.And(
children=tuple(_bigram(token) for token in dict.fromkeys(bigrams)),
children=tuple(bigram(token) for token in dict.fromkeys(bigrams)),
),
)
@@ -361,11 +354,11 @@ class TestAnalyzedTree:
has no bigram. An accepted gap, the same on the positive and
the negated side
"""
analyzed = _analyze(_content("東・大阪"))
analyzed = _analyze(content("東・大阪"))
assert analyzed == _widened(
wc_ast.And(children=(_content("東"), _content("大阪"))),
_bigram("大阪"),
wc_ast.And(children=(content("東"), content("大阪"))),
bigram("大阪"),
)
def test_the_title_boost_wraps_the_widened_title_leaf(self) -> None:
@@ -393,14 +386,14 @@ class TestAnalyzedTree:
assert (
wc_ast.Boosted(
child=_widened(
wc_ast.Term(field=_TITLE, text="東京"),
wc_ast.Term(field=_BIGRAM_TITLE, text="東京"),
wc_ast.Term(field=TITLE, text="東京"),
wc_ast.Term(field=BIGRAM_TITLE, text="東京"),
),
boost=_FIELD_BOOSTS["title"],
)
in analyzed.children
)
assert {_BIGRAM_CONTENT, wc.FieldRef("bigram_tag")} <= {
assert {BIGRAM_CONTENT, wc.FieldRef("bigram_tag")} <= {
child.field for child in analyzed.children if isinstance(child, wc_ast.Term)
}
@@ -418,18 +411,18 @@ class TestAnalyzedTree:
alternative is not a single child here and _widened() does
not fit; the tree is written out literally
"""
tree = wc_ast.And(children=(_content("東京・大阪"), _content("report")))
tree = wc_ast.And(children=(content("東京・大阪"), content("report")))
assert _analyze(tree) == wc_ast.And(
children=(
wc_ast.Or(
children=(
wc_ast.And(children=(_content("東京"), _content("大阪"))),
_bigram("東京"),
_bigram("大阪"),
wc_ast.And(children=(content("東京"), content("大阪"))),
bigram("東京"),
bigram("大阪"),
),
),
_content("report"),
content("report"),
),
)
@@ -443,15 +436,15 @@ class TestAnalyzedTree:
- The original side ORs its tokens, as the group says, and the
hook's Or flattens into the group
"""
tree = wc_ast.Or(children=(_content("東京・大阪"), _content("report")))
tree = wc_ast.Or(children=(content("東京・大阪"), content("report")))
assert _analyze(tree) == wc_ast.Or(
children=(
_content("東京"),
_content("大阪"),
_bigram("東京"),
_bigram("大阪"),
_content("report"),
content("東京"),
content("大阪"),
bigram("東京"),
bigram("大阪"),
content("report"),
),
)
@@ -460,12 +453,12 @@ class TestAnalyzedTree:
[
pytest.param(lambda leaf: wc_ast.Not(child=leaf), id="not"),
pytest.param(
lambda leaf: wc_ast.AndNot(positive=_content("invoice"), negative=leaf),
lambda leaf: wc_ast.AndNot(positive=content("invoice"), negative=leaf),
id="andnot",
),
pytest.param(
lambda leaf: wc_ast.AndMaybe(
required=_content("invoice"),
required=content("invoice"),
optional=leaf,
),
id="andmaybe",
@@ -473,7 +466,7 @@ class TestAnalyzedTree:
pytest.param(
lambda leaf: wc_ast.Require(
scored=leaf,
filter_only=_content("invoice"),
filter_only=content("invoice"),
),
id="require",
),
@@ -500,6 +493,6 @@ class TestAnalyzedTree:
position. A negated CJK leaf is widened too, so a negation
excludes exactly what the positive search would match
"""
assert _analyze(build(_content("東京"))) == build(
_widened(_content("東京"), _bigram("東京")),
assert _analyze(build(content("東京"))) == build(
_widened(content("東京"), bigram("東京")),
)
src/documents/tests/search/test_compact_date_forms.py
+28 -42
View File
@@ -20,67 +20,53 @@ from typing import TYPE_CHECKING
import pytest
from documents.models import Document
if TYPE_CHECKING:
from documents.search._backend import TantivyBackend
from collections.abc import Callable
from documents.models import Document
pytestmark = [pytest.mark.search, pytest.mark.django_db]
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
return set(backend.search_ids(query, user=None))
def _index(backend: TantivyBackend, **kwargs: object) -> Document:
doc = Document.objects.create(**kwargs)
backend.add_or_update(doc)
return doc
@pytest.fixture
def docs(backend: TantivyBackend) -> dict[str, int]:
def docs(index_document: Callable[..., Document]) -> dict[str, int]:
return {
"instant": _index(
backend,
"instant": index_document(
title="On the instant",
content="x",
checksum="compact-date-instant",
added=datetime(2005, 3, 4, 15, 30, tzinfo=UTC),
).pk,
"same_day": _index(
backend,
"same_day": index_document(
title="Same day, other hour",
content="x",
checksum="compact-date-same-day",
added=datetime(2005, 3, 4, 9, 0, tzinfo=UTC),
).pk,
"next_day": _index(
backend,
"next_day": index_document(
title="Next day, same hour",
content="x",
checksum="compact-date-next-day",
added=datetime(2005, 3, 5, 15, 30, tzinfo=UTC),
).pk,
}
def test_fourteen_digits_is_a_single_instant(
backend: TantivyBackend,
docs: dict[str, int],
) -> None:
"""
GIVEN:
- Three documents indexed on the ``added`` DATETIME fast field:
one at 2005-03-04T15:30:00, one on the same calendar day at a
different hour, and one on the next day at the same hour
WHEN:
- Searching with the 14-digit compact date form
``added:20050304153000``
THEN:
- Only the document at that exact instant matches; the same-day
document is what tells this apart from the 8-digit day-window
form, and the next-day document from a form that ignored the
time of day altogether
"""
assert _matched_ids(backend, "added:20050304153000") == {docs["instant"]}
class TestCompactDateForms:
def test_fourteen_digits_is_a_single_instant(
self,
matched_ids: Callable[[str], set[int]],
docs: dict[str, int],
) -> None:
"""
GIVEN:
- Three documents indexed on the ``added`` DATETIME fast field:
one at 2005-03-04T15:30:00, one on the same calendar day at a
different hour, and one on the next day at the same hour
WHEN:
- Searching with the 14-digit compact date form
``added:20050304153000``
THEN:
- Only the document at that exact instant matches; the same-day
document is what tells this apart from the 8-digit day-window
form, and the next-day document from a form that ignored the
time of day altogether
"""
assert matched_ids("added:20050304153000") == {docs["instant"]}
src/documents/tests/search/test_date_keyword_phrase_removal.py
+9 -22
View File
@@ -19,24 +19,14 @@ from typing import TYPE_CHECKING
import pytest
from documents.models import Document
if TYPE_CHECKING:
from documents.search._backend import TantivyBackend
from collections.abc import Callable
from documents.models import Document
pytestmark = [pytest.mark.search, pytest.mark.django_db]
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
return set(backend.search_ids(query, user=None))
def _index(backend: TantivyBackend, **kwargs: object) -> Document:
doc = Document.objects.create(**kwargs)
backend.add_or_update(doc)
return doc
class TestQuotedStringContainingDateKeywordText:
"""A quoted title phrase containing the literal text
"added:previous month" as running words must match on that literal
@@ -46,7 +36,8 @@ class TestQuotedStringContainingDateKeywordText:
def test_matches_only_the_literal_phrase(
self,
backend: TantivyBackend,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
@@ -67,11 +58,9 @@ class TestQuotedStringContainingDateKeywordText:
never spill into an unfielded search for "previous" and
"month" across the default search fields
"""
literal = _index(
backend,
literal = index_document(
title="see added:previous month notes",
content="quarterly filing",
checksum="dkp-literal",
archive_serial_number=920,
)
# Under the deleted rewrite, this decoy would incorrectly match:
@@ -79,13 +68,11 @@ class TestQuotedStringContainingDateKeywordText:
# corrupted parse required as title phrases, and its content
# supplies "previous" and "month" as the decomposed word-match
# clauses the rewrite turned the middle of the phrase into.
decoy = _index(
backend,
decoy = index_document(
title="see added: quarterly report notes",
content="we reviewed the previous statement about month end",
checksum="dkp-decoy",
archive_serial_number=921,
)
query = 'title:"see added:previous month notes"'
assert _matched_ids(backend, query) == {literal.pk}
assert decoy.pk not in _matched_ids(backend, query)
assert matched_ids(query) == {literal.pk}
assert decoy.pk not in matched_ids(query)
src/documents/tests/search/test_date_keyword_timezone.py
+8 -21
View File
@@ -32,32 +32,23 @@ from typing import TYPE_CHECKING
import pytest
import time_machine
from documents.models import Document
if TYPE_CHECKING:
from collections.abc import Callable
from pytest_django.fixtures import SettingsWrapper
from documents.search._backend import TantivyBackend
from documents.models import Document
pytestmark = [pytest.mark.search, pytest.mark.django_db]
FROZEN_NOW = datetime(2026, 6, 15, 2, 0, tzinfo=UTC)
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
return set(backend.search_ids(query, user=None))
def _index(backend: TantivyBackend, **kwargs: object) -> Document:
doc = Document.objects.create(**kwargs)
backend.add_or_update(doc)
return doc
class TestDateKeywordUsesTheActiveTimezone:
def test_today_matches_the_new_york_calendar_day_not_the_utc_one(
self,
backend: TantivyBackend,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
settings: SettingsWrapper,
) -> None:
"""
@@ -79,22 +70,18 @@ class TestDateKeywordUsesTheActiveTimezone:
"""
settings.TIME_ZONE = "America/New_York"
with time_machine.travel(FROZEN_NOW, tick=False):
in_ny_today = _index(
backend,
in_ny_today = index_document(
title="NY today",
content="x",
checksum="tz-keyword-ny-today",
added=datetime(2026, 6, 14, 20, 0, tzinfo=UTC),
)
# Not captured: the exact-set assertion below already proves
# this document (inside a naive UTC-calendar-day window, but
# outside New York's actual "today") does not match.
_index(
backend,
index_document(
title="UTC calendar day only",
content="x",
checksum="tz-keyword-utc-calendar-day-only",
added=datetime(2026, 6, 15, 10, 0, tzinfo=UTC),
)
assert _matched_ids(backend, "added:today") == {in_ny_today.pk}
assert matched_ids("added:today") == {in_ny_today.pk}
src/documents/tests/search/test_documented_syntax.py
+39 -61
View File
@@ -24,16 +24,18 @@ from typing import TYPE_CHECKING
import pytest
import time_machine
from documents.models import Document
from documents.models import Note
from documents.models import Tag
from documents.search._errors import InvalidDateQuery
from documents.tests.factories import DocumentFactory
if TYPE_CHECKING:
from collections.abc import Callable
from collections.abc import Generator
from django.contrib.auth.models import User
from documents.models import Document
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
@@ -45,37 +47,23 @@ FROZEN_NOW = datetime(2026, 6, 15, 12, 0, tzinfo=UTC)
DOC_CHECKSUM = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
return set(backend.search_ids(query, user=None))
def _index(backend: TantivyBackend, **kwargs: object) -> Document:
doc = Document.objects.create(**kwargs)
backend.add_or_update(doc)
return doc
class TestLogicalExpressions:
@pytest.fixture
def docs(self, backend: TantivyBackend) -> dict[str, int]:
def docs(self, index_document: Callable[..., Document]) -> dict[str, int]:
return {
"secret": _index(
backend,
"secret": index_document(
title="Invoice one",
content="invoice secret contents",
checksum="doc-syntax-secret",
).pk,
"plain": _index(
backend,
"plain": index_document(
title="Invoice two",
content="invoice ordinary contents",
checksum="doc-syntax-plain",
).pk,
}
def test_not_excludes_a_term(
self,
backend: TantivyBackend,
matched_ids: Callable[[str], set[int]],
docs: dict[str, int],
) -> None:
"""
@@ -86,11 +74,11 @@ class TestLogicalExpressions:
THEN:
- Only the document without "secret" matches
"""
assert _matched_ids(backend, "invoice NOT secret") == {docs["plain"]}
assert matched_ids("invoice NOT secret") == {docs["plain"]}
def test_leading_hyphen_requires_the_term_instead_of_excluding_it(
self,
backend: TantivyBackend,
matched_ids: Callable[[str], set[int]],
docs: dict[str, int],
) -> None:
"""
@@ -104,11 +92,11 @@ class TestLogicalExpressions:
indexed as the plain term "secret" and the query becomes an
AND rather than an exclusion, exactly as the docs warn
"""
assert _matched_ids(backend, "invoice -secret") == {docs["secret"]}
assert matched_ids("invoice -secret") == {docs["secret"]}
def test_or_inside_parentheses_matches_either_branch(
self,
backend: TantivyBackend,
matched_ids: Callable[[str], set[int]],
docs: dict[str, int],
) -> None:
"""
@@ -120,14 +108,15 @@ class TestLogicalExpressions:
THEN:
- Both documents match
"""
matched = _matched_ids(backend, "invoice AND (secret OR ordinary)")
matched = matched_ids("invoice AND (secret OR ordinary)")
assert matched == {docs["secret"], docs["plain"]}
class TestPhraseSearch:
def test_quoted_phrase_requires_the_words_in_order(
self,
backend: TantivyBackend,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
@@ -138,14 +127,12 @@ class TestPhraseSearch:
- The in-order phrase matches, and the same words reordered do
not
"""
doc = _index(
backend,
doc = index_document(
title="Phrase",
content="the quick brown fox jumps",
checksum="doc-syntax-phrase",
)
assert _matched_ids(backend, '"quick brown fox"') == {doc.pk}
assert _matched_ids(backend, '"brown quick fox"') == set()
assert matched_ids('"quick brown fox"') == {doc.pk}
assert matched_ids('"brown quick fox"') == set()
class TestTagCommaList:
@@ -166,6 +153,7 @@ class TestTagCommaList:
def test_comma_list_requires_every_listed_tag(
self,
backend: TantivyBackend,
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
@@ -181,30 +169,22 @@ class TestTagCommaList:
unpaid = Tag.objects.create(name="unpaid")
archived = Tag.objects.create(name="archived")
both = Document.objects.create(
title="Both tags",
content="body",
checksum="doc-syntax-tag-both",
)
both = DocumentFactory(title="Both tags", content="body")
both.tags.add(bills, unpaid)
backend.add_or_update(both)
one = Document.objects.create(
title="One tag",
content="body",
checksum="doc-syntax-tag-one",
)
one = DocumentFactory(title="One tag", content="body")
one.tags.add(bills, archived)
backend.add_or_update(one)
assert _matched_ids(backend, "tag:bills,unpaid") == {both.pk}
assert _matched_ids(backend, "tag:bills") == {both.pk, one.pk}
assert matched_ids("tag:bills,unpaid") == {both.pk}
assert matched_ids("tag:bills") == {both.pk, one.pk}
class TestArchiveMetadataFields:
@pytest.fixture
def doc(self, backend: TantivyBackend, admin_user: User) -> Document:
doc = Document.objects.create(
doc = DocumentFactory(
title="Metadata",
content="body",
checksum=DOC_CHECKSUM,
@@ -237,7 +217,7 @@ class TestArchiveMetadataFields:
)
def test_documented_metadata_query_matches(
self,
backend: TantivyBackend,
matched_ids: Callable[[str], set[int]],
doc: Document,
query: str,
) -> None:
@@ -253,7 +233,7 @@ class TestArchiveMetadataFields:
THEN:
- Each one matches the document
"""
assert _matched_ids(backend, query) == {doc.pk}
assert matched_ids(query) == {doc.pk}
@pytest.mark.parametrize(
"query",
@@ -265,7 +245,7 @@ class TestArchiveMetadataFields:
)
def test_partial_or_uppercase_checksum_matches_nothing(
self,
backend: TantivyBackend,
matched_ids: Callable[[str], set[int]],
doc: Document,
query: str,
) -> None:
@@ -279,7 +259,7 @@ class TestArchiveMetadataFields:
- Nothing matches, as the docs say only a complete, lowercase
checksum matches as an exact value
"""
assert _matched_ids(backend, query) == set()
assert matched_ids(query) == set()
class TestDocumentedDateForms:
@@ -289,7 +269,7 @@ class TestDocumentedDateForms:
yield
@pytest.fixture
def dated(self, backend: TantivyBackend) -> dict[str, int]:
def dated(self, index_document: Callable[..., Document]) -> dict[str, int]:
stamps = {
"today": datetime(2026, 6, 15, 9, 0, tzinfo=UTC),
"yesterday": datetime(2026, 6, 14, 9, 0, tzinfo=UTC),
@@ -300,11 +280,9 @@ class TestDocumentedDateForms:
"old": datetime(2005, 3, 4, 15, 30, tzinfo=UTC),
}
return {
label: _index(
backend,
label: index_document(
title=label,
content="dated body",
checksum=f"doc-syntax-date-{label}",
added=stamp,
).pk
for label, stamp in stamps.items()
@@ -335,7 +313,7 @@ class TestDocumentedDateForms:
)
def test_documented_date_form_matches_its_day_or_month(
self,
backend: TantivyBackend,
matched_ids: Callable[[str], set[int]],
dated: dict[str, int],
query: str,
label: str,
@@ -355,7 +333,7 @@ class TestDocumentedDateForms:
- Each form matches exactly the document dated on its day or
within its month
"""
assert _matched_ids(backend, query) == {dated[label]}
assert matched_ids(query) == {dated[label]}
@pytest.mark.parametrize(
"query",
@@ -381,7 +359,7 @@ class TestDocumentedDateForms:
)
def test_forms_the_docs_warn_about_match_nothing(
self,
backend: TantivyBackend,
matched_ids: Callable[[str], set[int]],
dated: dict[str, int],
query: str,
) -> None:
@@ -400,11 +378,11 @@ class TestDocumentedDateForms:
- Nothing matches, exactly as the docs warn, rather than
presenting these as usable spellings
"""
assert _matched_ids(backend, query) == set()
assert matched_ids(query) == set()
def test_bare_timestamp_is_rejected_rather_than_matching_nothing(
self,
backend: TantivyBackend,
matched_ids: Callable[[str], set[int]],
dated: dict[str, int],
) -> None:
"""
@@ -423,13 +401,13 @@ class TestDocumentedDateForms:
prefix the date grammar's tokenizer first split on
"""
with pytest.raises(InvalidDateQuery) as exc_info:
_matched_ids(backend, "added:2005-03-04T15:30:00Z")
matched_ids("added:2005-03-04T15:30:00Z")
assert exc_info.value.field == "added"
assert exc_info.value.value == "2005-03-04T15:30:00Z"
def test_relative_offset_as_a_range_bound_is_a_real_window(
self,
backend: TantivyBackend,
matched_ids: Callable[[str], set[int]],
dated: dict[str, int],
) -> None:
"""
@@ -447,14 +425,14 @@ class TestDocumentedDateForms:
the offset itself and not a whole-day rounding of it, as
the docs say next to the warning about the standalone form
"""
assert _matched_ids(backend, "added:['-1 week' to now]") == {
assert matched_ids("added:['-1 week' to now]") == {
dated["today"],
dated["yesterday"],
}
def test_double_quoted_range_bound_is_rejected(
self,
backend: TantivyBackend,
matched_ids: Callable[[str], set[int]],
dated: dict[str, int],
) -> None:
"""
@@ -471,5 +449,5 @@ class TestDocumentedDateForms:
attached and is not a recognizable date
"""
with pytest.raises(InvalidDateQuery) as exc_info:
_matched_ids(backend, 'added:["2005-03-04" to 2005-03-05]')
matched_ids('added:["2005-03-04" to 2005-03-05]')
assert exc_info.value.value == '"2005-03-04"'
src/documents/tests/search/test_error_routing.py
+5 -11
View File
@@ -8,9 +8,9 @@ from __future__ import annotations
import logging
from datetime import UTC
from typing import TYPE_CHECKING
import pytest
import tantivy
from whoosh_compat.errors import Diagnostic
from whoosh_compat.errors import DiagnosticKind
from whoosh_compat.errors import QueryError
@@ -22,22 +22,16 @@ from documents.search._errors import SearchQueryError
from documents.search._query import _map_emit_error
from documents.search._query import _single_diagnostic_to_error
from documents.search._query import parse_user_query
from documents.search._schema import build_schema
from documents.search._tokenizer import register_tokenizers
if TYPE_CHECKING:
import tantivy
pytestmark = pytest.mark.search
_LIBRARY_PROSE = "INTERNAL LIBRARY WORDING WITH raw tantivy detail"
@pytest.fixture(scope="module")
def query_index() -> tantivy.Index:
"""An in-memory, unstemmed index; these tests only parse, never index."""
idx = tantivy.Index(build_schema(), path=None)
register_tokenizers(idx, "")
return idx
def _diagnostic(
kind: DiagnosticKind,
*,
src/documents/tests/search/test_exists_on_json_fields.py
+5 -10
View File
@@ -19,9 +19,9 @@ from __future__ import annotations
import logging
from datetime import UTC
from typing import TYPE_CHECKING
import pytest
import tantivy
from whoosh_compat.errors import Diagnostic
from whoosh_compat.errors import DiagnosticKind
from whoosh_compat.errors import QueryError
@@ -32,8 +32,10 @@ from whoosh_compat.fields import FieldRef
from documents.search._errors import SearchQueryError
from documents.search._query import _map_emit_error
from documents.search._query import parse_user_query
from documents.search._schema import build_schema
from documents.search._tokenizer import register_tokenizers
if TYPE_CHECKING:
import tantivy
pytestmark = pytest.mark.search
@@ -48,13 +50,6 @@ EXISTS_QUERIES = [
]
@pytest.fixture(scope="module")
def query_index() -> tantivy.Index:
idx = tantivy.Index(build_schema(), path=None)
register_tokenizers(idx, "")
return idx
class TestJsonExistsIsUserError:
@pytest.mark.parametrize("query", EXISTS_QUERIES)
def test_query_is_a_400_that_emits_no_error_log(
src/documents/tests/search/test_fuzzy_alternative.py
+19 -22
View File
@@ -14,11 +14,12 @@ import whoosh_compat as wc
import whoosh_compat.ast as wc_ast
from documents.search._query import _fuzzy_alternative
from documents.tests.search._ast_helpers import CONTENT
from documents.tests.search._ast_helpers import TITLE
from documents.tests.search._ast_helpers import content
pytestmark = pytest.mark.search
_CONTENT = wc.FieldRef("content")
_TITLE = wc.FieldRef("title")
# 130 characters, past the analyzer's 129-byte remove_long limit (128 is
# kept, 129 is dropped), so the index never holds it and neither side
@@ -26,10 +27,6 @@ _TITLE = wc.FieldRef("title")
_TOO_LONG = "x" * 130
def _content(text: str) -> wc_ast.Term:
return wc_ast.Term(field=_CONTENT, text=text)
def _fuzzy(field: wc.FieldRef, text: str) -> wc_ast.Fuzzy:
return wc_ast.Fuzzy(field=field, text=text, distance=1, prefix=True)
@@ -45,7 +42,7 @@ class TestTheAlternative:
- It is a single Fuzzy leaf on the same field, distance 1 and
prefix matching, which is what the old clause used
"""
assert _fuzzy_alternative(_content("invoice")) == _fuzzy(_CONTENT, "invoice")
assert _fuzzy_alternative(content("invoice")) == _fuzzy(CONTENT, "invoice")
def test_the_leafs_own_field_is_used(self) -> None:
"""
@@ -58,8 +55,8 @@ class TestTheAlternative:
fields. This is the fielding fix: today's clause searches a
fielded word everywhere
"""
assert _fuzzy_alternative(wc_ast.Term(field=_TITLE, text="invoice")) == _fuzzy(
_TITLE,
assert _fuzzy_alternative(wc_ast.Term(field=TITLE, text="invoice")) == _fuzzy(
TITLE,
"invoice",
)
@@ -74,8 +71,8 @@ class TestTheAlternative:
side, which is OR'd for an unfielded term, and would lose
today's per-word recall inside a term
"""
assert _fuzzy_alternative(_content("COVID-19")) == wc_ast.Or(
children=(_fuzzy(_CONTENT, "covid"), _fuzzy(_CONTENT, "19")),
assert _fuzzy_alternative(content("COVID-19")) == wc_ast.Or(
children=(_fuzzy(CONTENT, "covid"), _fuzzy(CONTENT, "19")),
)
def test_words_are_split_where_the_index_splits_them(self) -> None:
@@ -88,7 +85,7 @@ class TestTheAlternative:
- The Fuzzy text is the analyzer's output, not the raw text,
so it is in the same shape as the index terms
"""
assert _fuzzy_alternative(_content("Éclair")) == _fuzzy(_CONTENT, "eclair")
assert _fuzzy_alternative(content("Éclair")) == _fuzzy(CONTENT, "eclair")
def test_the_splitter_does_not_stem(self) -> None:
"""
@@ -101,8 +98,8 @@ class TestTheAlternative:
pattern_normalizer, which stems it once; stemming here too
would search for a term the index does not hold
"""
assert _fuzzy_alternative(_content("universities")) == _fuzzy(
_CONTENT,
assert _fuzzy_alternative(content("universities")) == _fuzzy(
CONTENT,
"universities",
)
@@ -130,7 +127,7 @@ class TestTheAlternative:
that can never match, making a query return LESS with fuzzy
on than off
"""
assert _fuzzy_alternative(_content(text)) is None
assert _fuzzy_alternative(content(text)) is None
def test_a_one_character_word_is_dropped_from_a_longer_term(self) -> None:
"""
@@ -142,7 +139,7 @@ class TestTheAlternative:
- Only the real word survives. A one-character prefix fuzzy
term matches every term in the field
"""
assert _fuzzy_alternative(_content("h52.1")) == _fuzzy(_CONTENT, "h52")
assert _fuzzy_alternative(content("h52.1")) == _fuzzy(CONTENT, "h52")
def test_the_leaf_span_is_copied(self) -> None:
"""
@@ -155,7 +152,7 @@ class TestTheAlternative:
- Every node it builds carries that span, so an emit-time
diagnostic still points into the query text
"""
leaf = wc_ast.Term(field=_CONTENT, text="ab-cd", startchar=4, endchar=9)
leaf = wc_ast.Term(field=CONTENT, text="ab-cd", startchar=4, endchar=9)
alternative = _fuzzy_alternative(leaf)
@@ -181,10 +178,10 @@ class TestPhrasesAreNotWidened:
quoted phrase match strictly more than the same two words
unquoted
"""
leaf = wc_ast.Phrase(field=_CONTENT, text="tax report")
leaf = wc_ast.Phrase(field=CONTENT, text="tax report")
assert _fuzzy_alternative(leaf) == wc_ast.And(
children=(_fuzzy(_CONTENT, "tax"), _fuzzy(_CONTENT, "report")),
children=(_fuzzy(CONTENT, "tax"), _fuzzy(CONTENT, "report")),
)
@@ -209,7 +206,7 @@ class TestCjkGetsNoFuzzySide:
one edit of its start, which is the "東京都 matches 京都"
failure the bigram fields exist to avoid
"""
assert _fuzzy_alternative(_content(text)) is None
assert _fuzzy_alternative(content(text)) is None
def test_latin_beside_cjk_still_gets_its_fuzzy_side(self) -> None:
"""
@@ -221,6 +218,6 @@ class TestCjkGetsNoFuzzySide:
- Only the latin word is fuzzed. Skipping CJK words must not
cost the latin half its near-match
"""
leaf = _content("東京 report")
leaf = content("東京 report")
assert _fuzzy_alternative(leaf) == _fuzzy(_CONTENT, "report")
assert _fuzzy_alternative(leaf) == _fuzzy(CONTENT, "report")
src/documents/tests/search/test_fuzzy_tokenization.py
+32 -69
View File
@@ -13,38 +13,23 @@ from typing import TYPE_CHECKING
import pytest
from documents.models import Document
if TYPE_CHECKING:
from pytest_django.fixtures import SettingsWrapper
from collections.abc import Callable
from documents.search._backend import TantivyBackend
from documents.models import Document
pytestmark = [pytest.mark.search, pytest.mark.django_db]
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
return set(backend.search_ids(query, user=None))
def _index(backend: TantivyBackend, **kwargs: object) -> Document:
doc = Document.objects.create(**kwargs)
backend.add_or_update(doc)
return doc
@pytest.fixture(autouse=True)
def fuzzy_enabled(settings: SettingsWrapper) -> None:
"""Enable the fuzzy blend clause. The threshold doubles as a minimum
score filter, so it is set to 0.0: every hit passes and the test sees
the clause's matching behaviour, not the filter's."""
settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.0
pytestmark = [
pytest.mark.search,
pytest.mark.django_db,
pytest.mark.usefixtures("fuzzy_enabled"),
]
class TestFuzzyClauseWords:
def test_a_stemmed_word_is_not_stemmed_a_second_time(
self,
backend: TantivyBackend,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
@@ -62,36 +47,29 @@ class TestFuzzyClauseWords:
reaches unrelated words - the clause must stay wide enough
for a typo and no wider
"""
wanted = _index(
backend,
wanted = index_document(
title="A",
content="universities of europe",
checksum="fuzz-stem-1",
)
typo = _index(
backend,
typo = index_document(
title="B",
content="universties of europe",
checksum="fuzz-stem-2",
)
_index(
backend,
index_document(
title="C",
content="univalent chemical bonds",
checksum="fuzz-stem-3",
)
_index(
backend,
index_document(
title="D",
content="unicycle repair manual",
checksum="fuzz-stem-4",
)
assert _matched_ids(backend, "universities") == {wanted.pk, typo.pk}
assert matched_ids("universities") == {wanted.pk, typo.pk}
def test_a_hyphenated_term_still_reaches_the_clause(
self,
backend: TantivyBackend,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
@@ -105,18 +83,17 @@ class TestFuzzyClauseWords:
would read as grammar, is dropped, and the whole query loses
its fuzzy clause
"""
misspelled = _index(
backend,
misspelled = index_document(
title="A",
content="covidx testing results",
checksum="fuzz-hyphen-1",
)
assert _matched_ids(backend, "COVID-19") == {misspelled.pk}
assert matched_ids("COVID-19") == {misspelled.pk}
def test_a_phrase_still_reaches_the_clause(
self,
backend: TantivyBackend,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
@@ -129,14 +106,12 @@ class TestFuzzyClauseWords:
space, and is the whole query's only free text here, so it
must still reach the clause
"""
near_miss = _index(
backend,
near_miss = index_document(
title="A",
content="taxation reportage weekly",
checksum="fuzz-phrase-1",
)
assert _matched_ids(backend, '"tax reports"') == {near_miss.pk}
assert matched_ids('"tax reports"') == {near_miss.pk}
class TestBooleanKeywordsInRawText:
@@ -146,24 +121,18 @@ class TestBooleanKeywordsInRawText:
now, which closes that off structurally; these pin it shut."""
@pytest.fixture
def corpus(self, backend: TantivyBackend) -> dict[str, int]:
both = _index(
backend,
def corpus(self, index_document: Callable[..., Document]) -> dict[str, int]:
both = index_document(
title="A",
content="taxation reportage weekly",
checksum="fuzz-kw-1",
)
tax_only = _index(
backend,
tax_only = index_document(
title="B",
content="taxation only here",
checksum="fuzz-kw-2",
)
report_only = _index(
backend,
report_only = index_document(
title="C",
content="reportage only here",
checksum="fuzz-kw-3",
)
return {
"both": both.pk,
@@ -182,7 +151,7 @@ class TestBooleanKeywordsInRawText:
)
def test_a_keyword_inside_a_phrase_stays_an_ordinary_word(
self,
backend: TantivyBackend,
matched_ids: Callable[[str], set[int]],
corpus: dict[str, int],
keyword_spelling: str,
ordinary_spelling: str,
@@ -201,14 +170,11 @@ class TestBooleanKeywordsInRawText:
exclusion, IN does not fail the parse. Only the upper-case
spelling was ever grammar
"""
assert _matched_ids(backend, keyword_spelling) == _matched_ids(
backend,
ordinary_spelling,
)
assert matched_ids(keyword_spelling) == matched_ids(ordinary_spelling)
def test_a_phrase_needs_a_near_match_for_every_word(
self,
backend: TantivyBackend,
matched_ids: Callable[[str], set[int]],
corpus: dict[str, int],
) -> None:
"""
@@ -222,11 +188,11 @@ class TestBooleanKeywordsInRawText:
quoted phrase asks for more than the bare words, so its
fuzzy side requires every one of them
"""
assert _matched_ids(backend, '"tax reports"') == {corpus["both"]}
assert matched_ids('"tax reports"') == {corpus["both"]}
def test_a_trailing_keyword_is_just_a_word(
self,
backend: TantivyBackend,
matched_ids: Callable[[str], set[int]],
corpus: dict[str, int],
) -> None:
"""
@@ -242,7 +208,4 @@ class TestBooleanKeywordsInRawText:
is re-parsed any more, so a trailing keyword cannot cost the
query its fuzzy side
"""
assert _matched_ids(backend, '"tax AND"') == _matched_ids(
backend,
'"tax and"',
)
assert matched_ids('"tax AND"') == matched_ids('"tax and"')
src/documents/tests/search/test_highlight_query_guard.py
-11
View File
@@ -29,8 +29,6 @@ from rest_framework import status
from documents.search._backend import SearchMode
from documents.search._query import parse_simple_text_highlight_query
from documents.search._schema import build_schema
from documents.search._tokenizer import register_tokenizers
from documents.tests.factories import DocumentFactory
if TYPE_CHECKING:
@@ -55,15 +53,6 @@ _MALFORMED_QUERIES = [
]
@pytest.fixture(scope="module")
def query_index() -> tantivy.Index:
"""An in-memory, unstemmed index for parse-only tests."""
schema = build_schema()
idx = tantivy.Index(schema, path=None)
register_tokenizers(idx, "")
return idx
class TestParseSimpleTextHighlightQueryDoesNotRaise:
"""The query builder itself must tolerate Tantivy syntax in its tokens."""
src/documents/tests/search/test_json_field_prefixes.py
+26 -52
View File
@@ -21,29 +21,24 @@ from django.contrib.auth.models import User
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import Note
from documents.tests.factories import DocumentFactory
if TYPE_CHECKING:
from collections.abc import Callable
from documents.models import Document
from documents.search._backend import TantivyBackend
pytestmark = [pytest.mark.search, pytest.mark.django_db]
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
return set(backend.search_ids(query, user=None))
def _index(backend: TantivyBackend, **kwargs: object) -> Document:
doc = Document.objects.create(**kwargs)
backend.add_or_update(doc)
return doc
class TestBareJsonFieldPrefixes:
def test_bare_notes_prefix_searches_note_text(
self,
backend: TantivyBackend,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
@@ -59,26 +54,22 @@ class TestBareJsonFieldPrefixes:
text search
"""
alice = User.objects.create_user(username="alice")
with_note = Document.objects.create(
title="Has note",
content="x",
checksum="bare-notes-with",
)
with_note = DocumentFactory(title="Has note", content="x")
Note.objects.create(document=with_note, user=alice, note="crocodile")
backend.add_or_update(with_note)
# This document's CONTENT contains the words a demoted text search
# would match; it must NOT match once the prefix addresses notes.
_index(
backend,
index_document(
title="Notes about things",
content="notes crocodile mention",
checksum="bare-notes-decoy",
)
assert _matched_ids(backend, "notes:crocodile") == {with_note.pk}
assert matched_ids("notes:crocodile") == {with_note.pk}
def test_bare_custom_fields_prefix_searches_values(
self,
backend: TantivyBackend,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
@@ -96,28 +87,23 @@ class TestBareJsonFieldPrefixes:
name="Policy Number",
data_type=CustomField.FieldDataType.STRING,
)
with_value = Document.objects.create(
title="Has field",
content="x",
checksum="bare-cf-with",
)
with_value = DocumentFactory(title="Has field", content="x")
CustomFieldInstance.objects.create(
document=with_value,
field=field,
value_text="crocodile",
)
backend.add_or_update(with_value)
_index(
backend,
index_document(
title="Custom things",
content="custom fields crocodile",
checksum="bare-cf-decoy",
)
assert _matched_ids(backend, "custom_fields:crocodile") == {with_value.pk}
assert matched_ids("custom_fields:crocodile") == {with_value.pk}
def test_subpath_spellings_are_untouched(
self,
backend: TantivyBackend,
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
@@ -131,15 +117,11 @@ class TestBareJsonFieldPrefixes:
prefix does not interfere with explicit subpath addressing
"""
bob = User.objects.create_user(username="bob")
doc = Document.objects.create(
title="Bob note",
content="x",
checksum="bare-subpath",
)
doc = DocumentFactory(title="Bob note", content="x")
Note.objects.create(document=doc, user=bob, note="remark")
backend.add_or_update(doc)
assert _matched_ids(backend, "notes.user:bob") == {doc.pk}
assert _matched_ids(backend, "notes.note:remark") == {doc.pk}
assert matched_ids("notes.user:bob") == {doc.pk}
assert matched_ids("notes.note:remark") == {doc.pk}
class TestQuotedPhraseContainingNotesColonIsNotCorrupted:
@@ -152,7 +134,8 @@ class TestQuotedPhraseContainingNotesColonIsNotCorrupted:
def test_quoted_phrase_with_notes_colon_matches_by_content(
self,
backend: TantivyBackend,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
@@ -167,20 +150,16 @@ class TestQuotedPhraseContainingNotesColonIsNotCorrupted:
that matches nothing (the bug the deleted regex rewrite
caused, since it was blind to quoting)
"""
target = _index(
backend,
target = index_document(
title="Statement",
content="payment notes: none",
checksum="quoted-phrase-notes-colon",
)
assert _matched_ids(
backend,
'content:"payment notes: none"',
) == {target.pk}
assert matched_ids('content:"payment notes: none"') == {target.pk}
def test_quoted_phrase_matches_the_same_document_unquoted(
self,
backend: TantivyBackend,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
) -> None:
"""
GIVEN:
@@ -194,13 +173,8 @@ class TestQuotedPhraseContainingNotesColonIsNotCorrupted:
quote-awareness specifically, not about the words
themselves being unsearchable
"""
target = _index(
backend,
target = index_document(
title="Statement",
content="payment notes none",
checksum="quoted-phrase-no-colon",
)
assert _matched_ids(
backend,
'content:"payment notes none"',
) == {target.pk}
assert matched_ids('content:"payment notes none"') == {target.pk}
src/documents/tests/search/test_negated_leaf_ids.py
+16 -21
View File
@@ -11,19 +11,14 @@ identity.
from __future__ import annotations
import pytest
import whoosh_compat as wc
import whoosh_compat.ast as wc_ast
from documents.search._query import _negated_leaf_ids
from documents.tests.search._ast_helpers import CONTENT
from documents.tests.search._ast_helpers import content
pytestmark = pytest.mark.search
_CONTENT = wc.FieldRef("content")
def _term(text: str) -> wc_ast.Term:
return wc_ast.Term(field=_CONTENT, text=text)
class TestCollection:
def test_a_bare_tree_has_no_negated_leaves(self) -> None:
@@ -35,7 +30,7 @@ class TestCollection:
THEN:
- The set is empty
"""
tree = wc_ast.And(children=(_term("invoice"), _term("report")))
tree = wc_ast.And(children=(content("invoice"), content("report")))
assert _negated_leaf_ids(tree) == frozenset()
@@ -48,8 +43,8 @@ class TestCollection:
THEN:
- That leaf's id is collected, and the positive one is not
"""
positive = _term("invoice")
negated = _term("secret")
positive = content("invoice")
negated = content("secret")
tree = wc_ast.And(children=(positive, wc_ast.Not(child=negated)))
ids = _negated_leaf_ids(tree)
@@ -66,8 +61,8 @@ class TestCollection:
THEN:
- Only the negative side is collected
"""
positive = _term("invoice")
negated = _term("secret")
positive = content("invoice")
negated = content("secret")
tree = wc_ast.AndNot(positive=positive, negative=negated)
ids = _negated_leaf_ids(tree)
@@ -86,9 +81,9 @@ class TestCollection:
- All of them are collected, at any depth and whatever the
leaf type
"""
a = _term("alpha")
b = _term("beta")
phrase = wc_ast.Phrase(field=_CONTENT, text="gamma delta")
a = content("alpha")
b = content("beta")
phrase = wc_ast.Phrase(field=CONTENT, text="gamma delta")
tree = wc_ast.Not(
child=wc_ast.Or(
children=(a, wc_ast.And(children=(b, phrase))),
@@ -109,7 +104,7 @@ class TestCollection:
under-collecting would make a negation exclude far more than
the user asked
"""
leaf = _term("tax")
leaf = content("tax")
tree = wc_ast.Not(child=wc_ast.Not(child=leaf))
assert _negated_leaf_ids(tree) == frozenset({id(leaf)})
@@ -144,7 +139,7 @@ class TestCollection:
- Nothing is collected. Only Not.child and AndNot.negative are
negative positions
"""
leaf = _term("invoice")
leaf = content("invoice")
assert _negated_leaf_ids(build(leaf)) == frozenset()
@@ -157,7 +152,7 @@ class TestCollection:
THEN:
- The set is empty and nothing raises
"""
assert _negated_leaf_ids(_term("invoice")) == frozenset()
assert _negated_leaf_ids(content("invoice")) == frozenset()
class TestTotality:
@@ -167,11 +162,11 @@ class TestTotality:
pytest.param(wc_ast.Every(), id="every"),
pytest.param(wc_ast.Nothing(), id="nothing"),
pytest.param(
wc_ast.Wildcard(field=_CONTENT, pattern="inv*"),
wc_ast.Wildcard(field=CONTENT, pattern="inv*"),
id="wildcard",
),
pytest.param(
wc_ast.Fuzzy(field=_CONTENT, text="invoce", distance=1, prefix=True),
wc_ast.Fuzzy(field=CONTENT, text="invoce", distance=1, prefix=True),
id="fuzzy",
),
],
@@ -200,7 +195,7 @@ class TestTotality:
- It completes. The walk is iterative, so depth costs heap
rather than Python stack frames
"""
leaf = _term("invoice")
leaf = content("invoice")
node: wc_ast.Node = leaf
for _ in range(5000):
node = wc_ast.Not(child=node)
src/documents/tests/search/test_pattern_stemming.py
+24 -32
View File
@@ -18,10 +18,10 @@ from typing import TYPE_CHECKING
import pytest
from documents.models import Document
if TYPE_CHECKING:
from documents.search._backend import TantivyBackend
from collections.abc import Callable
from documents.models import Document
pytestmark = [pytest.mark.search, pytest.mark.django_db]
@@ -31,20 +31,13 @@ CONTENT = (
)
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
return set(backend.search_ids(query, user=None))
@pytest.fixture
def indexed_doc(backend: TantivyBackend) -> Document:
doc = Document.objects.create(
def indexed_doc(index_document: Callable[..., Document]) -> Document:
return index_document(
title="Invoice 2020 productname",
content=CONTENT,
checksum="pattern-stemming-1",
archive_serial_number=900,
)
backend.add_or_update(doc)
return doc
class TestPrefixStemming:
@@ -61,7 +54,7 @@ class TestPrefixStemming:
)
def test_full_word_prefix_matches_its_stem(
self,
backend: TantivyBackend,
matched_ids: Callable[[str], set[int]],
indexed_doc: Document,
query: str,
) -> None:
@@ -78,12 +71,12 @@ class TestPrefixStemming:
the word's stem as an alternative alongside the typed run,
reaching the stemmed index term
"""
assert _matched_ids(backend, query) == {indexed_doc.id}
assert matched_ids(query) == {indexed_doc.id}
@pytest.mark.parametrize("query", ["invoic*", "electr*", "payment*"])
def test_already_stemmed_prefix_still_matches(
self,
backend: TantivyBackend,
matched_ids: Callable[[str], set[int]],
indexed_doc: Document,
query: str,
) -> None:
@@ -97,12 +90,12 @@ class TestPrefixStemming:
- The document still matches, since the typed-run alternative
is itself a prefix of the stored stemmed term
"""
assert _matched_ids(backend, query) == {indexed_doc.id}
assert matched_ids(query) == {indexed_doc.id}
@pytest.mark.parametrize("query", ["univers*", "librar*"])
def test_partial_prefix_reaches_the_stemmed_term(
self,
backend: TantivyBackend,
matched_ids: Callable[[str], set[int]],
indexed_doc: Document,
query: str,
) -> None:
@@ -123,11 +116,11 @@ class TestPrefixStemming:
genuinely diverge, and only one of them matches, is
test_stem_substitution_reaches_both_the_inflection_and_the_compound
"""
assert _matched_ids(backend, query) == {indexed_doc.id}
assert matched_ids(query) == {indexed_doc.id}
def test_full_word_reaches_the_stem_but_a_fragment_of_it_does_not(
self,
backend: TantivyBackend,
matched_ids: Callable[[str], set[int]],
indexed_doc: Document,
) -> None:
"""
@@ -144,13 +137,13 @@ class TestPrefixStemming:
usage.md tells a reader whose `universit*` finds nothing to
shorten it to `univers*`, which matches
"""
assert _matched_ids(backend, "universities*") == {indexed_doc.id}
assert _matched_ids(backend, "universit*") == set()
assert _matched_ids(backend, "univers*") == {indexed_doc.id}
assert matched_ids("universities*") == {indexed_doc.id}
assert matched_ids("universit*") == set()
assert matched_ids("univers*") == {indexed_doc.id}
def test_pattern_past_the_stem_boundary_is_documented_not_fixed(
self,
backend: TantivyBackend,
matched_ids: Callable[[str], set[int]],
indexed_doc: Document,
) -> None:
"""
@@ -165,11 +158,12 @@ class TestPrefixStemming:
index, and usage.md must not advertise it. Pinned so the
limitation is deliberate, not accidental
"""
assert _matched_ids(backend, "produ*name") == set()
assert matched_ids("produ*name") == set()
def test_stem_substitution_reaches_both_the_inflection_and_the_compound(
self,
backend: TantivyBackend,
index_document: Callable[..., Document],
matched_ids: Callable[[str], set[int]],
indexed_doc: Document,
) -> None:
"""
@@ -189,22 +183,20 @@ class TestPrefixStemming:
folded and stemmed forms, and "copy*" reaches the base
word, its inflections and the compound alike
"""
compound = Document.objects.create(
compound = index_document(
title="Copyright notice",
content="copyright notice for the work",
checksum="pattern-stemming-2",
archive_serial_number=901,
)
backend.add_or_update(compound)
assert _matched_ids(backend, "copy*") == {indexed_doc.id, compound.id}
assert _matched_ids(backend, "copyright*") == {compound.id}
assert matched_ids("copy*") == {indexed_doc.id, compound.id}
assert matched_ids("copyright*") == {compound.id}
class TestBracketClassStillFolds:
def test_class_body_matches_case_insensitively(
self,
backend: TantivyBackend,
matched_ids: Callable[[str], set[int]],
indexed_doc: Document,
) -> None:
"""
@@ -218,4 +210,4 @@ class TestBracketClassStillFolds:
the alternatives contract preserves only because a lone
character stems to itself
"""
assert _matched_ids(backend, "title:[IP]nvoice*") == {indexed_doc.id}
assert matched_ids("title:[IP]nvoice*") == {indexed_doc.id}
src/documents/tests/search/test_query.py
-10
View File
@@ -24,16 +24,6 @@ if TYPE_CHECKING:
pytestmark = pytest.mark.search
@pytest.fixture(scope="module")
def query_index() -> tantivy.Index:
"""An in-memory, unstemmed index shared read-only across this module's
parse-only tests (none of them index documents)."""
schema = build_schema()
idx = tantivy.Index(schema, path=None)
register_tokenizers(idx, "")
return idx
@pytest.fixture(scope="module")
def populated_index() -> tantivy.Index:
"""An index holding one document, so a query matching nothing is
src/documents/tests/search/test_query_negation.py
-10
View File
@@ -16,22 +16,12 @@ import pytest
if TYPE_CHECKING:
from collections.abc import Callable
from pytest_django.fixtures import SettingsWrapper
from documents.models import Document
pytestmark = [pytest.mark.search, pytest.mark.django_db]
@pytest.fixture
def fuzzy_enabled(settings: SettingsWrapper) -> None:
"""Enable the fuzzy blend clause. The threshold doubles as a minimum
score filter, so it is set to 0.0: every hit passes and the test sees
the clause's matching behaviour, not the filter's."""
settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.0
class TestNegationConstrainsEveryClause:
@pytest.mark.usefixtures("fuzzy_enabled")
def test_fuzzy_clause_does_not_readmit_an_excluded_document(
src/documents/tests/search/test_widen_leaf.py
+8 -14
View File
@@ -7,22 +7,16 @@ negated leaf keeps its CJK side while losing its fuzzy one.
from __future__ import annotations
import pytest
import whoosh_compat as wc
import whoosh_compat.ast as wc_ast
from documents.search._query import _cjk_alternative
from documents.search._query import _fuzzy_alternative
from documents.search._query import _widen_leaf
from documents.tests.search._ast_helpers import NOTES
from documents.tests.search._ast_helpers import content
pytestmark = pytest.mark.search
_CONTENT = wc.FieldRef("content")
_NOTES = wc.FieldRef("notes", "note")
def _content(text: str) -> wc_ast.Term:
return wc_ast.Term(field=_CONTENT, text=text)
def _widen(
leaf: wc_ast.Term,
@@ -44,7 +38,7 @@ class TestWhichAlternativesAreAdded:
- The Or holds the leaf and the boosted fuzzy alternative,
and nothing else
"""
leaf = _content("invoice")
leaf = content("invoice")
assert _widen(leaf) == wc_ast.Or(
children=(
@@ -63,7 +57,7 @@ class TestWhichAlternativesAreAdded:
- Only the bigram alternative is added, which is exactly what
the CJK work shipped
"""
leaf = _content("東京")
leaf = content("東京")
assert _widen(leaf, fuzzy=False) == wc_ast.Or(
children=(leaf, _cjk_alternative(leaf)),
@@ -81,7 +75,7 @@ class TestWhichAlternativesAreAdded:
any run within one edit of its start, and _cjk_alternative
already supplies the in-run recall
"""
leaf = _content("東京")
leaf = content("東京")
assert _fuzzy_alternative(leaf) is None
assert _widen(leaf) == wc_ast.Or(children=(leaf, _cjk_alternative(leaf)))
@@ -97,7 +91,7 @@ class TestWhichAlternativesAreAdded:
NOT X should exclude what X matches, which needs the bigram
side, but prefix fuzzy matching would exclude far more
"""
leaf = _content("東京")
leaf = content("東京")
assert _widen(leaf, negated=frozenset({id(leaf)})) == wc_ast.Or(
children=(leaf, _cjk_alternative(leaf)),
@@ -113,7 +107,7 @@ class TestWhichAlternativesAreAdded:
- The leaf itself comes back. It qualifies for no alternative
at all, so there is no Or to build
"""
leaf = _content("tax")
leaf = content("tax")
assert _widen(leaf, negated=frozenset({id(leaf)})) is leaf
@@ -128,6 +122,6 @@ class TestWhichAlternativesAreAdded:
- The leaf comes back untouched. Widening is scoped to the
default search fields, as it was for CJK
"""
leaf = wc_ast.Term(field=_NOTES, text="invoice")
leaf = wc_ast.Term(field=NOTES, text="invoice")
assert _widen(leaf) is leaf
src/documents/tests/test_api_search_errors.py
-10
View File
@@ -19,7 +19,6 @@ from whoosh_compat.errors import DiagnosticKind
from whoosh_compat.errors import QueryError
from documents.search import SearchQueryError
from documents.tests.factories import DocumentFactory
if TYPE_CHECKING:
from rest_framework.test import APIClient
@@ -29,15 +28,6 @@ if TYPE_CHECKING:
pytestmark = [pytest.mark.django_db, pytest.mark.usefixtures("_search_index")]
@pytest.fixture
def indexed_document() -> Document:
from documents.search import get_backend
doc = DocumentFactory.create(title="quarterly invoice", content="acme corp")
get_backend().add_or_update(doc)
return doc
class TestSearchQueryErrorStillBecomesA400:
def test_search_query_error_becomes_a_400_naming_the_field(
self,
src/documents/tests/test_api_search_query_length.py
-10
View File
@@ -22,7 +22,6 @@ import pytest
from rest_framework import status
import documents.search._backend
from documents.tests.factories import DocumentFactory
from documents.views import _MAX_QUERY_LENGTH
if TYPE_CHECKING:
@@ -33,15 +32,6 @@ if TYPE_CHECKING:
pytestmark = [pytest.mark.django_db, pytest.mark.usefixtures("_search_index")]
@pytest.fixture
def indexed_document() -> Document:
from documents.search import get_backend
doc = DocumentFactory.create(title="quarterly invoice", content="acme corp")
get_backend().add_or_update(doc)
return doc
class TestGetSearchEndpointEnforcesTheCap:
def test_query_one_over_the_cap_is_a_400(
self,
src/documents/tests/test_api_search_unterminated_date_range.py
-11
View File
@@ -19,8 +19,6 @@ from typing import TYPE_CHECKING
import pytest
from rest_framework import status
from documents.tests.factories import DocumentFactory
if TYPE_CHECKING:
from rest_framework.test import APIClient
@@ -29,15 +27,6 @@ if TYPE_CHECKING:
pytestmark = [pytest.mark.django_db, pytest.mark.usefixtures("_search_index")]
@pytest.fixture
def indexed_document() -> Document:
from documents.search import get_backend
doc = DocumentFactory.create(title="quarterly invoice", content="acme corp")
get_backend().add_or_update(doc)
return doc
class TestUnterminatedBracketReturnsA400:
@pytest.mark.parametrize(
"query",