diff --git a/src/documents/tests/conftest.py b/src/documents/tests/conftest.py index 99f53d614..626760d31 100644 --- a/src/documents/tests/conftest.py +++ b/src/documents/tests/conftest.py @@ -182,3 +182,15 @@ def faker_session_locale(): @pytest.fixture(scope="session", autouse=True) def faker_seed(): return 12345 + + +@pytest.fixture +def indexed_document(_search_index: None) -> "Document": + """One searchable document, for tests about what the search endpoint + returns rather than about what it finds. + """ + from documents.search import get_backend + + doc = DocumentFactory.create(title="quarterly invoice", content="acme corp") + get_backend().add_or_update(doc) + return doc diff --git a/src/documents/tests/search/_ast_helpers.py b/src/documents/tests/search/_ast_helpers.py new file mode 100644 index 000000000..aa38c25af --- /dev/null +++ b/src/documents/tests/search/_ast_helpers.py @@ -0,0 +1,30 @@ +"""Field references and leaf constructors shared by the unit-level tests. + +These are pure whoosh-compat AST values with no state and no database +behind them, so they stay plain module-level functions rather than +fixtures. ``NOTES`` is the reason this module exists at all: notes is a +JSON field, so addressing its text means naming a subpath, and spelling +``wc.FieldRef("notes", "note")`` out once per test file invites two of +them to disagree. +""" + +from __future__ import annotations + +import whoosh_compat as wc +import whoosh_compat.ast as wc_ast + +CONTENT = wc.FieldRef("content") +TITLE = wc.FieldRef("title") +NOTES = wc.FieldRef("notes", "note") +BIGRAM_CONTENT = wc.FieldRef("bigram_content") +BIGRAM_TITLE = wc.FieldRef("bigram_title") + + +def content(text: str) -> wc_ast.Term: + """A Term on the content field, the default leaf these tests widen.""" + return wc_ast.Term(field=CONTENT, text=text) + + +def bigram(text: str) -> wc_ast.Term: + """A Term on the bigram side of content.""" + return wc_ast.Term(field=BIGRAM_CONTENT, text=text) diff --git a/src/documents/tests/search/conftest.py b/src/documents/tests/search/conftest.py index 7bb065cb0..64dfcfe0c 100644 --- a/src/documents/tests/search/conftest.py +++ b/src/documents/tests/search/conftest.py @@ -3,9 +3,12 @@ from __future__ import annotations from typing import TYPE_CHECKING import pytest +import tantivy from documents.search._backend import TantivyBackend from documents.search._backend import reset_backend +from documents.search._schema import build_schema +from documents.search._tokenizer import register_tokenizers from documents.tests.factories import DocumentFactory if TYPE_CHECKING: @@ -72,3 +75,23 @@ def matched_ids(backend: TantivyBackend) -> Callable[[str], set[int]]: return set(backend.search_ids(query, user=None)) return _matched_ids + + +@pytest.fixture(scope="module") +def query_index() -> tantivy.Index: + """An in-memory, unstemmed index for the parse-only tests. + + These never index a document, so one index per module is shared + read-only across that module's tests. + """ + idx = tantivy.Index(build_schema(), path=None) + register_tokenizers(idx, "") + return idx + + +@pytest.fixture +def fuzzy_enabled(settings: Settings) -> None: + """Enable the fuzzy blend clause. The threshold doubles as a minimum + score filter, so it is set to 0.0: every hit passes and the test sees + the clause's matching behaviour, not the filter's.""" + settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.0 diff --git a/src/documents/tests/search/test_acceptance.py b/src/documents/tests/search/test_acceptance.py index 10fd41fc8..097fc28de 100644 --- a/src/documents/tests/search/test_acceptance.py +++ b/src/documents/tests/search/test_acceptance.py @@ -18,13 +18,16 @@ from django.contrib.auth.models import User from documents.models import CustomField from documents.models import CustomFieldInstance -from documents.models import Document from documents.models import DocumentType from documents.models import Note from documents.models import StoragePath from documents.search._query import parse_user_query +from documents.tests.factories import DocumentFactory if TYPE_CHECKING: + from collections.abc import Callable + + from documents.models import Document from documents.search._backend import TantivyBackend pytestmark = [pytest.mark.search, pytest.mark.django_db] @@ -32,49 +35,28 @@ pytestmark = [pytest.mark.search, pytest.mark.django_db] FROZEN_NOW = datetime(2026, 6, 15, 12, 0, tzinfo=UTC) -def _matched_ids(backend: TantivyBackend, query: str) -> set[int]: - return set(backend.search_ids(query, user=None)) - - -def _index(backend: TantivyBackend, **kwargs: object) -> Document: - """Create a Document and index it in one step, for the common case - where nothing needs to happen between the two (no related Note/ - CustomFieldInstance to attach first).""" - doc = Document.objects.create(**kwargs) - backend.add_or_update(doc) - return doc - - @pytest.fixture -def indexed_documents(backend: TantivyBackend) -> dict[str, int]: +def indexed_documents(index_document: Callable[..., Document]) -> dict[str, int]: """Index a small fixture set, return {label: doc_id} for corpus queries.""" docs = { - "invoice_2020": _index( - backend, + "invoice_2020": index_document( title="Invoice 2020", content="invoice total due", - checksum="acc-invoice-2020", archive_serial_number=100, ), - "invoice_2021": _index( - backend, + "invoice_2021": index_document( title="Invoice 2021", content="invoice total due", - checksum="acc-invoice-2021", archive_serial_number=101, ), - "invoice_2023": _index( - backend, + "invoice_2023": index_document( title="Invoice 2023", content="invoice total due", - checksum="acc-invoice-2023", archive_serial_number=102, ), - "receipt_2022": _index( - backend, + "receipt_2022": index_document( title="Receipt 2022", content="receipt total due", - checksum="acc-receipt-2022", archive_serial_number=103, ), } @@ -87,7 +69,7 @@ class TestIssue13568BracketWildcard: def test_bracket_class_wildcard_matches_only_in_range_years( self, - backend: TantivyBackend, + matched_ids: Callable[[str], set[int]], indexed_documents: dict[str, int], ) -> None: """ @@ -105,7 +87,7 @@ class TestIssue13568BracketWildcard: - Only the 2020 and 2021 documents match, proving the bracket character class survived (issue #13568's original bug) """ - matched = _matched_ids(backend, "title:202[0-1]*") + matched = matched_ids("title:202[0-1]*") expected = { indexed_documents["invoice_2020"], indexed_documents["invoice_2021"], @@ -121,6 +103,7 @@ class TestFieldBoosts: def test_title_boost_ranks_title_match_above_content_only_match( self, backend: TantivyBackend, + index_document: Callable[..., Document], ) -> None: """ GIVEN: @@ -132,17 +115,13 @@ class TestFieldBoosts: - The title match ranks first, proving our title field boost actually affects ranking """ - title_match = _index( - backend, + title_match = index_document( title="urgent", content="nothing else relevant", - checksum="acc-boost-title", ) - _index( - backend, + index_document( title="nothing", content="urgent matter here", - checksum="acc-boost-content", ) query = parse_user_query(backend._index, "urgent", UTC) searcher = backend._index.searcher() @@ -157,6 +136,8 @@ class TestJsonSubpaths: def test_notes_user_matches_document_with_that_note_author( self, backend: TantivyBackend, + index_document: Callable[..., Document], + matched_ids: Callable[[str], set[int]], ) -> None: """ GIVEN: @@ -168,20 +149,20 @@ class TestJsonSubpaths: - Only the document with alice's note matches """ alice = User.objects.create_user(username="alice") - doc_with_note = Document.objects.create( + doc_with_note = DocumentFactory( title="Has note", content="x", - checksum="acc-note-with", ) Note.objects.create(document=doc_with_note, user=alice, note="reminder") backend.add_or_update(doc_with_note) - _index(backend, title="No note", content="x", checksum="acc-note-without") - matched = _matched_ids(backend, "notes.user:alice") + index_document(title="No note", content="x") + matched = matched_ids("notes.user:alice") assert matched == {doc_with_note.pk} def test_custom_fields_name_and_value_combine( self, backend: TantivyBackend, + matched_ids: Callable[[str], set[int]], ) -> None: """ GIVEN: @@ -203,10 +184,9 @@ class TestJsonSubpaths: name="Other Field", data_type=CustomField.FieldDataType.STRING, ) - matching = Document.objects.create( + matching = DocumentFactory( title="Matching", content="x", - checksum="acc-cf-matching", ) CustomFieldInstance.objects.create( document=matching, @@ -214,10 +194,9 @@ class TestJsonSubpaths: value_text="policy", ) backend.add_or_update(matching) - non_matching = Document.objects.create( + non_matching = DocumentFactory( title="Non-matching", content="x", - checksum="acc-cf-nonmatching", ) CustomFieldInstance.objects.create( document=non_matching, @@ -225,8 +204,7 @@ class TestJsonSubpaths: value_text="policy", ) backend.add_or_update(non_matching) - matched = _matched_ids( - backend, + matched = matched_ids( 'custom_fields.name:"Contract Number" custom_fields.value:policy', ) assert matched == {matching.pk} @@ -240,7 +218,7 @@ class TestUnregisteredIdFieldFoldsToLiteralText: def test_tag_id_query_matches_nothing( self, - backend: TantivyBackend, + matched_ids: Callable[[str], set[int]], indexed_documents: dict[str, int], ) -> None: """ @@ -254,7 +232,7 @@ class TestUnregisteredIdFieldFoldsToLiteralText: - It folds to a literal text search and matches nothing, rather than erroring """ - matched = _matched_ids(backend, "tag_id:5") + matched = matched_ids("tag_id:5") assert matched == set() @@ -268,7 +246,8 @@ class TestFuzzyBlendSurvivesWhooshGrammar: def test_typo_fuzzy_matches_alongside_date_keyword( self, - backend: TantivyBackend, + index_document: Callable[..., Document], + matched_ids: Callable[[str], set[int]], settings, ) -> None: """ @@ -287,26 +266,25 @@ class TestFuzzyBlendSurvivesWhooshGrammar: """ settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.5 with time_machine.travel(FROZEN_NOW, tick=False): - doc = _index( - backend, + doc = index_document( title="Receipt March", content="receipt total due", - checksum="fuzzy-blend-1", archive_serial_number=900, ) # Sanity: the exact spelling matches through the exact clause. - assert doc.pk in _matched_ids(backend, "added:today receipt") + assert doc.pk in matched_ids("added:today receipt") # The regression: the misspelling (one transposition) only # matches via the fuzzy clause, and "added:today" is # whoosh-only grammar tantivy's parser rejects, so raw-string # fuzzy parsing skips the clause entirely and this returns # nothing. The typo is deliberate; keep codespell away from it. typo_query = "added:today reciept" # codespell:ignore reciept - assert doc.pk in _matched_ids(backend, typo_query) + assert doc.pk in matched_ids(typo_query) def test_negated_words_do_not_fuzzy_match( self, - backend: TantivyBackend, + index_document: Callable[..., Document], + matched_ids: Callable[[str], set[int]], settings, ) -> None: """ @@ -329,14 +307,12 @@ class TestFuzzyBlendSurvivesWhooshGrammar: """ settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.5 with time_machine.travel(FROZEN_NOW, tick=False): - _index( - backend, + index_document( title="Receipt Archive", content="receipt archived stack", - checksum="fuzzy-blend-2", archive_serial_number=901, ) - assert _matched_ids(backend, "added:today total NOT receipt") == set() + assert matched_ids("added:today total NOT receipt") == set() class TestUnquotedDateKeywordPhrases: @@ -346,21 +322,20 @@ class TestUnquotedDateKeywordPhrases: spelling keeps working now that paperless no longer pre-quotes it.""" @pytest.fixture - def period_documents(self, backend: TantivyBackend) -> dict[str, int]: + def period_documents( + self, + index_document: Callable[..., Document], + ) -> dict[str, int]: with time_machine.travel(FROZEN_NOW, tick=False): - in_may = _index( - backend, + in_may = index_document( title="May Doc", content="statement", - checksum="kw-may", archive_serial_number=910, added=datetime(2026, 5, 20, 12, 0, tzinfo=UTC), ) - in_june = _index( - backend, + in_june = index_document( title="June Doc", content="statement", - checksum="kw-june", archive_serial_number=911, added=datetime(2026, 6, 10, 12, 0, tzinfo=UTC), ) @@ -376,7 +351,7 @@ class TestUnquotedDateKeywordPhrases: ) def test_unquoted_matches_the_same_documents_as_quoted( self, - backend: TantivyBackend, + matched_ids: Callable[[str], set[int]], period_documents: dict[str, int], query: str, ) -> None: @@ -393,7 +368,7 @@ class TestUnquotedDateKeywordPhrases: whoosh-compat's own grammar to accept it unquoted natively """ with time_machine.travel(FROZEN_NOW, tick=False): - assert _matched_ids(backend, query) == {period_documents["in_may"]} + assert matched_ids(query) == {period_documents["in_may"]} @pytest.mark.parametrize( "query", @@ -409,7 +384,7 @@ class TestUnquotedDateKeywordPhrases: ) def test_every_phrase_and_date_field_parses_without_error( self, - backend: TantivyBackend, + matched_ids: Callable[[str], set[int]], period_documents: dict[str, int], query: str, ) -> None: @@ -426,11 +401,12 @@ class TestUnquotedDateKeywordPhrases: are whoosh-compat's own and are pinned in its own suite """ with time_machine.travel(FROZEN_NOW, tick=False): - _matched_ids(backend, query) + matched_ids(query) def test_text_field_keyword_words_are_ordinary_text( self, - backend: TantivyBackend, + index_document: Callable[..., Document], + matched_ids: Callable[[str], set[int]], period_documents: dict[str, int], ) -> None: """ @@ -447,14 +423,12 @@ class TestUnquotedDateKeywordPhrases: documents do not match """ with time_machine.travel(FROZEN_NOW, tick=False): - wordy = _index( - backend, + wordy = index_document( title="Notes from the previous month", content="meeting notes", - checksum="kw-text", archive_serial_number=912, ) - assert _matched_ids(backend, "title:previous month") == {wordy.pk} + assert matched_ids("title:previous month") == {wordy.pk} class TestFieldAliases: @@ -464,7 +438,8 @@ class TestFieldAliases: def test_type_alias_and_canonical_name_match_the_same_document( self, - backend: TantivyBackend, + index_document: Callable[..., Document], + matched_ids: Callable[[str], set[int]], ) -> None: """ GIVEN: @@ -487,25 +462,22 @@ class TestFieldAliases: real index """ invoice_type = DocumentType.objects.create(name="invoice") - typed = _index( - backend, + typed = index_document( title="First", content="quarterly statement", - checksum="alias-type-1", document_type=invoice_type, ) - _index( - backend, + index_document( title="Second", content="invoice mentioned in body", - checksum="alias-type-2", ) - assert _matched_ids(backend, "type:invoice") == {typed.pk} - assert _matched_ids(backend, "document_type:invoice") == {typed.pk} + assert matched_ids("type:invoice") == {typed.pk} + assert matched_ids("document_type:invoice") == {typed.pk} def test_path_alias_and_canonical_name_match_the_same_document( self, - backend: TantivyBackend, + index_document: Callable[..., Document], + matched_ids: Callable[[str], set[int]], ) -> None: """ GIVEN: @@ -523,18 +495,14 @@ class TestFieldAliases: real index """ archive = StoragePath.objects.create(name="archive", path="archive/{title}") - stored = _index( - backend, + stored = index_document( title="Stored", content="quarterly statement", - checksum="alias-path-1", storage_path=archive, ) - _index( - backend, + index_document( title="Loose", content="archive mentioned in body", - checksum="alias-path-2", ) - assert _matched_ids(backend, "path:archive") == {stored.pk} - assert _matched_ids(backend, "storage_path:archive") == {stored.pk} + assert matched_ids("path:archive") == {stored.pk} + assert matched_ids("storage_path:archive") == {stored.pk} diff --git a/src/documents/tests/search/test_cjk_emit_registry.py b/src/documents/tests/search/test_cjk_emit_registry.py index 78db2d11e..da35bf1a7 100644 --- a/src/documents/tests/search/test_cjk_emit_registry.py +++ b/src/documents/tests/search/test_cjk_emit_registry.py @@ -10,19 +10,17 @@ from __future__ import annotations import pytest import tantivy -import whoosh_compat as wc import whoosh_compat.ast as wc_ast from documents.search._query import _CJK_BIGRAM_FIELDS from documents.search._query import _get_emit_field_registry from documents.search._schema import field_descriptors from documents.search._tokenizer import register_tokenizers +from documents.tests.search._ast_helpers import BIGRAM_CONTENT +from documents.tests.search._ast_helpers import CONTENT pytestmark = pytest.mark.search -_BIGRAM_CONTENT = wc.FieldRef("bigram_content") -_CONTENT = wc.FieldRef("content") - class TestEmitFieldRegistry: def test_a_multi_bigram_run_requires_every_bigram_even_under_an_or( @@ -41,8 +39,8 @@ class TestEmitFieldRegistry: """ tree = wc_ast.Or( children=( - wc_ast.Term(field=_BIGRAM_CONTENT, text="東京都"), - wc_ast.Term(field=_CONTENT, text="report"), + wc_ast.Term(field=BIGRAM_CONTENT, text="東京都"), + wc_ast.Term(field=CONTENT, text="report"), ), ) @@ -52,11 +50,11 @@ class TestEmitFieldRegistry: children=( wc_ast.And( children=( - wc_ast.Term(field=_BIGRAM_CONTENT, text="東京"), - wc_ast.Term(field=_BIGRAM_CONTENT, text="京都"), + wc_ast.Term(field=BIGRAM_CONTENT, text="東京"), + wc_ast.Term(field=BIGRAM_CONTENT, text="京都"), ), ), - wc_ast.Term(field=_CONTENT, text="report"), + wc_ast.Term(field=CONTENT, text="report"), ), ) @@ -95,7 +93,7 @@ class TestEmitFieldRegistry: writer.commit() index.reload() - resolved = _get_emit_field_registry(None).resolve(_BIGRAM_CONTENT) + resolved = _get_emit_field_registry(None).resolve(BIGRAM_CONTENT) assert resolved is not None tokens = resolved.spec.analyzer(text) diff --git a/src/documents/tests/search/test_cjk_leaf_rewrite.py b/src/documents/tests/search/test_cjk_leaf_rewrite.py index 815ec5d22..c98a774ac 100644 --- a/src/documents/tests/search/test_cjk_leaf_rewrite.py +++ b/src/documents/tests/search/test_cjk_leaf_rewrite.py @@ -19,26 +19,19 @@ from documents.search._query import _FIELD_BOOSTS from documents.search._query import _get_emit_field_registry from documents.search._query import _widen_leaf from documents.search._registry import get_field_registry +from documents.tests.search._ast_helpers import BIGRAM_CONTENT +from documents.tests.search._ast_helpers import BIGRAM_TITLE +from documents.tests.search._ast_helpers import CONTENT +from documents.tests.search._ast_helpers import NOTES +from documents.tests.search._ast_helpers import TITLE +from documents.tests.search._ast_helpers import bigram +from documents.tests.search._ast_helpers import content if TYPE_CHECKING: from collections.abc import Callable pytestmark = pytest.mark.search -_CONTENT = wc.FieldRef("content") -_TITLE = wc.FieldRef("title") -_NOTES = wc.FieldRef("notes", "note") -_BIGRAM_CONTENT = wc.FieldRef("bigram_content") -_BIGRAM_TITLE = wc.FieldRef("bigram_title") - - -def _content(text: str) -> wc_ast.Term: - return wc_ast.Term(field=_CONTENT, text=text) - - -def _bigram(text: str) -> wc_ast.Term: - return wc_ast.Term(field=_BIGRAM_CONTENT, text=text) - def _widened(original: wc_ast.Node, bigram_side: wc_ast.Node) -> wc_ast.Or: return wc_ast.Or(children=(original, bigram_side)) @@ -61,9 +54,9 @@ class TestTheHook: @pytest.mark.parametrize( "leaf", [ - pytest.param(_content("invoice"), id="latin_term"), + pytest.param(content("invoice"), id="latin_term"), pytest.param( - wc_ast.Term(field=_NOTES, text="東京"), + wc_ast.Term(field=NOTES, text="東京"), id="non_default_field", ), pytest.param(wc_ast.Term(field=None, text="東京"), id="unfielded"), @@ -96,7 +89,7 @@ class TestTheHook: - Neither cjk_terms nor latin_terms gains a piece, and the leaf is returned unchanged rather than becoming an empty Or """ - leaf = _content("⺀") + leaf = content("⺀") assert _widen_cjk_leaf(leaf) is leaf @@ -111,7 +104,7 @@ class TestTheHook: copy: analyze() only keeps the leaf's own enclosing-group analysis for that exact object """ - leaf = _content("東京") + leaf = content("東京") widened = _widen_cjk_leaf(leaf) @@ -128,11 +121,11 @@ class TestTheHook: - The bigram side targets bigram_title, the leaf's own field's companion, so a fielded term stays fielded """ - leaf = wc_ast.Term(field=_TITLE, text="東京") + leaf = wc_ast.Term(field=TITLE, text="東京") assert _widen_cjk_leaf(leaf) == _widened( leaf, - wc_ast.Term(field=_BIGRAM_TITLE, text="東京"), + wc_ast.Term(field=BIGRAM_TITLE, text="東京"), ) def test_each_cjk_run_gets_its_own_bigram_leaf(self) -> None: @@ -151,11 +144,11 @@ class TestTheHook: any one of its runs, which is what the separate bigram clause did before this change """ - leaf = _content("東京・大阪") + leaf = content("東京・大阪") assert _widen_cjk_leaf(leaf) == _widened( leaf, - wc_ast.Or(children=(_bigram("東京"), _bigram("大阪"))), + wc_ast.Or(children=(bigram("東京"), bigram("大阪"))), ) def test_each_run_keeps_its_own_bigrams_required(self) -> None: @@ -172,11 +165,11 @@ class TestTheHook: it does not inherit from the enclosing group. Without this, 東京都 would match a document holding only 京都 """ - assert _analyze(_content("東京都・大阪府")) == wc_ast.Or( + assert _analyze(content("東京都・大阪府")) == wc_ast.Or( children=( - wc_ast.And(children=(_content("東京都"), _content("大阪府"))), - wc_ast.And(children=(_bigram("東京"), _bigram("京都"))), - wc_ast.And(children=(_bigram("大阪"), _bigram("阪府"))), + wc_ast.And(children=(content("東京都"), content("大阪府"))), + wc_ast.And(children=(bigram("東京"), bigram("京都"))), + wc_ast.And(children=(bigram("大阪"), bigram("阪府"))), ), ) @@ -191,9 +184,9 @@ class TestTheHook: every token at position 0, so a positional Phrase against it could never match """ - leaf = wc_ast.Phrase(field=_CONTENT, text="東京都") + leaf = wc_ast.Phrase(field=CONTENT, text="東京都") - assert _widen_cjk_leaf(leaf) == _widened(leaf, _bigram("東京都")) + assert _widen_cjk_leaf(leaf) == _widened(leaf, bigram("東京都")) def test_a_multi_word_phrase_requires_every_run(self) -> None: """ @@ -208,27 +201,27 @@ class TestTheHook: the quoted phrase match strictly more than the same two words unquoted, and quoting must not widen a search """ - leaf = wc_ast.Phrase(field=_CONTENT, text="東京都 大阪府") + leaf = wc_ast.Phrase(field=CONTENT, text="東京都 大阪府") assert _widen_cjk_leaf(leaf) == _widened( leaf, - wc_ast.And(children=(_bigram("東京都"), _bigram("大阪府"))), + wc_ast.And(children=(bigram("東京都"), bigram("大阪府"))), ) @pytest.mark.parametrize( ("leaf", "alternative"), [ pytest.param( - _content("東京-report"), - wc_ast.And(children=(_bigram("東京"), _content("report"))), + content("東京-report"), + wc_ast.And(children=(bigram("東京"), content("report"))), id="separated_term", ), pytest.param( - wc_ast.Phrase(field=_CONTENT, text="東京 report"), - wc_ast.And(children=(_bigram("東京"), _content("report"))), + wc_ast.Phrase(field=CONTENT, text="東京 report"), + wc_ast.And(children=(bigram("東京"), content("report"))), id="phrase", ), - pytest.param(_content("東京report"), _bigram("東京"), id="glued"), + pytest.param(content("東京report"), bigram("東京"), id="glued"), ], ) def test_latin_is_required_only_where_the_analyzer_splits_it_off( @@ -265,9 +258,9 @@ class TestTheHook: in And (a single piece collapses to itself), even though the CJK side contributed nothing """ - leaf = _content("⺀report") + leaf = content("⺀report") - assert _widen_cjk_leaf(leaf) == _widened(leaf, _content("report")) + assert _widen_cjk_leaf(leaf) == _widened(leaf, content("report")) def test_new_nodes_carry_the_leaf_span(self) -> None: """ @@ -280,7 +273,7 @@ class TestTheHook: span, so an emit-time diagnostic still points into the query text """ - leaf = wc_ast.Term(field=_CONTENT, text="東京・大阪", startchar=3, endchar=8) + leaf = wc_ast.Term(field=CONTENT, text="東京・大阪", startchar=3, endchar=8) widened = _widen_cjk_leaf(leaf) @@ -298,9 +291,9 @@ class TestAnalyzedTree: @pytest.mark.parametrize( "node", [ - pytest.param(wc_ast.Prefix(field=_CONTENT, text="東京"), id="cjk_prefix"), + pytest.param(wc_ast.Prefix(field=CONTENT, text="東京"), id="cjk_prefix"), pytest.param( - wc_ast.Wildcard(field=_CONTENT, pattern="東*"), + wc_ast.Wildcard(field=CONTENT, pattern="東*"), id="cjk_wildcard", ), ], @@ -340,13 +333,13 @@ class TestAnalyzedTree: still negated under NOT. A NOT left holding nothing would turn into "match everything" instead """ - resolved = _get_emit_field_registry(None).resolve(_BIGRAM_CONTENT) + resolved = _get_emit_field_registry(None).resolve(BIGRAM_CONTENT) assert resolved is not None bigrams = resolved.spec.analyzer(long_cjk_run) - assert _analyze(build(_content(long_cjk_run))) == build( + assert _analyze(build(content(long_cjk_run))) == build( wc_ast.And( - children=tuple(_bigram(token) for token in dict.fromkeys(bigrams)), + children=tuple(bigram(token) for token in dict.fromkeys(bigrams)), ), ) @@ -361,11 +354,11 @@ class TestAnalyzedTree: has no bigram. An accepted gap, the same on the positive and the negated side """ - analyzed = _analyze(_content("東・大阪")) + analyzed = _analyze(content("東・大阪")) assert analyzed == _widened( - wc_ast.And(children=(_content("東"), _content("大阪"))), - _bigram("大阪"), + wc_ast.And(children=(content("東"), content("大阪"))), + bigram("大阪"), ) def test_the_title_boost_wraps_the_widened_title_leaf(self) -> None: @@ -393,14 +386,14 @@ class TestAnalyzedTree: assert ( wc_ast.Boosted( child=_widened( - wc_ast.Term(field=_TITLE, text="東京"), - wc_ast.Term(field=_BIGRAM_TITLE, text="東京"), + wc_ast.Term(field=TITLE, text="東京"), + wc_ast.Term(field=BIGRAM_TITLE, text="東京"), ), boost=_FIELD_BOOSTS["title"], ) in analyzed.children ) - assert {_BIGRAM_CONTENT, wc.FieldRef("bigram_tag")} <= { + assert {BIGRAM_CONTENT, wc.FieldRef("bigram_tag")} <= { child.field for child in analyzed.children if isinstance(child, wc_ast.Term) } @@ -418,18 +411,18 @@ class TestAnalyzedTree: alternative is not a single child here and _widened() does not fit; the tree is written out literally """ - tree = wc_ast.And(children=(_content("東京・大阪"), _content("report"))) + tree = wc_ast.And(children=(content("東京・大阪"), content("report"))) assert _analyze(tree) == wc_ast.And( children=( wc_ast.Or( children=( - wc_ast.And(children=(_content("東京"), _content("大阪"))), - _bigram("東京"), - _bigram("大阪"), + wc_ast.And(children=(content("東京"), content("大阪"))), + bigram("東京"), + bigram("大阪"), ), ), - _content("report"), + content("report"), ), ) @@ -443,15 +436,15 @@ class TestAnalyzedTree: - The original side ORs its tokens, as the group says, and the hook's Or flattens into the group """ - tree = wc_ast.Or(children=(_content("東京・大阪"), _content("report"))) + tree = wc_ast.Or(children=(content("東京・大阪"), content("report"))) assert _analyze(tree) == wc_ast.Or( children=( - _content("東京"), - _content("大阪"), - _bigram("東京"), - _bigram("大阪"), - _content("report"), + content("東京"), + content("大阪"), + bigram("東京"), + bigram("大阪"), + content("report"), ), ) @@ -460,12 +453,12 @@ class TestAnalyzedTree: [ pytest.param(lambda leaf: wc_ast.Not(child=leaf), id="not"), pytest.param( - lambda leaf: wc_ast.AndNot(positive=_content("invoice"), negative=leaf), + lambda leaf: wc_ast.AndNot(positive=content("invoice"), negative=leaf), id="andnot", ), pytest.param( lambda leaf: wc_ast.AndMaybe( - required=_content("invoice"), + required=content("invoice"), optional=leaf, ), id="andmaybe", @@ -473,7 +466,7 @@ class TestAnalyzedTree: pytest.param( lambda leaf: wc_ast.Require( scored=leaf, - filter_only=_content("invoice"), + filter_only=content("invoice"), ), id="require", ), @@ -500,6 +493,6 @@ class TestAnalyzedTree: position. A negated CJK leaf is widened too, so a negation excludes exactly what the positive search would match """ - assert _analyze(build(_content("東京"))) == build( - _widened(_content("東京"), _bigram("東京")), + assert _analyze(build(content("東京"))) == build( + _widened(content("東京"), bigram("東京")), ) diff --git a/src/documents/tests/search/test_compact_date_forms.py b/src/documents/tests/search/test_compact_date_forms.py index 48e0f2fd2..8ad4a54f5 100644 --- a/src/documents/tests/search/test_compact_date_forms.py +++ b/src/documents/tests/search/test_compact_date_forms.py @@ -20,67 +20,53 @@ from typing import TYPE_CHECKING import pytest -from documents.models import Document - if TYPE_CHECKING: - from documents.search._backend import TantivyBackend + from collections.abc import Callable + + from documents.models import Document pytestmark = [pytest.mark.search, pytest.mark.django_db] -def _matched_ids(backend: TantivyBackend, query: str) -> set[int]: - return set(backend.search_ids(query, user=None)) - - -def _index(backend: TantivyBackend, **kwargs: object) -> Document: - doc = Document.objects.create(**kwargs) - backend.add_or_update(doc) - return doc - - @pytest.fixture -def docs(backend: TantivyBackend) -> dict[str, int]: +def docs(index_document: Callable[..., Document]) -> dict[str, int]: return { - "instant": _index( - backend, + "instant": index_document( title="On the instant", content="x", - checksum="compact-date-instant", added=datetime(2005, 3, 4, 15, 30, tzinfo=UTC), ).pk, - "same_day": _index( - backend, + "same_day": index_document( title="Same day, other hour", content="x", - checksum="compact-date-same-day", added=datetime(2005, 3, 4, 9, 0, tzinfo=UTC), ).pk, - "next_day": _index( - backend, + "next_day": index_document( title="Next day, same hour", content="x", - checksum="compact-date-next-day", added=datetime(2005, 3, 5, 15, 30, tzinfo=UTC), ).pk, } -def test_fourteen_digits_is_a_single_instant( - backend: TantivyBackend, - docs: dict[str, int], -) -> None: - """ - GIVEN: - - Three documents indexed on the ``added`` DATETIME fast field: - one at 2005-03-04T15:30:00, one on the same calendar day at a - different hour, and one on the next day at the same hour - WHEN: - - Searching with the 14-digit compact date form - ``added:20050304153000`` - THEN: - - Only the document at that exact instant matches; the same-day - document is what tells this apart from the 8-digit day-window - form, and the next-day document from a form that ignored the - time of day altogether - """ - assert _matched_ids(backend, "added:20050304153000") == {docs["instant"]} +class TestCompactDateForms: + def test_fourteen_digits_is_a_single_instant( + self, + matched_ids: Callable[[str], set[int]], + docs: dict[str, int], + ) -> None: + """ + GIVEN: + - Three documents indexed on the ``added`` DATETIME fast field: + one at 2005-03-04T15:30:00, one on the same calendar day at a + different hour, and one on the next day at the same hour + WHEN: + - Searching with the 14-digit compact date form + ``added:20050304153000`` + THEN: + - Only the document at that exact instant matches; the same-day + document is what tells this apart from the 8-digit day-window + form, and the next-day document from a form that ignored the + time of day altogether + """ + assert matched_ids("added:20050304153000") == {docs["instant"]} diff --git a/src/documents/tests/search/test_date_keyword_phrase_removal.py b/src/documents/tests/search/test_date_keyword_phrase_removal.py index 97b814351..7dbd7998b 100644 --- a/src/documents/tests/search/test_date_keyword_phrase_removal.py +++ b/src/documents/tests/search/test_date_keyword_phrase_removal.py @@ -19,24 +19,14 @@ from typing import TYPE_CHECKING import pytest -from documents.models import Document - if TYPE_CHECKING: - from documents.search._backend import TantivyBackend + from collections.abc import Callable + + from documents.models import Document pytestmark = [pytest.mark.search, pytest.mark.django_db] -def _matched_ids(backend: TantivyBackend, query: str) -> set[int]: - return set(backend.search_ids(query, user=None)) - - -def _index(backend: TantivyBackend, **kwargs: object) -> Document: - doc = Document.objects.create(**kwargs) - backend.add_or_update(doc) - return doc - - class TestQuotedStringContainingDateKeywordText: """A quoted title phrase containing the literal text "added:previous month" as running words must match on that literal @@ -46,7 +36,8 @@ class TestQuotedStringContainingDateKeywordText: def test_matches_only_the_literal_phrase( self, - backend: TantivyBackend, + index_document: Callable[..., Document], + matched_ids: Callable[[str], set[int]], ) -> None: """ GIVEN: @@ -67,11 +58,9 @@ class TestQuotedStringContainingDateKeywordText: never spill into an unfielded search for "previous" and "month" across the default search fields """ - literal = _index( - backend, + literal = index_document( title="see added:previous month notes", content="quarterly filing", - checksum="dkp-literal", archive_serial_number=920, ) # Under the deleted rewrite, this decoy would incorrectly match: @@ -79,13 +68,11 @@ class TestQuotedStringContainingDateKeywordText: # corrupted parse required as title phrases, and its content # supplies "previous" and "month" as the decomposed word-match # clauses the rewrite turned the middle of the phrase into. - decoy = _index( - backend, + decoy = index_document( title="see added: quarterly report notes", content="we reviewed the previous statement about month end", - checksum="dkp-decoy", archive_serial_number=921, ) query = 'title:"see added:previous month notes"' - assert _matched_ids(backend, query) == {literal.pk} - assert decoy.pk not in _matched_ids(backend, query) + assert matched_ids(query) == {literal.pk} + assert decoy.pk not in matched_ids(query) diff --git a/src/documents/tests/search/test_date_keyword_timezone.py b/src/documents/tests/search/test_date_keyword_timezone.py index 73eca2bbe..1071ccaec 100644 --- a/src/documents/tests/search/test_date_keyword_timezone.py +++ b/src/documents/tests/search/test_date_keyword_timezone.py @@ -32,32 +32,23 @@ from typing import TYPE_CHECKING import pytest import time_machine -from documents.models import Document - if TYPE_CHECKING: + from collections.abc import Callable + from pytest_django.fixtures import SettingsWrapper - from documents.search._backend import TantivyBackend + from documents.models import Document pytestmark = [pytest.mark.search, pytest.mark.django_db] FROZEN_NOW = datetime(2026, 6, 15, 2, 0, tzinfo=UTC) -def _matched_ids(backend: TantivyBackend, query: str) -> set[int]: - return set(backend.search_ids(query, user=None)) - - -def _index(backend: TantivyBackend, **kwargs: object) -> Document: - doc = Document.objects.create(**kwargs) - backend.add_or_update(doc) - return doc - - class TestDateKeywordUsesTheActiveTimezone: def test_today_matches_the_new_york_calendar_day_not_the_utc_one( self, - backend: TantivyBackend, + index_document: Callable[..., Document], + matched_ids: Callable[[str], set[int]], settings: SettingsWrapper, ) -> None: """ @@ -79,22 +70,18 @@ class TestDateKeywordUsesTheActiveTimezone: """ settings.TIME_ZONE = "America/New_York" with time_machine.travel(FROZEN_NOW, tick=False): - in_ny_today = _index( - backend, + in_ny_today = index_document( title="NY today", content="x", - checksum="tz-keyword-ny-today", added=datetime(2026, 6, 14, 20, 0, tzinfo=UTC), ) # Not captured: the exact-set assertion below already proves # this document (inside a naive UTC-calendar-day window, but # outside New York's actual "today") does not match. - _index( - backend, + index_document( title="UTC calendar day only", content="x", - checksum="tz-keyword-utc-calendar-day-only", added=datetime(2026, 6, 15, 10, 0, tzinfo=UTC), ) - assert _matched_ids(backend, "added:today") == {in_ny_today.pk} + assert matched_ids("added:today") == {in_ny_today.pk} diff --git a/src/documents/tests/search/test_documented_syntax.py b/src/documents/tests/search/test_documented_syntax.py index fa69f3f68..d00551c25 100644 --- a/src/documents/tests/search/test_documented_syntax.py +++ b/src/documents/tests/search/test_documented_syntax.py @@ -24,16 +24,18 @@ from typing import TYPE_CHECKING import pytest import time_machine -from documents.models import Document from documents.models import Note from documents.models import Tag from documents.search._errors import InvalidDateQuery +from documents.tests.factories import DocumentFactory if TYPE_CHECKING: + from collections.abc import Callable from collections.abc import Generator from django.contrib.auth.models import User + from documents.models import Document from documents.search._backend import TantivyBackend pytestmark = [pytest.mark.search, pytest.mark.django_db] @@ -45,37 +47,23 @@ FROZEN_NOW = datetime(2026, 6, 15, 12, 0, tzinfo=UTC) DOC_CHECKSUM = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" -def _matched_ids(backend: TantivyBackend, query: str) -> set[int]: - return set(backend.search_ids(query, user=None)) - - -def _index(backend: TantivyBackend, **kwargs: object) -> Document: - doc = Document.objects.create(**kwargs) - backend.add_or_update(doc) - return doc - - class TestLogicalExpressions: @pytest.fixture - def docs(self, backend: TantivyBackend) -> dict[str, int]: + def docs(self, index_document: Callable[..., Document]) -> dict[str, int]: return { - "secret": _index( - backend, + "secret": index_document( title="Invoice one", content="invoice secret contents", - checksum="doc-syntax-secret", ).pk, - "plain": _index( - backend, + "plain": index_document( title="Invoice two", content="invoice ordinary contents", - checksum="doc-syntax-plain", ).pk, } def test_not_excludes_a_term( self, - backend: TantivyBackend, + matched_ids: Callable[[str], set[int]], docs: dict[str, int], ) -> None: """ @@ -86,11 +74,11 @@ class TestLogicalExpressions: THEN: - Only the document without "secret" matches """ - assert _matched_ids(backend, "invoice NOT secret") == {docs["plain"]} + assert matched_ids("invoice NOT secret") == {docs["plain"]} def test_leading_hyphen_requires_the_term_instead_of_excluding_it( self, - backend: TantivyBackend, + matched_ids: Callable[[str], set[int]], docs: dict[str, int], ) -> None: """ @@ -104,11 +92,11 @@ class TestLogicalExpressions: indexed as the plain term "secret" and the query becomes an AND rather than an exclusion, exactly as the docs warn """ - assert _matched_ids(backend, "invoice -secret") == {docs["secret"]} + assert matched_ids("invoice -secret") == {docs["secret"]} def test_or_inside_parentheses_matches_either_branch( self, - backend: TantivyBackend, + matched_ids: Callable[[str], set[int]], docs: dict[str, int], ) -> None: """ @@ -120,14 +108,15 @@ class TestLogicalExpressions: THEN: - Both documents match """ - matched = _matched_ids(backend, "invoice AND (secret OR ordinary)") + matched = matched_ids("invoice AND (secret OR ordinary)") assert matched == {docs["secret"], docs["plain"]} class TestPhraseSearch: def test_quoted_phrase_requires_the_words_in_order( self, - backend: TantivyBackend, + index_document: Callable[..., Document], + matched_ids: Callable[[str], set[int]], ) -> None: """ GIVEN: @@ -138,14 +127,12 @@ class TestPhraseSearch: - The in-order phrase matches, and the same words reordered do not """ - doc = _index( - backend, + doc = index_document( title="Phrase", content="the quick brown fox jumps", - checksum="doc-syntax-phrase", ) - assert _matched_ids(backend, '"quick brown fox"') == {doc.pk} - assert _matched_ids(backend, '"brown quick fox"') == set() + assert matched_ids('"quick brown fox"') == {doc.pk} + assert matched_ids('"brown quick fox"') == set() class TestTagCommaList: @@ -166,6 +153,7 @@ class TestTagCommaList: def test_comma_list_requires_every_listed_tag( self, backend: TantivyBackend, + matched_ids: Callable[[str], set[int]], ) -> None: """ GIVEN: @@ -181,30 +169,22 @@ class TestTagCommaList: unpaid = Tag.objects.create(name="unpaid") archived = Tag.objects.create(name="archived") - both = Document.objects.create( - title="Both tags", - content="body", - checksum="doc-syntax-tag-both", - ) + both = DocumentFactory(title="Both tags", content="body") both.tags.add(bills, unpaid) backend.add_or_update(both) - one = Document.objects.create( - title="One tag", - content="body", - checksum="doc-syntax-tag-one", - ) + one = DocumentFactory(title="One tag", content="body") one.tags.add(bills, archived) backend.add_or_update(one) - assert _matched_ids(backend, "tag:bills,unpaid") == {both.pk} - assert _matched_ids(backend, "tag:bills") == {both.pk, one.pk} + assert matched_ids("tag:bills,unpaid") == {both.pk} + assert matched_ids("tag:bills") == {both.pk, one.pk} class TestArchiveMetadataFields: @pytest.fixture def doc(self, backend: TantivyBackend, admin_user: User) -> Document: - doc = Document.objects.create( + doc = DocumentFactory( title="Metadata", content="body", checksum=DOC_CHECKSUM, @@ -237,7 +217,7 @@ class TestArchiveMetadataFields: ) def test_documented_metadata_query_matches( self, - backend: TantivyBackend, + matched_ids: Callable[[str], set[int]], doc: Document, query: str, ) -> None: @@ -253,7 +233,7 @@ class TestArchiveMetadataFields: THEN: - Each one matches the document """ - assert _matched_ids(backend, query) == {doc.pk} + assert matched_ids(query) == {doc.pk} @pytest.mark.parametrize( "query", @@ -265,7 +245,7 @@ class TestArchiveMetadataFields: ) def test_partial_or_uppercase_checksum_matches_nothing( self, - backend: TantivyBackend, + matched_ids: Callable[[str], set[int]], doc: Document, query: str, ) -> None: @@ -279,7 +259,7 @@ class TestArchiveMetadataFields: - Nothing matches, as the docs say only a complete, lowercase checksum matches as an exact value """ - assert _matched_ids(backend, query) == set() + assert matched_ids(query) == set() class TestDocumentedDateForms: @@ -289,7 +269,7 @@ class TestDocumentedDateForms: yield @pytest.fixture - def dated(self, backend: TantivyBackend) -> dict[str, int]: + def dated(self, index_document: Callable[..., Document]) -> dict[str, int]: stamps = { "today": datetime(2026, 6, 15, 9, 0, tzinfo=UTC), "yesterday": datetime(2026, 6, 14, 9, 0, tzinfo=UTC), @@ -300,11 +280,9 @@ class TestDocumentedDateForms: "old": datetime(2005, 3, 4, 15, 30, tzinfo=UTC), } return { - label: _index( - backend, + label: index_document( title=label, content="dated body", - checksum=f"doc-syntax-date-{label}", added=stamp, ).pk for label, stamp in stamps.items() @@ -335,7 +313,7 @@ class TestDocumentedDateForms: ) def test_documented_date_form_matches_its_day_or_month( self, - backend: TantivyBackend, + matched_ids: Callable[[str], set[int]], dated: dict[str, int], query: str, label: str, @@ -355,7 +333,7 @@ class TestDocumentedDateForms: - Each form matches exactly the document dated on its day or within its month """ - assert _matched_ids(backend, query) == {dated[label]} + assert matched_ids(query) == {dated[label]} @pytest.mark.parametrize( "query", @@ -381,7 +359,7 @@ class TestDocumentedDateForms: ) def test_forms_the_docs_warn_about_match_nothing( self, - backend: TantivyBackend, + matched_ids: Callable[[str], set[int]], dated: dict[str, int], query: str, ) -> None: @@ -400,11 +378,11 @@ class TestDocumentedDateForms: - Nothing matches, exactly as the docs warn, rather than presenting these as usable spellings """ - assert _matched_ids(backend, query) == set() + assert matched_ids(query) == set() def test_bare_timestamp_is_rejected_rather_than_matching_nothing( self, - backend: TantivyBackend, + matched_ids: Callable[[str], set[int]], dated: dict[str, int], ) -> None: """ @@ -423,13 +401,13 @@ class TestDocumentedDateForms: prefix the date grammar's tokenizer first split on """ with pytest.raises(InvalidDateQuery) as exc_info: - _matched_ids(backend, "added:2005-03-04T15:30:00Z") + matched_ids("added:2005-03-04T15:30:00Z") assert exc_info.value.field == "added" assert exc_info.value.value == "2005-03-04T15:30:00Z" def test_relative_offset_as_a_range_bound_is_a_real_window( self, - backend: TantivyBackend, + matched_ids: Callable[[str], set[int]], dated: dict[str, int], ) -> None: """ @@ -447,14 +425,14 @@ class TestDocumentedDateForms: the offset itself and not a whole-day rounding of it, as the docs say next to the warning about the standalone form """ - assert _matched_ids(backend, "added:['-1 week' to now]") == { + assert matched_ids("added:['-1 week' to now]") == { dated["today"], dated["yesterday"], } def test_double_quoted_range_bound_is_rejected( self, - backend: TantivyBackend, + matched_ids: Callable[[str], set[int]], dated: dict[str, int], ) -> None: """ @@ -471,5 +449,5 @@ class TestDocumentedDateForms: attached and is not a recognizable date """ with pytest.raises(InvalidDateQuery) as exc_info: - _matched_ids(backend, 'added:["2005-03-04" to 2005-03-05]') + matched_ids('added:["2005-03-04" to 2005-03-05]') assert exc_info.value.value == '"2005-03-04"' diff --git a/src/documents/tests/search/test_error_routing.py b/src/documents/tests/search/test_error_routing.py index cd8822560..3382a395e 100644 --- a/src/documents/tests/search/test_error_routing.py +++ b/src/documents/tests/search/test_error_routing.py @@ -8,9 +8,9 @@ from __future__ import annotations import logging from datetime import UTC +from typing import TYPE_CHECKING import pytest -import tantivy from whoosh_compat.errors import Diagnostic from whoosh_compat.errors import DiagnosticKind from whoosh_compat.errors import QueryError @@ -22,22 +22,16 @@ from documents.search._errors import SearchQueryError from documents.search._query import _map_emit_error from documents.search._query import _single_diagnostic_to_error from documents.search._query import parse_user_query -from documents.search._schema import build_schema -from documents.search._tokenizer import register_tokenizers + +if TYPE_CHECKING: + import tantivy + pytestmark = pytest.mark.search _LIBRARY_PROSE = "INTERNAL LIBRARY WORDING WITH raw tantivy detail" -@pytest.fixture(scope="module") -def query_index() -> tantivy.Index: - """An in-memory, unstemmed index; these tests only parse, never index.""" - idx = tantivy.Index(build_schema(), path=None) - register_tokenizers(idx, "") - return idx - - def _diagnostic( kind: DiagnosticKind, *, diff --git a/src/documents/tests/search/test_exists_on_json_fields.py b/src/documents/tests/search/test_exists_on_json_fields.py index 875087439..945668f86 100644 --- a/src/documents/tests/search/test_exists_on_json_fields.py +++ b/src/documents/tests/search/test_exists_on_json_fields.py @@ -19,9 +19,9 @@ from __future__ import annotations import logging from datetime import UTC +from typing import TYPE_CHECKING import pytest -import tantivy from whoosh_compat.errors import Diagnostic from whoosh_compat.errors import DiagnosticKind from whoosh_compat.errors import QueryError @@ -32,8 +32,10 @@ from whoosh_compat.fields import FieldRef from documents.search._errors import SearchQueryError from documents.search._query import _map_emit_error from documents.search._query import parse_user_query -from documents.search._schema import build_schema -from documents.search._tokenizer import register_tokenizers + +if TYPE_CHECKING: + import tantivy + pytestmark = pytest.mark.search @@ -48,13 +50,6 @@ EXISTS_QUERIES = [ ] -@pytest.fixture(scope="module") -def query_index() -> tantivy.Index: - idx = tantivy.Index(build_schema(), path=None) - register_tokenizers(idx, "") - return idx - - class TestJsonExistsIsUserError: @pytest.mark.parametrize("query", EXISTS_QUERIES) def test_query_is_a_400_that_emits_no_error_log( diff --git a/src/documents/tests/search/test_fuzzy_alternative.py b/src/documents/tests/search/test_fuzzy_alternative.py index b788bc7c4..0e81893e9 100644 --- a/src/documents/tests/search/test_fuzzy_alternative.py +++ b/src/documents/tests/search/test_fuzzy_alternative.py @@ -14,11 +14,12 @@ import whoosh_compat as wc import whoosh_compat.ast as wc_ast from documents.search._query import _fuzzy_alternative +from documents.tests.search._ast_helpers import CONTENT +from documents.tests.search._ast_helpers import TITLE +from documents.tests.search._ast_helpers import content pytestmark = pytest.mark.search -_CONTENT = wc.FieldRef("content") -_TITLE = wc.FieldRef("title") # 130 characters, past the analyzer's 129-byte remove_long limit (128 is # kept, 129 is dropped), so the index never holds it and neither side @@ -26,10 +27,6 @@ _TITLE = wc.FieldRef("title") _TOO_LONG = "x" * 130 -def _content(text: str) -> wc_ast.Term: - return wc_ast.Term(field=_CONTENT, text=text) - - def _fuzzy(field: wc.FieldRef, text: str) -> wc_ast.Fuzzy: return wc_ast.Fuzzy(field=field, text=text, distance=1, prefix=True) @@ -45,7 +42,7 @@ class TestTheAlternative: - It is a single Fuzzy leaf on the same field, distance 1 and prefix matching, which is what the old clause used """ - assert _fuzzy_alternative(_content("invoice")) == _fuzzy(_CONTENT, "invoice") + assert _fuzzy_alternative(content("invoice")) == _fuzzy(CONTENT, "invoice") def test_the_leafs_own_field_is_used(self) -> None: """ @@ -58,8 +55,8 @@ class TestTheAlternative: fields. This is the fielding fix: today's clause searches a fielded word everywhere """ - assert _fuzzy_alternative(wc_ast.Term(field=_TITLE, text="invoice")) == _fuzzy( - _TITLE, + assert _fuzzy_alternative(wc_ast.Term(field=TITLE, text="invoice")) == _fuzzy( + TITLE, "invoice", ) @@ -74,8 +71,8 @@ class TestTheAlternative: side, which is OR'd for an unfielded term, and would lose today's per-word recall inside a term """ - assert _fuzzy_alternative(_content("COVID-19")) == wc_ast.Or( - children=(_fuzzy(_CONTENT, "covid"), _fuzzy(_CONTENT, "19")), + assert _fuzzy_alternative(content("COVID-19")) == wc_ast.Or( + children=(_fuzzy(CONTENT, "covid"), _fuzzy(CONTENT, "19")), ) def test_words_are_split_where_the_index_splits_them(self) -> None: @@ -88,7 +85,7 @@ class TestTheAlternative: - The Fuzzy text is the analyzer's output, not the raw text, so it is in the same shape as the index terms """ - assert _fuzzy_alternative(_content("Éclair")) == _fuzzy(_CONTENT, "eclair") + assert _fuzzy_alternative(content("Éclair")) == _fuzzy(CONTENT, "eclair") def test_the_splitter_does_not_stem(self) -> None: """ @@ -101,8 +98,8 @@ class TestTheAlternative: pattern_normalizer, which stems it once; stemming here too would search for a term the index does not hold """ - assert _fuzzy_alternative(_content("universities")) == _fuzzy( - _CONTENT, + assert _fuzzy_alternative(content("universities")) == _fuzzy( + CONTENT, "universities", ) @@ -130,7 +127,7 @@ class TestTheAlternative: that can never match, making a query return LESS with fuzzy on than off """ - assert _fuzzy_alternative(_content(text)) is None + assert _fuzzy_alternative(content(text)) is None def test_a_one_character_word_is_dropped_from_a_longer_term(self) -> None: """ @@ -142,7 +139,7 @@ class TestTheAlternative: - Only the real word survives. A one-character prefix fuzzy term matches every term in the field """ - assert _fuzzy_alternative(_content("h52.1")) == _fuzzy(_CONTENT, "h52") + assert _fuzzy_alternative(content("h52.1")) == _fuzzy(CONTENT, "h52") def test_the_leaf_span_is_copied(self) -> None: """ @@ -155,7 +152,7 @@ class TestTheAlternative: - Every node it builds carries that span, so an emit-time diagnostic still points into the query text """ - leaf = wc_ast.Term(field=_CONTENT, text="ab-cd", startchar=4, endchar=9) + leaf = wc_ast.Term(field=CONTENT, text="ab-cd", startchar=4, endchar=9) alternative = _fuzzy_alternative(leaf) @@ -181,10 +178,10 @@ class TestPhrasesAreNotWidened: quoted phrase match strictly more than the same two words unquoted """ - leaf = wc_ast.Phrase(field=_CONTENT, text="tax report") + leaf = wc_ast.Phrase(field=CONTENT, text="tax report") assert _fuzzy_alternative(leaf) == wc_ast.And( - children=(_fuzzy(_CONTENT, "tax"), _fuzzy(_CONTENT, "report")), + children=(_fuzzy(CONTENT, "tax"), _fuzzy(CONTENT, "report")), ) @@ -209,7 +206,7 @@ class TestCjkGetsNoFuzzySide: one edit of its start, which is the "東京都 matches 京都" failure the bigram fields exist to avoid """ - assert _fuzzy_alternative(_content(text)) is None + assert _fuzzy_alternative(content(text)) is None def test_latin_beside_cjk_still_gets_its_fuzzy_side(self) -> None: """ @@ -221,6 +218,6 @@ class TestCjkGetsNoFuzzySide: - Only the latin word is fuzzed. Skipping CJK words must not cost the latin half its near-match """ - leaf = _content("東京 report") + leaf = content("東京 report") - assert _fuzzy_alternative(leaf) == _fuzzy(_CONTENT, "report") + assert _fuzzy_alternative(leaf) == _fuzzy(CONTENT, "report") diff --git a/src/documents/tests/search/test_fuzzy_tokenization.py b/src/documents/tests/search/test_fuzzy_tokenization.py index 8eaff30bb..e95b12df0 100644 --- a/src/documents/tests/search/test_fuzzy_tokenization.py +++ b/src/documents/tests/search/test_fuzzy_tokenization.py @@ -13,38 +13,23 @@ from typing import TYPE_CHECKING import pytest -from documents.models import Document - if TYPE_CHECKING: - from pytest_django.fixtures import SettingsWrapper + from collections.abc import Callable - from documents.search._backend import TantivyBackend + from documents.models import Document -pytestmark = [pytest.mark.search, pytest.mark.django_db] - - -def _matched_ids(backend: TantivyBackend, query: str) -> set[int]: - return set(backend.search_ids(query, user=None)) - - -def _index(backend: TantivyBackend, **kwargs: object) -> Document: - doc = Document.objects.create(**kwargs) - backend.add_or_update(doc) - return doc - - -@pytest.fixture(autouse=True) -def fuzzy_enabled(settings: SettingsWrapper) -> None: - """Enable the fuzzy blend clause. The threshold doubles as a minimum - score filter, so it is set to 0.0: every hit passes and the test sees - the clause's matching behaviour, not the filter's.""" - settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.0 +pytestmark = [ + pytest.mark.search, + pytest.mark.django_db, + pytest.mark.usefixtures("fuzzy_enabled"), +] class TestFuzzyClauseWords: def test_a_stemmed_word_is_not_stemmed_a_second_time( self, - backend: TantivyBackend, + index_document: Callable[..., Document], + matched_ids: Callable[[str], set[int]], ) -> None: """ GIVEN: @@ -62,36 +47,29 @@ class TestFuzzyClauseWords: reaches unrelated words - the clause must stay wide enough for a typo and no wider """ - wanted = _index( - backend, + wanted = index_document( title="A", content="universities of europe", - checksum="fuzz-stem-1", ) - typo = _index( - backend, + typo = index_document( title="B", content="universties of europe", - checksum="fuzz-stem-2", ) - _index( - backend, + index_document( title="C", content="univalent chemical bonds", - checksum="fuzz-stem-3", ) - _index( - backend, + index_document( title="D", content="unicycle repair manual", - checksum="fuzz-stem-4", ) - assert _matched_ids(backend, "universities") == {wanted.pk, typo.pk} + assert matched_ids("universities") == {wanted.pk, typo.pk} def test_a_hyphenated_term_still_reaches_the_clause( self, - backend: TantivyBackend, + index_document: Callable[..., Document], + matched_ids: Callable[[str], set[int]], ) -> None: """ GIVEN: @@ -105,18 +83,17 @@ class TestFuzzyClauseWords: would read as grammar, is dropped, and the whole query loses its fuzzy clause """ - misspelled = _index( - backend, + misspelled = index_document( title="A", content="covidx testing results", - checksum="fuzz-hyphen-1", ) - assert _matched_ids(backend, "COVID-19") == {misspelled.pk} + assert matched_ids("COVID-19") == {misspelled.pk} def test_a_phrase_still_reaches_the_clause( self, - backend: TantivyBackend, + index_document: Callable[..., Document], + matched_ids: Callable[[str], set[int]], ) -> None: """ GIVEN: @@ -129,14 +106,12 @@ class TestFuzzyClauseWords: space, and is the whole query's only free text here, so it must still reach the clause """ - near_miss = _index( - backend, + near_miss = index_document( title="A", content="taxation reportage weekly", - checksum="fuzz-phrase-1", ) - assert _matched_ids(backend, '"tax reports"') == {near_miss.pk} + assert matched_ids('"tax reports"') == {near_miss.pk} class TestBooleanKeywordsInRawText: @@ -146,24 +121,18 @@ class TestBooleanKeywordsInRawText: now, which closes that off structurally; these pin it shut.""" @pytest.fixture - def corpus(self, backend: TantivyBackend) -> dict[str, int]: - both = _index( - backend, + def corpus(self, index_document: Callable[..., Document]) -> dict[str, int]: + both = index_document( title="A", content="taxation reportage weekly", - checksum="fuzz-kw-1", ) - tax_only = _index( - backend, + tax_only = index_document( title="B", content="taxation only here", - checksum="fuzz-kw-2", ) - report_only = _index( - backend, + report_only = index_document( title="C", content="reportage only here", - checksum="fuzz-kw-3", ) return { "both": both.pk, @@ -182,7 +151,7 @@ class TestBooleanKeywordsInRawText: ) def test_a_keyword_inside_a_phrase_stays_an_ordinary_word( self, - backend: TantivyBackend, + matched_ids: Callable[[str], set[int]], corpus: dict[str, int], keyword_spelling: str, ordinary_spelling: str, @@ -201,14 +170,11 @@ class TestBooleanKeywordsInRawText: exclusion, IN does not fail the parse. Only the upper-case spelling was ever grammar """ - assert _matched_ids(backend, keyword_spelling) == _matched_ids( - backend, - ordinary_spelling, - ) + assert matched_ids(keyword_spelling) == matched_ids(ordinary_spelling) def test_a_phrase_needs_a_near_match_for_every_word( self, - backend: TantivyBackend, + matched_ids: Callable[[str], set[int]], corpus: dict[str, int], ) -> None: """ @@ -222,11 +188,11 @@ class TestBooleanKeywordsInRawText: quoted phrase asks for more than the bare words, so its fuzzy side requires every one of them """ - assert _matched_ids(backend, '"tax reports"') == {corpus["both"]} + assert matched_ids('"tax reports"') == {corpus["both"]} def test_a_trailing_keyword_is_just_a_word( self, - backend: TantivyBackend, + matched_ids: Callable[[str], set[int]], corpus: dict[str, int], ) -> None: """ @@ -242,7 +208,4 @@ class TestBooleanKeywordsInRawText: is re-parsed any more, so a trailing keyword cannot cost the query its fuzzy side """ - assert _matched_ids(backend, '"tax AND"') == _matched_ids( - backend, - '"tax and"', - ) + assert matched_ids('"tax AND"') == matched_ids('"tax and"') diff --git a/src/documents/tests/search/test_highlight_query_guard.py b/src/documents/tests/search/test_highlight_query_guard.py index 48bf83502..cf9e96378 100644 --- a/src/documents/tests/search/test_highlight_query_guard.py +++ b/src/documents/tests/search/test_highlight_query_guard.py @@ -29,8 +29,6 @@ from rest_framework import status from documents.search._backend import SearchMode from documents.search._query import parse_simple_text_highlight_query -from documents.search._schema import build_schema -from documents.search._tokenizer import register_tokenizers from documents.tests.factories import DocumentFactory if TYPE_CHECKING: @@ -55,15 +53,6 @@ _MALFORMED_QUERIES = [ ] -@pytest.fixture(scope="module") -def query_index() -> tantivy.Index: - """An in-memory, unstemmed index for parse-only tests.""" - schema = build_schema() - idx = tantivy.Index(schema, path=None) - register_tokenizers(idx, "") - return idx - - class TestParseSimpleTextHighlightQueryDoesNotRaise: """The query builder itself must tolerate Tantivy syntax in its tokens.""" diff --git a/src/documents/tests/search/test_json_field_prefixes.py b/src/documents/tests/search/test_json_field_prefixes.py index 8d0ccedd4..df737f39d 100644 --- a/src/documents/tests/search/test_json_field_prefixes.py +++ b/src/documents/tests/search/test_json_field_prefixes.py @@ -21,29 +21,24 @@ from django.contrib.auth.models import User from documents.models import CustomField from documents.models import CustomFieldInstance -from documents.models import Document from documents.models import Note +from documents.tests.factories import DocumentFactory if TYPE_CHECKING: + from collections.abc import Callable + + from documents.models import Document from documents.search._backend import TantivyBackend pytestmark = [pytest.mark.search, pytest.mark.django_db] -def _matched_ids(backend: TantivyBackend, query: str) -> set[int]: - return set(backend.search_ids(query, user=None)) - - -def _index(backend: TantivyBackend, **kwargs: object) -> Document: - doc = Document.objects.create(**kwargs) - backend.add_or_update(doc) - return doc - - class TestBareJsonFieldPrefixes: def test_bare_notes_prefix_searches_note_text( self, backend: TantivyBackend, + index_document: Callable[..., Document], + matched_ids: Callable[[str], set[int]], ) -> None: """ GIVEN: @@ -59,26 +54,22 @@ class TestBareJsonFieldPrefixes: text search """ alice = User.objects.create_user(username="alice") - with_note = Document.objects.create( - title="Has note", - content="x", - checksum="bare-notes-with", - ) + with_note = DocumentFactory(title="Has note", content="x") Note.objects.create(document=with_note, user=alice, note="crocodile") backend.add_or_update(with_note) # This document's CONTENT contains the words a demoted text search # would match; it must NOT match once the prefix addresses notes. - _index( - backend, + index_document( title="Notes about things", content="notes crocodile mention", - checksum="bare-notes-decoy", ) - assert _matched_ids(backend, "notes:crocodile") == {with_note.pk} + assert matched_ids("notes:crocodile") == {with_note.pk} def test_bare_custom_fields_prefix_searches_values( self, backend: TantivyBackend, + index_document: Callable[..., Document], + matched_ids: Callable[[str], set[int]], ) -> None: """ GIVEN: @@ -96,28 +87,23 @@ class TestBareJsonFieldPrefixes: name="Policy Number", data_type=CustomField.FieldDataType.STRING, ) - with_value = Document.objects.create( - title="Has field", - content="x", - checksum="bare-cf-with", - ) + with_value = DocumentFactory(title="Has field", content="x") CustomFieldInstance.objects.create( document=with_value, field=field, value_text="crocodile", ) backend.add_or_update(with_value) - _index( - backend, + index_document( title="Custom things", content="custom fields crocodile", - checksum="bare-cf-decoy", ) - assert _matched_ids(backend, "custom_fields:crocodile") == {with_value.pk} + assert matched_ids("custom_fields:crocodile") == {with_value.pk} def test_subpath_spellings_are_untouched( self, backend: TantivyBackend, + matched_ids: Callable[[str], set[int]], ) -> None: """ GIVEN: @@ -131,15 +117,11 @@ class TestBareJsonFieldPrefixes: prefix does not interfere with explicit subpath addressing """ bob = User.objects.create_user(username="bob") - doc = Document.objects.create( - title="Bob note", - content="x", - checksum="bare-subpath", - ) + doc = DocumentFactory(title="Bob note", content="x") Note.objects.create(document=doc, user=bob, note="remark") backend.add_or_update(doc) - assert _matched_ids(backend, "notes.user:bob") == {doc.pk} - assert _matched_ids(backend, "notes.note:remark") == {doc.pk} + assert matched_ids("notes.user:bob") == {doc.pk} + assert matched_ids("notes.note:remark") == {doc.pk} class TestQuotedPhraseContainingNotesColonIsNotCorrupted: @@ -152,7 +134,8 @@ class TestQuotedPhraseContainingNotesColonIsNotCorrupted: def test_quoted_phrase_with_notes_colon_matches_by_content( self, - backend: TantivyBackend, + index_document: Callable[..., Document], + matched_ids: Callable[[str], set[int]], ) -> None: """ GIVEN: @@ -167,20 +150,16 @@ class TestQuotedPhraseContainingNotesColonIsNotCorrupted: that matches nothing (the bug the deleted regex rewrite caused, since it was blind to quoting) """ - target = _index( - backend, + target = index_document( title="Statement", content="payment notes: none", - checksum="quoted-phrase-notes-colon", ) - assert _matched_ids( - backend, - 'content:"payment notes: none"', - ) == {target.pk} + assert matched_ids('content:"payment notes: none"') == {target.pk} def test_quoted_phrase_matches_the_same_document_unquoted( self, - backend: TantivyBackend, + index_document: Callable[..., Document], + matched_ids: Callable[[str], set[int]], ) -> None: """ GIVEN: @@ -194,13 +173,8 @@ class TestQuotedPhraseContainingNotesColonIsNotCorrupted: quote-awareness specifically, not about the words themselves being unsearchable """ - target = _index( - backend, + target = index_document( title="Statement", content="payment notes none", - checksum="quoted-phrase-no-colon", ) - assert _matched_ids( - backend, - 'content:"payment notes none"', - ) == {target.pk} + assert matched_ids('content:"payment notes none"') == {target.pk} diff --git a/src/documents/tests/search/test_negated_leaf_ids.py b/src/documents/tests/search/test_negated_leaf_ids.py index bea7069ec..9df5daec0 100644 --- a/src/documents/tests/search/test_negated_leaf_ids.py +++ b/src/documents/tests/search/test_negated_leaf_ids.py @@ -11,19 +11,14 @@ identity. from __future__ import annotations import pytest -import whoosh_compat as wc import whoosh_compat.ast as wc_ast from documents.search._query import _negated_leaf_ids +from documents.tests.search._ast_helpers import CONTENT +from documents.tests.search._ast_helpers import content pytestmark = pytest.mark.search -_CONTENT = wc.FieldRef("content") - - -def _term(text: str) -> wc_ast.Term: - return wc_ast.Term(field=_CONTENT, text=text) - class TestCollection: def test_a_bare_tree_has_no_negated_leaves(self) -> None: @@ -35,7 +30,7 @@ class TestCollection: THEN: - The set is empty """ - tree = wc_ast.And(children=(_term("invoice"), _term("report"))) + tree = wc_ast.And(children=(content("invoice"), content("report"))) assert _negated_leaf_ids(tree) == frozenset() @@ -48,8 +43,8 @@ class TestCollection: THEN: - That leaf's id is collected, and the positive one is not """ - positive = _term("invoice") - negated = _term("secret") + positive = content("invoice") + negated = content("secret") tree = wc_ast.And(children=(positive, wc_ast.Not(child=negated))) ids = _negated_leaf_ids(tree) @@ -66,8 +61,8 @@ class TestCollection: THEN: - Only the negative side is collected """ - positive = _term("invoice") - negated = _term("secret") + positive = content("invoice") + negated = content("secret") tree = wc_ast.AndNot(positive=positive, negative=negated) ids = _negated_leaf_ids(tree) @@ -86,9 +81,9 @@ class TestCollection: - All of them are collected, at any depth and whatever the leaf type """ - a = _term("alpha") - b = _term("beta") - phrase = wc_ast.Phrase(field=_CONTENT, text="gamma delta") + a = content("alpha") + b = content("beta") + phrase = wc_ast.Phrase(field=CONTENT, text="gamma delta") tree = wc_ast.Not( child=wc_ast.Or( children=(a, wc_ast.And(children=(b, phrase))), @@ -109,7 +104,7 @@ class TestCollection: under-collecting would make a negation exclude far more than the user asked """ - leaf = _term("tax") + leaf = content("tax") tree = wc_ast.Not(child=wc_ast.Not(child=leaf)) assert _negated_leaf_ids(tree) == frozenset({id(leaf)}) @@ -144,7 +139,7 @@ class TestCollection: - Nothing is collected. Only Not.child and AndNot.negative are negative positions """ - leaf = _term("invoice") + leaf = content("invoice") assert _negated_leaf_ids(build(leaf)) == frozenset() @@ -157,7 +152,7 @@ class TestCollection: THEN: - The set is empty and nothing raises """ - assert _negated_leaf_ids(_term("invoice")) == frozenset() + assert _negated_leaf_ids(content("invoice")) == frozenset() class TestTotality: @@ -167,11 +162,11 @@ class TestTotality: pytest.param(wc_ast.Every(), id="every"), pytest.param(wc_ast.Nothing(), id="nothing"), pytest.param( - wc_ast.Wildcard(field=_CONTENT, pattern="inv*"), + wc_ast.Wildcard(field=CONTENT, pattern="inv*"), id="wildcard", ), pytest.param( - wc_ast.Fuzzy(field=_CONTENT, text="invoce", distance=1, prefix=True), + wc_ast.Fuzzy(field=CONTENT, text="invoce", distance=1, prefix=True), id="fuzzy", ), ], @@ -200,7 +195,7 @@ class TestTotality: - It completes. The walk is iterative, so depth costs heap rather than Python stack frames """ - leaf = _term("invoice") + leaf = content("invoice") node: wc_ast.Node = leaf for _ in range(5000): node = wc_ast.Not(child=node) diff --git a/src/documents/tests/search/test_pattern_stemming.py b/src/documents/tests/search/test_pattern_stemming.py index 79bc4ff46..6fe4d8629 100644 --- a/src/documents/tests/search/test_pattern_stemming.py +++ b/src/documents/tests/search/test_pattern_stemming.py @@ -18,10 +18,10 @@ from typing import TYPE_CHECKING import pytest -from documents.models import Document - if TYPE_CHECKING: - from documents.search._backend import TantivyBackend + from collections.abc import Callable + + from documents.models import Document pytestmark = [pytest.mark.search, pytest.mark.django_db] @@ -31,20 +31,13 @@ CONTENT = ( ) -def _matched_ids(backend: TantivyBackend, query: str) -> set[int]: - return set(backend.search_ids(query, user=None)) - - @pytest.fixture -def indexed_doc(backend: TantivyBackend) -> Document: - doc = Document.objects.create( +def indexed_doc(index_document: Callable[..., Document]) -> Document: + return index_document( title="Invoice 2020 productname", content=CONTENT, - checksum="pattern-stemming-1", archive_serial_number=900, ) - backend.add_or_update(doc) - return doc class TestPrefixStemming: @@ -61,7 +54,7 @@ class TestPrefixStemming: ) def test_full_word_prefix_matches_its_stem( self, - backend: TantivyBackend, + matched_ids: Callable[[str], set[int]], indexed_doc: Document, query: str, ) -> None: @@ -78,12 +71,12 @@ class TestPrefixStemming: the word's stem as an alternative alongside the typed run, reaching the stemmed index term """ - assert _matched_ids(backend, query) == {indexed_doc.id} + assert matched_ids(query) == {indexed_doc.id} @pytest.mark.parametrize("query", ["invoic*", "electr*", "payment*"]) def test_already_stemmed_prefix_still_matches( self, - backend: TantivyBackend, + matched_ids: Callable[[str], set[int]], indexed_doc: Document, query: str, ) -> None: @@ -97,12 +90,12 @@ class TestPrefixStemming: - The document still matches, since the typed-run alternative is itself a prefix of the stored stemmed term """ - assert _matched_ids(backend, query) == {indexed_doc.id} + assert matched_ids(query) == {indexed_doc.id} @pytest.mark.parametrize("query", ["univers*", "librar*"]) def test_partial_prefix_reaches_the_stemmed_term( self, - backend: TantivyBackend, + matched_ids: Callable[[str], set[int]], indexed_doc: Document, query: str, ) -> None: @@ -123,11 +116,11 @@ class TestPrefixStemming: genuinely diverge, and only one of them matches, is test_stem_substitution_reaches_both_the_inflection_and_the_compound """ - assert _matched_ids(backend, query) == {indexed_doc.id} + assert matched_ids(query) == {indexed_doc.id} def test_full_word_reaches_the_stem_but_a_fragment_of_it_does_not( self, - backend: TantivyBackend, + matched_ids: Callable[[str], set[int]], indexed_doc: Document, ) -> None: """ @@ -144,13 +137,13 @@ class TestPrefixStemming: usage.md tells a reader whose `universit*` finds nothing to shorten it to `univers*`, which matches """ - assert _matched_ids(backend, "universities*") == {indexed_doc.id} - assert _matched_ids(backend, "universit*") == set() - assert _matched_ids(backend, "univers*") == {indexed_doc.id} + assert matched_ids("universities*") == {indexed_doc.id} + assert matched_ids("universit*") == set() + assert matched_ids("univers*") == {indexed_doc.id} def test_pattern_past_the_stem_boundary_is_documented_not_fixed( self, - backend: TantivyBackend, + matched_ids: Callable[[str], set[int]], indexed_doc: Document, ) -> None: """ @@ -165,11 +158,12 @@ class TestPrefixStemming: index, and usage.md must not advertise it. Pinned so the limitation is deliberate, not accidental """ - assert _matched_ids(backend, "produ*name") == set() + assert matched_ids("produ*name") == set() def test_stem_substitution_reaches_both_the_inflection_and_the_compound( self, - backend: TantivyBackend, + index_document: Callable[..., Document], + matched_ids: Callable[[str], set[int]], indexed_doc: Document, ) -> None: """ @@ -189,22 +183,20 @@ class TestPrefixStemming: folded and stemmed forms, and "copy*" reaches the base word, its inflections and the compound alike """ - compound = Document.objects.create( + compound = index_document( title="Copyright notice", content="copyright notice for the work", - checksum="pattern-stemming-2", archive_serial_number=901, ) - backend.add_or_update(compound) - assert _matched_ids(backend, "copy*") == {indexed_doc.id, compound.id} - assert _matched_ids(backend, "copyright*") == {compound.id} + assert matched_ids("copy*") == {indexed_doc.id, compound.id} + assert matched_ids("copyright*") == {compound.id} class TestBracketClassStillFolds: def test_class_body_matches_case_insensitively( self, - backend: TantivyBackend, + matched_ids: Callable[[str], set[int]], indexed_doc: Document, ) -> None: """ @@ -218,4 +210,4 @@ class TestBracketClassStillFolds: the alternatives contract preserves only because a lone character stems to itself """ - assert _matched_ids(backend, "title:[IP]nvoice*") == {indexed_doc.id} + assert matched_ids("title:[IP]nvoice*") == {indexed_doc.id} diff --git a/src/documents/tests/search/test_query.py b/src/documents/tests/search/test_query.py index aae802b85..4271df291 100644 --- a/src/documents/tests/search/test_query.py +++ b/src/documents/tests/search/test_query.py @@ -24,16 +24,6 @@ if TYPE_CHECKING: pytestmark = pytest.mark.search -@pytest.fixture(scope="module") -def query_index() -> tantivy.Index: - """An in-memory, unstemmed index shared read-only across this module's - parse-only tests (none of them index documents).""" - schema = build_schema() - idx = tantivy.Index(schema, path=None) - register_tokenizers(idx, "") - return idx - - @pytest.fixture(scope="module") def populated_index() -> tantivy.Index: """An index holding one document, so a query matching nothing is diff --git a/src/documents/tests/search/test_query_negation.py b/src/documents/tests/search/test_query_negation.py index b967da2aa..d7090894b 100644 --- a/src/documents/tests/search/test_query_negation.py +++ b/src/documents/tests/search/test_query_negation.py @@ -16,22 +16,12 @@ import pytest if TYPE_CHECKING: from collections.abc import Callable - from pytest_django.fixtures import SettingsWrapper - from documents.models import Document pytestmark = [pytest.mark.search, pytest.mark.django_db] -@pytest.fixture -def fuzzy_enabled(settings: SettingsWrapper) -> None: - """Enable the fuzzy blend clause. The threshold doubles as a minimum - score filter, so it is set to 0.0: every hit passes and the test sees - the clause's matching behaviour, not the filter's.""" - settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.0 - - class TestNegationConstrainsEveryClause: @pytest.mark.usefixtures("fuzzy_enabled") def test_fuzzy_clause_does_not_readmit_an_excluded_document( diff --git a/src/documents/tests/search/test_widen_leaf.py b/src/documents/tests/search/test_widen_leaf.py index 258960c4d..c12a89ca4 100644 --- a/src/documents/tests/search/test_widen_leaf.py +++ b/src/documents/tests/search/test_widen_leaf.py @@ -7,22 +7,16 @@ negated leaf keeps its CJK side while losing its fuzzy one. from __future__ import annotations import pytest -import whoosh_compat as wc import whoosh_compat.ast as wc_ast from documents.search._query import _cjk_alternative from documents.search._query import _fuzzy_alternative from documents.search._query import _widen_leaf +from documents.tests.search._ast_helpers import NOTES +from documents.tests.search._ast_helpers import content pytestmark = pytest.mark.search -_CONTENT = wc.FieldRef("content") -_NOTES = wc.FieldRef("notes", "note") - - -def _content(text: str) -> wc_ast.Term: - return wc_ast.Term(field=_CONTENT, text=text) - def _widen( leaf: wc_ast.Term, @@ -44,7 +38,7 @@ class TestWhichAlternativesAreAdded: - The Or holds the leaf and the boosted fuzzy alternative, and nothing else """ - leaf = _content("invoice") + leaf = content("invoice") assert _widen(leaf) == wc_ast.Or( children=( @@ -63,7 +57,7 @@ class TestWhichAlternativesAreAdded: - Only the bigram alternative is added, which is exactly what the CJK work shipped """ - leaf = _content("東京") + leaf = content("東京") assert _widen(leaf, fuzzy=False) == wc_ast.Or( children=(leaf, _cjk_alternative(leaf)), @@ -81,7 +75,7 @@ class TestWhichAlternativesAreAdded: any run within one edit of its start, and _cjk_alternative already supplies the in-run recall """ - leaf = _content("東京") + leaf = content("東京") assert _fuzzy_alternative(leaf) is None assert _widen(leaf) == wc_ast.Or(children=(leaf, _cjk_alternative(leaf))) @@ -97,7 +91,7 @@ class TestWhichAlternativesAreAdded: NOT X should exclude what X matches, which needs the bigram side, but prefix fuzzy matching would exclude far more """ - leaf = _content("東京") + leaf = content("東京") assert _widen(leaf, negated=frozenset({id(leaf)})) == wc_ast.Or( children=(leaf, _cjk_alternative(leaf)), @@ -113,7 +107,7 @@ class TestWhichAlternativesAreAdded: - The leaf itself comes back. It qualifies for no alternative at all, so there is no Or to build """ - leaf = _content("tax") + leaf = content("tax") assert _widen(leaf, negated=frozenset({id(leaf)})) is leaf @@ -128,6 +122,6 @@ class TestWhichAlternativesAreAdded: - The leaf comes back untouched. Widening is scoped to the default search fields, as it was for CJK """ - leaf = wc_ast.Term(field=_NOTES, text="invoice") + leaf = wc_ast.Term(field=NOTES, text="invoice") assert _widen(leaf) is leaf diff --git a/src/documents/tests/test_api_search_errors.py b/src/documents/tests/test_api_search_errors.py index cd89d7355..c8a2849b9 100644 --- a/src/documents/tests/test_api_search_errors.py +++ b/src/documents/tests/test_api_search_errors.py @@ -19,7 +19,6 @@ from whoosh_compat.errors import DiagnosticKind from whoosh_compat.errors import QueryError from documents.search import SearchQueryError -from documents.tests.factories import DocumentFactory if TYPE_CHECKING: from rest_framework.test import APIClient @@ -29,15 +28,6 @@ if TYPE_CHECKING: pytestmark = [pytest.mark.django_db, pytest.mark.usefixtures("_search_index")] -@pytest.fixture -def indexed_document() -> Document: - from documents.search import get_backend - - doc = DocumentFactory.create(title="quarterly invoice", content="acme corp") - get_backend().add_or_update(doc) - return doc - - class TestSearchQueryErrorStillBecomesA400: def test_search_query_error_becomes_a_400_naming_the_field( self, diff --git a/src/documents/tests/test_api_search_query_length.py b/src/documents/tests/test_api_search_query_length.py index 9ebc34aeb..2d52bc9ff 100644 --- a/src/documents/tests/test_api_search_query_length.py +++ b/src/documents/tests/test_api_search_query_length.py @@ -22,7 +22,6 @@ import pytest from rest_framework import status import documents.search._backend -from documents.tests.factories import DocumentFactory from documents.views import _MAX_QUERY_LENGTH if TYPE_CHECKING: @@ -33,15 +32,6 @@ if TYPE_CHECKING: pytestmark = [pytest.mark.django_db, pytest.mark.usefixtures("_search_index")] -@pytest.fixture -def indexed_document() -> Document: - from documents.search import get_backend - - doc = DocumentFactory.create(title="quarterly invoice", content="acme corp") - get_backend().add_or_update(doc) - return doc - - class TestGetSearchEndpointEnforcesTheCap: def test_query_one_over_the_cap_is_a_400( self, diff --git a/src/documents/tests/test_api_search_unterminated_date_range.py b/src/documents/tests/test_api_search_unterminated_date_range.py index 0019de1b6..f8c26d743 100644 --- a/src/documents/tests/test_api_search_unterminated_date_range.py +++ b/src/documents/tests/test_api_search_unterminated_date_range.py @@ -19,8 +19,6 @@ from typing import TYPE_CHECKING import pytest from rest_framework import status -from documents.tests.factories import DocumentFactory - if TYPE_CHECKING: from rest_framework.test import APIClient @@ -29,15 +27,6 @@ if TYPE_CHECKING: pytestmark = [pytest.mark.django_db, pytest.mark.usefixtures("_search_index")] -@pytest.fixture -def indexed_document() -> Document: - from documents.search import get_backend - - doc = DocumentFactory.create(title="quarterly invoice", content="acme corp") - get_backend().add_or_update(doc) - return doc - - class TestUnterminatedBracketReturnsA400: @pytest.mark.parametrize( "query",