mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-08-19 09:13:24 +00:00
Trenton Holmes
b48b1a7d08
test_unregistered_id_field_folds_to_literal_text_not_error only checked that parse_user_query() didn't raise for a query like tag_id:5. Add a result-level acceptance test (matching test_acceptance.py's _matched_ids pattern, indexed against real documents) that asserts the matched-document-ID set is genuinely empty, not just that the parse step succeeds.
264 lines
9.3 KiB
Python
264 lines
9.3 KiB
Python
"""Result-level acceptance corpus: real documents indexed via build_schema(),
|
|
real queries run through parse_user_query(), matched-document-ID sets
|
|
asserted — not intermediate ASTs or query strings. This is paperless-ngx's
|
|
analogue of whoosh-compat's own tests/emitter/test_acceptance_e2e.py.
|
|
|
|
Supersedes test_query.py's TestParseUserQuery result-level cases and the
|
|
now-deleted test_date_grammar_parity.py.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import UTC
|
|
from datetime import datetime
|
|
from typing import TYPE_CHECKING
|
|
|
|
import pytest
|
|
|
|
from documents.models import CustomField
|
|
from documents.models import CustomFieldInstance
|
|
from documents.models import Document
|
|
from documents.models import Note
|
|
from documents.models import Tag
|
|
from documents.search._query import parse_user_query
|
|
|
|
if TYPE_CHECKING:
|
|
from documents.search._backend import TantivyBackend
|
|
|
|
pytestmark = [pytest.mark.search, pytest.mark.django_db]
|
|
|
|
FROZEN_NOW = datetime(2026, 6, 15, 12, 0, tzinfo=UTC)
|
|
|
|
|
|
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
|
|
return set(backend.search_ids(query, user=None))
|
|
|
|
|
|
@pytest.fixture
|
|
def indexed_documents(backend: TantivyBackend) -> dict[str, int]:
|
|
"""Index a small fixture set, return {label: doc_id} for corpus queries."""
|
|
docs = {
|
|
"invoice_2020": Document.objects.create(
|
|
title="Invoice 2020",
|
|
content="invoice total due",
|
|
checksum="acc-invoice-2020",
|
|
archive_serial_number=100,
|
|
),
|
|
"invoice_2021": Document.objects.create(
|
|
title="Invoice 2021",
|
|
content="invoice total due",
|
|
checksum="acc-invoice-2021",
|
|
archive_serial_number=101,
|
|
),
|
|
"invoice_2023": Document.objects.create(
|
|
title="Invoice 2023",
|
|
content="invoice total due",
|
|
checksum="acc-invoice-2023",
|
|
archive_serial_number=102,
|
|
),
|
|
"receipt_2022": Document.objects.create(
|
|
title="Receipt 2022",
|
|
content="receipt total due",
|
|
checksum="acc-receipt-2022",
|
|
archive_serial_number=103,
|
|
),
|
|
}
|
|
for doc in docs.values():
|
|
backend.add_or_update(doc)
|
|
return {label: doc.pk for label, doc in docs.items()}
|
|
|
|
|
|
class TestIssue13568BracketWildcard:
|
|
"""paperless-ngx#13568: title:202[0-3]* must keep its character class,
|
|
not fold to a prefix query that silently drops it (whoosh-compat
|
|
DIVERGENCES.md entry 13)."""
|
|
|
|
def test_bracket_class_wildcard_matches_only_in_range_years(
|
|
self,
|
|
backend: TantivyBackend,
|
|
indexed_documents: dict[str, int],
|
|
) -> None:
|
|
# [0-1] (not [0-3]) is deliberate: the fixture's four years are
|
|
# 2020/2021/2022/2023, i.e. their trailing digit is 0/1/2/3
|
|
# respectively - a [0-3] class would match all four and the test
|
|
# would pass even if the character class were silently dropped and
|
|
# folded to an unconstrained "202*" prefix. [0-1] partitions the
|
|
# fixture into a genuine in-range/out-of-range split.
|
|
matched = _matched_ids(backend, "title:202[0-1]*")
|
|
expected = {
|
|
indexed_documents["invoice_2020"],
|
|
indexed_documents["invoice_2021"],
|
|
}
|
|
assert matched == expected, (
|
|
"title:202[0-1]* must match 2020/2021 titles and exclude 2022/2023 "
|
|
"- if this matches everything, the wildcard's character class was "
|
|
"silently dropped (issue #13568's original bug)"
|
|
)
|
|
|
|
|
|
class TestCommaValueLists:
|
|
"""whoosh-compat's CommaValuesPlugin splits `tag:foo,bar` into
|
|
`tag:foo AND tag:bar` (DIVERGENCES.md entries 17/36), matching real
|
|
Whoosh's KEYWORD(commas=True) analyzer-time comma splitting - not an OR
|
|
across the listed values. A document must carry every listed tag to
|
|
match."""
|
|
|
|
def test_tag_comma_list_matches_only_documents_with_both_tags(
|
|
self,
|
|
backend: TantivyBackend,
|
|
) -> None:
|
|
tag_foo = Tag.objects.create(name="foo")
|
|
tag_bar = Tag.objects.create(name="bar")
|
|
tag_baz = Tag.objects.create(name="baz")
|
|
|
|
doc_both = Document.objects.create(
|
|
title="Both",
|
|
content="x",
|
|
checksum="acc-comma-both",
|
|
)
|
|
doc_both.tags.add(tag_foo, tag_bar)
|
|
doc_foo_only = Document.objects.create(
|
|
title="FooOnly",
|
|
content="x",
|
|
checksum="acc-comma-foo",
|
|
)
|
|
doc_foo_only.tags.add(tag_foo)
|
|
doc_other = Document.objects.create(
|
|
title="Other",
|
|
content="x",
|
|
checksum="acc-comma-other",
|
|
)
|
|
doc_other.tags.add(tag_baz)
|
|
for doc in (doc_both, doc_foo_only, doc_other):
|
|
backend.add_or_update(doc)
|
|
matched = _matched_ids(backend, "tag:foo,bar")
|
|
assert matched == {doc_both.pk}
|
|
|
|
|
|
class TestFieldBoosts:
|
|
def test_title_boost_ranks_title_match_above_content_only_match(
|
|
self,
|
|
backend: TantivyBackend,
|
|
) -> None:
|
|
title_match = Document.objects.create(
|
|
title="urgent",
|
|
content="nothing else relevant",
|
|
checksum="acc-boost-title",
|
|
)
|
|
content_match = Document.objects.create(
|
|
title="nothing",
|
|
content="urgent matter here",
|
|
checksum="acc-boost-content",
|
|
)
|
|
backend.add_or_update(title_match)
|
|
backend.add_or_update(content_match)
|
|
query = parse_user_query(backend._index, "urgent", UTC)
|
|
searcher = backend._index.searcher()
|
|
results = searcher.search(query, limit=10)
|
|
ranked_ids = [
|
|
searcher.doc(addr).to_dict()["id"][0] for _score, addr in results.hits
|
|
]
|
|
assert ranked_ids[0] == title_match.pk
|
|
|
|
|
|
class TestJsonSubpaths:
|
|
def test_notes_user_matches_document_with_that_note_author(
|
|
self,
|
|
backend: TantivyBackend,
|
|
) -> None:
|
|
from django.contrib.auth.models import User
|
|
|
|
alice = User.objects.create_user(username="alice")
|
|
doc_with_note = Document.objects.create(
|
|
title="Has note",
|
|
content="x",
|
|
checksum="acc-note-with",
|
|
)
|
|
Note.objects.create(document=doc_with_note, user=alice, note="reminder")
|
|
doc_without = Document.objects.create(
|
|
title="No note",
|
|
content="x",
|
|
checksum="acc-note-without",
|
|
)
|
|
backend.add_or_update(doc_with_note)
|
|
backend.add_or_update(doc_without)
|
|
matched = _matched_ids(backend, "notes.user:alice")
|
|
assert matched == {doc_with_note.pk}
|
|
|
|
def test_custom_fields_name_and_value_combine(
|
|
self,
|
|
backend: TantivyBackend,
|
|
) -> None:
|
|
field = CustomField.objects.create(
|
|
name="Contract Number",
|
|
data_type=CustomField.FieldDataType.STRING,
|
|
)
|
|
other_field = CustomField.objects.create(
|
|
name="Other Field",
|
|
data_type=CustomField.FieldDataType.STRING,
|
|
)
|
|
matching = Document.objects.create(
|
|
title="Matching",
|
|
content="x",
|
|
checksum="acc-cf-matching",
|
|
)
|
|
CustomFieldInstance.objects.create(
|
|
document=matching,
|
|
field=field,
|
|
value_text="policy",
|
|
)
|
|
non_matching = Document.objects.create(
|
|
title="Non-matching",
|
|
content="x",
|
|
checksum="acc-cf-nonmatching",
|
|
)
|
|
CustomFieldInstance.objects.create(
|
|
document=non_matching,
|
|
field=other_field,
|
|
value_text="policy",
|
|
)
|
|
backend.add_or_update(matching)
|
|
backend.add_or_update(non_matching)
|
|
matched = _matched_ids(
|
|
backend,
|
|
'custom_fields.name:"Contract Number" custom_fields.value:policy',
|
|
)
|
|
assert matched == {matching.pk}
|
|
|
|
|
|
class TestMultitokenInNestedOr:
|
|
"""whoosh-compat DIVERGENCES.md entry 15: Multitoken.DEFAULT resolves by
|
|
syntactic enclosing group, not the parser's fixed default group. Prove
|
|
it doesn't matter for paperless's actual data/fields."""
|
|
|
|
def test_multitoken_tag_value_inside_top_level_or_matches_either_branch(
|
|
self,
|
|
backend: TantivyBackend,
|
|
) -> None:
|
|
# "multi word tag" is a multitoken field value; nested inside a
|
|
# top-level OR with an unrelated clause.
|
|
doc_a = Document.objects.create(title="A", content="x", checksum="acc-mt-a")
|
|
doc_a.tags.create(name="multi word tag")
|
|
doc_b = Document.objects.create(title="B", content="x", checksum="acc-mt-b")
|
|
doc_b.tags.create(name="unrelated")
|
|
backend.add_or_update(doc_a)
|
|
backend.add_or_update(doc_b)
|
|
matched = _matched_ids(backend, 'tag:"multi word tag" OR title:B')
|
|
assert matched == {doc_a.pk, doc_b.pk}
|
|
|
|
|
|
class TestUnregisteredIdFieldFoldsToLiteralText:
|
|
"""tag_id, owner_id, etc. are intentionally excluded from the
|
|
FieldRegistry - whoosh-compat parity leniency folds them into a literal
|
|
text search rather than raising a diagnostic/400 (see docs/usage.md's
|
|
advanced-search section). Prove the fold is inert against real data, not
|
|
just that parsing doesn't raise."""
|
|
|
|
def test_tag_id_query_matches_nothing(
|
|
self,
|
|
backend: TantivyBackend,
|
|
indexed_documents: dict[str, int],
|
|
) -> None:
|
|
matched = _matched_ids(backend, "tag_id:5")
|
|
assert matched == set()
|