mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-08-27 05:03:20 +00:00
test(search): pin currently-unasserted behaviours (Task 10 Add list)
Adds dedicated result-level tests, each indexing real documents and asserting on matched-ID sets rather than parse shape: - Deferred `-term` negation (G1): bare `-taxes` requires the term, and fielded `-title:alpha` drops the negation entirely, both matching v2. - Reversed date ranges: absolute bounds swap (whoosh parity); relative `now±` bounds day-bump instead, an inconsistency pinned as a whoosh-compat follow-up rather than "fixed" locally. - The six whoosh unit abbreviations (yrs/mos/wks/hrs/mins/secs). - Unterminated `[` date range brackets 400 at the API level. - `tag:foo,bar` comma value lists are conjunctive, with a decoy proving `correspondent:foo,bar` is not treated as a list. - Date keyword phrases (`today`) honour the active (non-UTC) timezone, not just relative ranges. - `_DEFAULT_SEARCH_FIELDS` stays a subset of PUBLIC_FIELDS. - Every declared JSON subpath is actually written at index time. Also replaces test_schema.py::TestFastFlagAgreement's tantivy-py __reduce__() pickling probe with one built on the paperless-owned, tantivy-independent field_descriptors(). Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
b3e3d8b23e
commit
2d2dad0e1a
@@ -0,0 +1,139 @@
|
||||
"""Comma-separated value lists, at result level.
|
||||
|
||||
``tag:foo,bar`` is a value list only for fields that opt into
|
||||
``comma_values`` (``tag`` is the only one today). Nothing in the tree runs
|
||||
a ``tag:`` fielded query against real documents; this covers foo-only,
|
||||
bar-only and both-tags documents against a real index, plus a decoy
|
||||
proving a non-comma_values field (``correspondent``) does NOT treat a
|
||||
comma as a list separator -- ``correspondent:foo,bar`` searches for the
|
||||
literal text "foo,bar", so a correspondent literally named that way
|
||||
matches while foo-only/bar-only correspondents do not.
|
||||
|
||||
The list semantics are AND (a document must carry every listed value),
|
||||
not OR: a foo-only or bar-only document does not match ``tag:foo,bar``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
from documents.models import Correspondent
|
||||
from documents.models import Document
|
||||
from documents.models import Tag
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from documents.search._backend import TantivyBackend
|
||||
|
||||
pytestmark = [pytest.mark.search, pytest.mark.django_db]
|
||||
|
||||
|
||||
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
|
||||
return set(backend.search_ids(query, user=None))
|
||||
|
||||
|
||||
class TestTagCommaValueListIsConjunctive:
|
||||
@pytest.fixture
|
||||
def docs(self, backend: TantivyBackend) -> dict[str, int]:
|
||||
foo = Tag.objects.create(name="foo")
|
||||
bar = Tag.objects.create(name="bar")
|
||||
baz = Tag.objects.create(name="baz")
|
||||
|
||||
foo_only = Document.objects.create(
|
||||
title="Foo only",
|
||||
content="x",
|
||||
checksum="comma-tag-foo-only",
|
||||
)
|
||||
foo_only.tags.set([foo])
|
||||
backend.add_or_update(foo_only)
|
||||
|
||||
bar_only = Document.objects.create(
|
||||
title="Bar only",
|
||||
content="x",
|
||||
checksum="comma-tag-bar-only",
|
||||
)
|
||||
bar_only.tags.set([bar])
|
||||
backend.add_or_update(bar_only)
|
||||
|
||||
both = Document.objects.create(
|
||||
title="Both tags",
|
||||
content="x",
|
||||
checksum="comma-tag-both",
|
||||
)
|
||||
both.tags.set([foo, bar])
|
||||
backend.add_or_update(both)
|
||||
|
||||
neither = Document.objects.create(
|
||||
title="Neither tag",
|
||||
content="x",
|
||||
checksum="comma-tag-neither",
|
||||
)
|
||||
neither.tags.set([baz])
|
||||
backend.add_or_update(neither)
|
||||
|
||||
return {
|
||||
"foo_only": foo_only.pk,
|
||||
"bar_only": bar_only.pk,
|
||||
"both": both.pk,
|
||||
"neither": neither.pk,
|
||||
}
|
||||
|
||||
def test_only_the_document_carrying_both_tags_matches(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
docs: dict[str, int],
|
||||
) -> None:
|
||||
assert _matched_ids(backend, "tag:foo,bar") == {docs["both"]}
|
||||
|
||||
def test_foo_only_document_does_not_match(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
docs: dict[str, int],
|
||||
) -> None:
|
||||
assert docs["foo_only"] not in _matched_ids(backend, "tag:foo,bar")
|
||||
|
||||
def test_bar_only_document_does_not_match(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
docs: dict[str, int],
|
||||
) -> None:
|
||||
assert docs["bar_only"] not in _matched_ids(backend, "tag:foo,bar")
|
||||
|
||||
|
||||
class TestNonCommaValuesFieldTreatsCommaAsLiteralText:
|
||||
def test_correspondent_comma_is_not_a_value_list(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
) -> None:
|
||||
literal = Correspondent.objects.create(name="foo,bar")
|
||||
literal_doc = Document.objects.create(
|
||||
title="Literal correspondent",
|
||||
content="x",
|
||||
checksum="comma-correspondent-literal",
|
||||
correspondent=literal,
|
||||
)
|
||||
backend.add_or_update(literal_doc)
|
||||
|
||||
foo_correspondent = Correspondent.objects.create(name="foo")
|
||||
foo_doc = Document.objects.create(
|
||||
title="Foo correspondent",
|
||||
content="x",
|
||||
checksum="comma-correspondent-foo",
|
||||
correspondent=foo_correspondent,
|
||||
)
|
||||
backend.add_or_update(foo_doc)
|
||||
|
||||
bar_correspondent = Correspondent.objects.create(name="bar")
|
||||
bar_doc = Document.objects.create(
|
||||
title="Bar correspondent",
|
||||
content="x",
|
||||
checksum="comma-correspondent-bar",
|
||||
correspondent=bar_correspondent,
|
||||
)
|
||||
backend.add_or_update(bar_doc)
|
||||
|
||||
# A value-list reading would match foo_doc and/or bar_doc (an OR)
|
||||
# or neither (an AND, since no single correspondent carries both
|
||||
# values). Either way it would NOT match literal_doc alone.
|
||||
assert _matched_ids(backend, "correspondent:foo,bar") == {literal_doc.pk}
|
||||
@@ -0,0 +1,120 @@
|
||||
"""Pins the current, deferred-negation behavior of a leading ``-``.
|
||||
|
||||
Negation via a bare ``-`` prefix (as opposed to the ``NOT`` keyword) is
|
||||
deferred to the companion plan's G1 item. Until G1 lands, a leading ``-``
|
||||
is not negation at all:
|
||||
|
||||
- Unfielded (``-taxes``): the separator is stripped at index time, so the
|
||||
term becomes an ordinary, *required* positive match on ``taxes`` -- the
|
||||
exact inverse of what a user typing ``-taxes`` to exclude a term intends.
|
||||
- Fielded (``-title:alpha``): the leading hyphen detaches from the field
|
||||
clause entirely (the parse tree emits it as its own token), and the
|
||||
field clause itself is left as an ordinary positive match. The
|
||||
negation is dropped, not inverted.
|
||||
|
||||
If G1 changes either of these, the corresponding test below inverts --
|
||||
that inversion is the intended, visible signal that G1 landed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
from documents.models import Document
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from documents.search._backend import TantivyBackend
|
||||
|
||||
pytestmark = [pytest.mark.search, pytest.mark.django_db]
|
||||
|
||||
|
||||
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
|
||||
return set(backend.search_ids(query, user=None))
|
||||
|
||||
|
||||
def _index(backend: TantivyBackend, **kwargs: object) -> Document:
|
||||
doc = Document.objects.create(**kwargs)
|
||||
backend.add_or_update(doc)
|
||||
return doc
|
||||
|
||||
|
||||
class TestUnfieldedDashBecomesARequiredTerm:
|
||||
@pytest.fixture
|
||||
def docs(self, backend: TantivyBackend) -> dict[str, int]:
|
||||
return {
|
||||
"with_taxes": _index(
|
||||
backend,
|
||||
title="Invoice one",
|
||||
content="invoice taxes included",
|
||||
checksum="dash-unfielded-with",
|
||||
).pk,
|
||||
"without_taxes": _index(
|
||||
backend,
|
||||
title="Invoice two",
|
||||
content="invoice ordinary contents",
|
||||
checksum="dash-unfielded-without",
|
||||
).pk,
|
||||
}
|
||||
|
||||
def test_dash_prefixed_term_matches_only_the_document_containing_it(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
docs: dict[str, int],
|
||||
) -> None:
|
||||
# If this were real negation, it would match "without_taxes" (the
|
||||
# document that does NOT contain "taxes"). It instead matches
|
||||
# "with_taxes" -- the document that DOES.
|
||||
assert _matched_ids(backend, "invoice -taxes") == {docs["with_taxes"]}
|
||||
|
||||
def test_bare_dash_prefixed_term_alone_is_a_positive_search(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
docs: dict[str, int],
|
||||
) -> None:
|
||||
assert _matched_ids(backend, "-taxes") == {docs["with_taxes"]}
|
||||
|
||||
|
||||
class TestFieldedDashDropsTheNegation:
|
||||
@pytest.fixture
|
||||
def docs(self, backend: TantivyBackend) -> dict[str, int]:
|
||||
return {
|
||||
"alpha": _index(
|
||||
backend,
|
||||
title="Alpha Title",
|
||||
content="alpha body",
|
||||
checksum="dash-fielded-alpha",
|
||||
).pk,
|
||||
"beta": _index(
|
||||
backend,
|
||||
title="Beta Title",
|
||||
content="beta body",
|
||||
checksum="dash-fielded-beta",
|
||||
).pk,
|
||||
}
|
||||
|
||||
def test_dash_fielded_term_matches_the_field_value_it_names(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
docs: dict[str, int],
|
||||
) -> None:
|
||||
# Real negation would exclude the "beta" document. Dropped
|
||||
# negation instead matches it, identically to "title:beta".
|
||||
assert _matched_ids(backend, "-title:beta") == {docs["beta"]}
|
||||
assert _matched_ids(backend, "title:beta") == _matched_ids(
|
||||
backend,
|
||||
"-title:beta",
|
||||
)
|
||||
|
||||
def test_dash_fielded_term_conjoined_still_requires_both_sides(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
docs: dict[str, int],
|
||||
) -> None:
|
||||
# Discriminating shape: if the dash-fielded clause were dropped
|
||||
# from the query entirely (rather than kept as a positive
|
||||
# requirement), this would match "alpha" alone. Because the
|
||||
# dropped negation still leaves an ordinary AND-ed requirement
|
||||
# behind, and no document has both titles, nothing matches.
|
||||
assert _matched_ids(backend, "title:alpha -title:beta") == set()
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Date keyword phrases (``today``, etc.) resolved in a non-UTC timezone,
|
||||
end to end.
|
||||
|
||||
paperless's own ``tz=get_current_timezone()`` plumbing
|
||||
(``TantivyBackend._parse_query``) is exercised elsewhere only for
|
||||
relative *ranges* (``added:[-1 week to now]``, in
|
||||
documents/tests/test_api_search.py). This covers a date *keyword*
|
||||
(``today``), whose day boundary depends on the active timezone the same
|
||||
way but goes through whoosh-compat's DateParserPlugin resolution instead
|
||||
of an explicit range.
|
||||
|
||||
Discriminating shape: frozen at 2026-06-15T02:00 UTC, which is
|
||||
2026-06-14T22:00 in America/New_York -- still "today" (06-14) there, but
|
||||
already "today" (06-15) in UTC. Two documents pin both directions of the
|
||||
mistake a hardcoded-UTC bug would make:
|
||||
|
||||
- ``in_ny_today`` (added 2026-06-14T20:00 UTC = 2026-06-14T16:00 NY) is
|
||||
inside New York's "today" window and outside a naive UTC-calendar-day
|
||||
window. A ``tz``-ignoring bug would miss it.
|
||||
- ``in_utc_calendar_day_only`` (added 2026-06-15T10:00 UTC =
|
||||
2026-06-15T06:00 NY) is inside a naive UTC-calendar-day window but
|
||||
outside New York's actual "today" window. A ``tz``-ignoring bug would
|
||||
wrongly match it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
import time_machine
|
||||
|
||||
from documents.models import Document
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pytest_django.fixtures import SettingsWrapper
|
||||
|
||||
from documents.search._backend import TantivyBackend
|
||||
|
||||
pytestmark = [pytest.mark.search, pytest.mark.django_db]
|
||||
|
||||
FROZEN_NOW = datetime(2026, 6, 15, 2, 0, tzinfo=UTC)
|
||||
|
||||
|
||||
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
|
||||
return set(backend.search_ids(query, user=None))
|
||||
|
||||
|
||||
def _index(backend: TantivyBackend, **kwargs: object) -> Document:
|
||||
doc = Document.objects.create(**kwargs)
|
||||
backend.add_or_update(doc)
|
||||
return doc
|
||||
|
||||
|
||||
class TestDateKeywordUsesTheActiveTimezone:
|
||||
def test_today_matches_the_new_york_calendar_day_not_the_utc_one(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
settings: SettingsWrapper,
|
||||
) -> None:
|
||||
settings.TIME_ZONE = "America/New_York"
|
||||
with time_machine.travel(FROZEN_NOW, tick=False):
|
||||
in_ny_today = _index(
|
||||
backend,
|
||||
title="NY today",
|
||||
content="x",
|
||||
checksum="tz-keyword-ny-today",
|
||||
added=datetime(2026, 6, 14, 20, 0, tzinfo=UTC),
|
||||
)
|
||||
# Not captured: the exact-set assertion below already proves
|
||||
# this document (inside a naive UTC-calendar-day window, but
|
||||
# outside New York's actual "today") does not match.
|
||||
_index(
|
||||
backend,
|
||||
title="UTC calendar day only",
|
||||
content="x",
|
||||
checksum="tz-keyword-utc-calendar-day-only",
|
||||
added=datetime(2026, 6, 15, 10, 0, tzinfo=UTC),
|
||||
)
|
||||
|
||||
assert _matched_ids(backend, "added:today") == {in_ny_today.pk}
|
||||
@@ -0,0 +1,20 @@
|
||||
"""``_DEFAULT_SEARCH_FIELDS`` must stay a subset of the registered public
|
||||
field names.
|
||||
|
||||
Nothing enforced this before: a rename in PUBLIC_FIELDS not mirrored in
|
||||
``_DEFAULT_SEARCH_FIELDS`` (documents/search/_query.py) would 400 every
|
||||
unfielded search at request time, since ``index.parse_query`` and the
|
||||
fuzzy/CJK clause builders are handed a field name the schema no longer
|
||||
has.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from documents.search._fields import PUBLIC_FIELDS
|
||||
from documents.search._query import _DEFAULT_SEARCH_FIELDS
|
||||
|
||||
|
||||
class TestDefaultSearchFieldsAreRegistered:
|
||||
def test_every_default_search_field_is_a_public_field(self) -> None:
|
||||
public_field_names = {f.name for f in PUBLIC_FIELDS}
|
||||
assert set(_DEFAULT_SEARCH_FIELDS) <= public_field_names
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Every declared JSON subpath must actually be written to the index.
|
||||
|
||||
PUBLIC_FIELDS declares each JSON field's subpaths (e.g. ``notes`` ->
|
||||
{"user", "note"}), but nothing coupled that declaration to what
|
||||
``_backend.py``'s document builder actually writes into the JSON blob at
|
||||
index time. A subpath declared but never written would be
|
||||
queryable-but-always-empty -- syntactically valid, silently matching
|
||||
nothing -- with no test failure anywhere.
|
||||
|
||||
This indexes one real document carrying values for every JSON field
|
||||
(a Note, a CustomFieldInstance) and inspects the document's own stored
|
||||
JSON payload, rather than running field-specific queries: that way a
|
||||
future JSON field's subpaths are covered automatically, without a new
|
||||
per-subpath query having to be added by hand each time.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
import tantivy
|
||||
from django.contrib.auth.models import User
|
||||
from whoosh_compat import FieldKind
|
||||
|
||||
from documents.models import CustomField
|
||||
from documents.models import CustomFieldInstance
|
||||
from documents.models import Document
|
||||
from documents.models import Note
|
||||
from documents.search._fields import PUBLIC_FIELDS
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from documents.search._backend import TantivyBackend
|
||||
|
||||
pytestmark = [pytest.mark.search, pytest.mark.django_db]
|
||||
|
||||
|
||||
class TestJsonSubpathsAreWrittenAtIndexTime:
|
||||
def test_every_declared_json_subpath_appears_in_the_stored_document(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
) -> None:
|
||||
user = User.objects.create_user(username="completeness-user")
|
||||
field = CustomField.objects.create(
|
||||
name="Completeness Field",
|
||||
data_type=CustomField.FieldDataType.STRING,
|
||||
)
|
||||
doc = Document.objects.create(
|
||||
title="Completeness doc",
|
||||
content="x",
|
||||
checksum="json-subpath-completeness",
|
||||
)
|
||||
Note.objects.create(document=doc, user=user, note="a note")
|
||||
CustomFieldInstance.objects.create(
|
||||
document=doc,
|
||||
field=field,
|
||||
value_text="a value",
|
||||
)
|
||||
backend.add_or_update(doc)
|
||||
|
||||
index = backend._index
|
||||
searcher = index.searcher()
|
||||
hits = searcher.search(
|
||||
tantivy.Query.term_query(index.schema, "id", doc.pk),
|
||||
limit=1,
|
||||
).hits
|
||||
assert hits, "the document was not indexed"
|
||||
stored = searcher.doc(hits[0][1]).to_dict()
|
||||
|
||||
json_fields = [f for f in PUBLIC_FIELDS if f.kind is FieldKind.JSON]
|
||||
assert json_fields, "no JSON fields declared - fixture is stale"
|
||||
for field_spec in json_fields:
|
||||
stored_values = stored.get(field_spec.name)
|
||||
assert stored_values, (
|
||||
f"{field_spec.name} was not written to the index at all"
|
||||
)
|
||||
written_keys = stored_values[0].keys()
|
||||
for subpath in field_spec.subpaths:
|
||||
assert subpath in written_keys, (
|
||||
f"{field_spec.name}.{subpath} is declared in PUBLIC_FIELDS "
|
||||
"but _backend.py's document builder never writes it - it "
|
||||
"would be queryable but always empty"
|
||||
)
|
||||
@@ -0,0 +1,143 @@
|
||||
"""Reversed date ranges: an internal inconsistency between relative and
|
||||
absolute bounds, pinned exactly as measured rather than "fixed" here.
|
||||
|
||||
Measured end to end (FROZEN_NOW = 2026-06-15T12:00:00Z):
|
||||
|
||||
added:[now-1h to now+1h] -> 2h window, correct order
|
||||
added:[now+1h to now-1h] -> ~22h window, DAY-BUMPED, not swapped
|
||||
added:[2020-01-01 to 2019-01-01] -> 366 days, SWAPPED to the forward order
|
||||
added:[2019-01-01 to 2020-01-01] -> 366 days (same result either way)
|
||||
|
||||
Absolute reversed ranges swap their bounds back into order, matching
|
||||
whoosh's own behavior. Relative (``now±``) reversed ranges do not swap --
|
||||
whoosh-compat's date grammar instead adds a day to the upper bound,
|
||||
producing a much wider window than either the forward or a swapped
|
||||
reading would give. This is a library-level inconsistency between the two
|
||||
range kinds, not an application-level rewrite paperless performs (there is
|
||||
no reversed-range handling in documents/search/_query.py), so it is not
|
||||
"fixed" here: fixing it belongs in whoosh-compat's date grammar, not in a
|
||||
pre-parse rewrite on this side (see also the CJK/pre-parse-rewrites
|
||||
docstrings elsewhere in this suite for the same discipline). Pinned as a
|
||||
known inconsistency and a library follow-up. When whoosh-compat's date
|
||||
grammar is fixed to swap consistently, the relative case's test below
|
||||
inverts -- that inversion is the intended, visible signal.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
import time_machine
|
||||
|
||||
from documents.models import Document
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from documents.search._backend import TantivyBackend
|
||||
|
||||
pytestmark = [pytest.mark.search, pytest.mark.django_db]
|
||||
|
||||
FROZEN_NOW = datetime(2026, 6, 15, 12, 0, tzinfo=UTC)
|
||||
|
||||
|
||||
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
|
||||
return set(backend.search_ids(query, user=None))
|
||||
|
||||
|
||||
def _index(backend: TantivyBackend, **kwargs: object) -> Document:
|
||||
doc = Document.objects.create(**kwargs)
|
||||
backend.add_or_update(doc)
|
||||
return doc
|
||||
|
||||
|
||||
class TestRelativeReversedRangeDayBumpsInsteadOfSwapping:
|
||||
@pytest.fixture
|
||||
def docs(self, backend: TantivyBackend) -> dict[str, int]:
|
||||
with time_machine.travel(FROZEN_NOW, tick=False):
|
||||
return {
|
||||
# Inside the correct (forward) 2h window [11:00, 13:00],
|
||||
# outside the day-bumped window [13:00, next-day 11:00).
|
||||
"in_forward_window": _index(
|
||||
backend,
|
||||
title="Forward window doc",
|
||||
content="x",
|
||||
checksum="reversed-relative-forward",
|
||||
added=FROZEN_NOW,
|
||||
).pk,
|
||||
# Outside the correct 2h window, inside the day-bumped
|
||||
# window the reversed query actually produces.
|
||||
"in_daybumped_window": _index(
|
||||
backend,
|
||||
title="Day-bumped window doc",
|
||||
content="x",
|
||||
checksum="reversed-relative-daybumped",
|
||||
added=datetime(2026, 6, 16, 8, 0, tzinfo=UTC),
|
||||
).pk,
|
||||
}
|
||||
|
||||
def test_forward_range_matches_only_the_two_hour_window(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
docs: dict[str, int],
|
||||
) -> None:
|
||||
with time_machine.travel(FROZEN_NOW, tick=False):
|
||||
assert _matched_ids(backend, "added:[now-1h to now+1h]") == {
|
||||
docs["in_forward_window"],
|
||||
}
|
||||
|
||||
def test_reversed_range_day_bumps_rather_than_swapping(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
docs: dict[str, int],
|
||||
) -> None:
|
||||
# If this swapped like the absolute case below, it would match
|
||||
# "in_forward_window" (the same set as the forward query). It
|
||||
# instead matches only the day-bumped document -- the documented
|
||||
# library inconsistency.
|
||||
with time_machine.travel(FROZEN_NOW, tick=False):
|
||||
assert _matched_ids(backend, "added:[now+1h to now-1h]") == {
|
||||
docs["in_daybumped_window"],
|
||||
}
|
||||
|
||||
|
||||
class TestAbsoluteReversedRangeSwaps:
|
||||
@pytest.fixture
|
||||
def docs(self, backend: TantivyBackend) -> dict[str, int]:
|
||||
return {
|
||||
"in_range": _index(
|
||||
backend,
|
||||
title="In-range doc",
|
||||
content="x",
|
||||
checksum="reversed-absolute-in-range",
|
||||
added=datetime(2019, 6, 1, tzinfo=UTC),
|
||||
).pk,
|
||||
"out_of_range": _index(
|
||||
backend,
|
||||
title="Out-of-range doc",
|
||||
content="x",
|
||||
checksum="reversed-absolute-out-of-range",
|
||||
added=datetime(2018, 6, 1, tzinfo=UTC),
|
||||
).pk,
|
||||
}
|
||||
|
||||
def test_forward_range_matches_the_in_range_document(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
docs: dict[str, int],
|
||||
) -> None:
|
||||
assert _matched_ids(
|
||||
backend,
|
||||
"added:[2019-01-01 to 2020-01-01]",
|
||||
) == {docs["in_range"]}
|
||||
|
||||
def test_reversed_range_swaps_to_match_the_same_document(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
docs: dict[str, int],
|
||||
) -> None:
|
||||
assert _matched_ids(
|
||||
backend,
|
||||
"added:[2020-01-01 to 2019-01-01]",
|
||||
) == {docs["in_range"]}
|
||||
@@ -11,6 +11,7 @@ import tantivy
|
||||
from documents.search._fields import PUBLIC_FIELDS
|
||||
from documents.search._schema import SCHEMA_VERSION
|
||||
from documents.search._schema import build_schema
|
||||
from documents.search._schema import field_descriptors
|
||||
from documents.search._schema import needs_rebuild
|
||||
from documents.search._schema import schema_fingerprint
|
||||
from documents.search._tokenizer import register_tokenizers
|
||||
@@ -145,12 +146,16 @@ class TestFastFlagAgreement:
|
||||
# pins the agreement for EVERY kind: a future fast=True
|
||||
# TEXT/KEYWORD/JSON entry the builder silently ignores fails here
|
||||
# instead of at a user's query.
|
||||
schema_fast = {
|
||||
name: bool(field["options"].get("fast", False))
|
||||
for name, field in _schema_fields(build_schema()).items()
|
||||
}
|
||||
#
|
||||
# field_descriptors() (not tantivy-py's __reduce__() pickling
|
||||
# internals) is used as the probe here: it is exactly the input
|
||||
# build_schema()'s SchemaBuilder consumes for the `fast` kwarg on
|
||||
# every field kind, so it pins the same agreement without depending
|
||||
# on a private pickled representation surviving a tantivy-py
|
||||
# upgrade.
|
||||
descriptor_fast = {d.name: d.fast for d in field_descriptors()}
|
||||
for public_field in PUBLIC_FIELDS:
|
||||
assert schema_fast[public_field.name] == public_field.fast, (
|
||||
f"{public_field.name}: PUBLIC_FIELDS says fast={public_field.fast} but the"
|
||||
f" built schema says fast={schema_fast[public_field.name]}"
|
||||
assert descriptor_fast[public_field.name] == public_field.fast, (
|
||||
f"{public_field.name}: PUBLIC_FIELDS says fast={public_field.fast} but"
|
||||
f" field_descriptors() says fast={descriptor_fast[public_field.name]}"
|
||||
)
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
"""Whoosh-style relative date unit abbreviations: yrs/mos/wks/hrs/mins/secs.
|
||||
|
||||
Zero assertions existed anywhere in this tree before this file -- the old
|
||||
v2-era ``TestWhooshUnitAbbreviations`` is gone, and whoosh-compat's own
|
||||
test corpus only carries these spellings as allowlisted (assertion-
|
||||
inverted) lines, i.e. lines it knows do not fully match its own grammar's
|
||||
documented behavior yet accepts anyway. This is the exact class of
|
||||
regression #13482 was: a spelling silently stops parsing (or silently
|
||||
stops matching) with nothing anywhere to notice.
|
||||
|
||||
Each abbreviation produces a relative, zero-width instant
|
||||
(``now - N<unit>`` .. same instant), not a span -- pinned separately in
|
||||
the date-keyword tests. What matters here is that each of the six
|
||||
spellings resolves to the *correct* instant, checked by indexing one
|
||||
document at exactly that instant per unit: a wrong offset (an "hrs" typo
|
||||
that resolves as minutes, say) lands on nothing, or on a sibling
|
||||
document's instant, rather than passing by accident.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
import time_machine
|
||||
|
||||
from documents.models import Document
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from documents.search._backend import TantivyBackend
|
||||
|
||||
pytestmark = [pytest.mark.search, pytest.mark.django_db]
|
||||
|
||||
FROZEN_NOW = datetime(2026, 6, 15, 12, 0, tzinfo=UTC)
|
||||
|
||||
|
||||
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
|
||||
return set(backend.search_ids(query, user=None))
|
||||
|
||||
|
||||
def _index(backend: TantivyBackend, **kwargs: object) -> Document:
|
||||
doc = Document.objects.create(**kwargs)
|
||||
backend.add_or_update(doc)
|
||||
return doc
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def unit_documents(backend: TantivyBackend) -> dict[str, int]:
|
||||
"""One document at the exact instant each abbreviation should resolve
|
||||
to, all indexed together so a wrong offset lands on the wrong (or no)
|
||||
document rather than passing coincidentally."""
|
||||
with time_machine.travel(FROZEN_NOW, tick=False):
|
||||
return {
|
||||
"yrs": _index(
|
||||
backend,
|
||||
title="Years doc",
|
||||
content="x",
|
||||
checksum="unit-abbrev-yrs",
|
||||
added=datetime(2024, 6, 15, 12, 0, tzinfo=UTC),
|
||||
).pk,
|
||||
"mos": _index(
|
||||
backend,
|
||||
title="Months doc",
|
||||
content="x",
|
||||
checksum="unit-abbrev-mos",
|
||||
added=datetime(2026, 3, 15, 12, 0, tzinfo=UTC),
|
||||
).pk,
|
||||
"wks": _index(
|
||||
backend,
|
||||
title="Weeks doc",
|
||||
content="x",
|
||||
checksum="unit-abbrev-wks",
|
||||
added=datetime(2026, 6, 1, 12, 0, tzinfo=UTC),
|
||||
).pk,
|
||||
"hrs": _index(
|
||||
backend,
|
||||
title="Hours doc",
|
||||
content="x",
|
||||
checksum="unit-abbrev-hrs",
|
||||
added=datetime(2026, 6, 15, 7, 0, tzinfo=UTC),
|
||||
).pk,
|
||||
"mins": _index(
|
||||
backend,
|
||||
title="Minutes doc",
|
||||
content="x",
|
||||
checksum="unit-abbrev-mins",
|
||||
added=datetime(2026, 6, 15, 11, 50, tzinfo=UTC),
|
||||
).pk,
|
||||
"secs": _index(
|
||||
backend,
|
||||
title="Seconds doc",
|
||||
content="x",
|
||||
checksum="unit-abbrev-secs",
|
||||
added=datetime(2026, 6, 15, 11, 59, 30, tzinfo=UTC),
|
||||
).pk,
|
||||
}
|
||||
|
||||
|
||||
class TestUnitAbbreviationsResolveToTheCorrectInstant:
|
||||
@pytest.mark.parametrize(
|
||||
("query", "label"),
|
||||
[
|
||||
pytest.param('added:"-2yrs"', "yrs", id="years"),
|
||||
pytest.param('added:"-3mos"', "mos", id="months"),
|
||||
pytest.param('added:"-2wks"', "wks", id="weeks"),
|
||||
pytest.param('added:"-5hrs"', "hrs", id="hours"),
|
||||
pytest.param('added:"-10mins"', "mins", id="minutes"),
|
||||
pytest.param('added:"-30secs"', "secs", id="seconds"),
|
||||
],
|
||||
)
|
||||
def test_quoted_abbreviation_matches_only_its_own_instant(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
unit_documents: dict[str, int],
|
||||
query: str,
|
||||
label: str,
|
||||
) -> None:
|
||||
with time_machine.travel(FROZEN_NOW, tick=False):
|
||||
assert _matched_ids(backend, query) == {unit_documents[label]}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("query", "label"),
|
||||
[
|
||||
pytest.param("added:-2yrs", "yrs", id="years"),
|
||||
pytest.param("added:-3mos", "mos", id="months"),
|
||||
pytest.param("added:-2wks", "wks", id="weeks"),
|
||||
pytest.param("added:-5hrs", "hrs", id="hours"),
|
||||
pytest.param("added:-10mins", "mins", id="minutes"),
|
||||
pytest.param("added:-30secs", "secs", id="seconds"),
|
||||
],
|
||||
)
|
||||
def test_bare_unquoted_abbreviation_matches_only_its_own_instant(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
unit_documents: dict[str, int],
|
||||
query: str,
|
||||
label: str,
|
||||
) -> None:
|
||||
with time_machine.travel(FROZEN_NOW, tick=False):
|
||||
assert _matched_ids(backend, query) == {unit_documents[label]}
|
||||
@@ -0,0 +1,67 @@
|
||||
"""An unterminated ``[`` date range bracket at the API level.
|
||||
|
||||
``created:[2020`` (with or without a dangling ``to <value>``) now raises
|
||||
BAD_DATE and the search endpoint returns HTTP 400, where it used to parse
|
||||
past the missing ``]`` and silently pass the malformed range through.
|
||||
A 400 is correct: malformed input should fail loudly rather than silently
|
||||
matching an unintended query. Pinned at the API level -- the layer a user
|
||||
or client actually sees -- rather than only against the parser directly.
|
||||
|
||||
The properly closed decoy proves the bracket is what matters, not
|
||||
whoosh-compat's date grammar generally: ``created:[2020 to 2021]`` parses
|
||||
and searches cleanly.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
from rest_framework import status
|
||||
|
||||
from documents.tests.factories import DocumentFactory
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from rest_framework.test import APIClient
|
||||
|
||||
from documents.models import Document
|
||||
|
||||
pytestmark = [pytest.mark.django_db, pytest.mark.usefixtures("_search_index")]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def indexed_document() -> Document:
|
||||
from documents.search import get_backend
|
||||
|
||||
doc = DocumentFactory.create(title="quarterly invoice", content="acme corp")
|
||||
get_backend().add_or_update(doc)
|
||||
return doc
|
||||
|
||||
|
||||
class TestUnterminatedBracketReturnsA400:
|
||||
@pytest.mark.parametrize(
|
||||
"query",
|
||||
[
|
||||
pytest.param("created:[2020", id="missing_upper_bound_and_bracket"),
|
||||
pytest.param("created:[2020 to 2021", id="missing_closing_bracket"),
|
||||
],
|
||||
)
|
||||
def test_unterminated_bracket_is_a_400(
|
||||
self,
|
||||
admin_client: APIClient,
|
||||
indexed_document: Document,
|
||||
query: str,
|
||||
) -> None:
|
||||
response = admin_client.get(f"/api/documents/?query={query}")
|
||||
assert response.status_code == status.HTTP_400_BAD_REQUEST
|
||||
assert "created" in str(response.data["query"])
|
||||
|
||||
def test_properly_closed_bracket_still_searches_cleanly(
|
||||
self,
|
||||
admin_client: APIClient,
|
||||
indexed_document: Document,
|
||||
) -> None:
|
||||
response = admin_client.get(
|
||||
"/api/documents/?query=created:[2020 to 2021]",
|
||||
)
|
||||
assert response.status_code == status.HTTP_200_OK
|
||||
Reference in New Issue
Block a user