mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-08-19 09:13:24 +00:00
Compare commits
53
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
86c21826bb | ||
|
|
9e394ed914 | ||
|
|
432c13430a | ||
|
|
289b50a0ad | ||
|
|
5ae9276f1c | ||
|
|
86f3f3ea26 | ||
|
|
92f51da44a | ||
|
|
80fd47656e | ||
|
|
a9c4d505b7 | ||
|
|
2141756435 | ||
|
|
f272b74b18 | ||
|
|
482a1c1780 | ||
|
|
7a9cb8738b | ||
|
|
04602ef4e0 | ||
|
|
ac40b38cad | ||
|
|
f6866828ed | ||
|
|
194c2bce48 | ||
|
|
a37bd5bec4 | ||
|
|
3ae29d9da4 | ||
|
|
2aee7f9c74 | ||
|
|
1cb07030b0 | ||
|
|
a418487f3c | ||
|
|
171b0a6f77 | ||
|
|
7bab9622c8 | ||
|
|
a717684a60 | ||
|
|
970730394e | ||
|
|
9f0e4c6d89 | ||
|
|
0bf06f89a7 | ||
|
|
a66237f614 | ||
|
|
b48b1a7d08 | ||
|
|
eaa6dc1eed | ||
|
|
d779a1437a | ||
|
|
a0de357ae9 | ||
|
|
5942cfb860 | ||
|
|
846c6f88ee | ||
|
|
bb157726c9 | ||
|
|
0353b04f4b | ||
|
|
d7ccff138b | ||
|
|
4577a0a00a | ||
|
|
f6090fe5d4 | ||
|
|
d4e88d2e88 | ||
|
|
bc5360cd6b | ||
|
|
d0e93790ce | ||
|
|
5e8a607d87 | ||
|
|
876db6d744 | ||
|
|
0042b0c4f7 | ||
|
|
93f018b87d | ||
|
|
da7d486ff3 | ||
|
|
abae0c74fb | ||
|
|
35d688be78 | ||
|
|
b26c7fb05f | ||
|
|
a50923c89d | ||
|
|
bb5c2407c6 |
@@ -899,6 +899,22 @@ added:yesterday
|
||||
modified:today
|
||||
```
|
||||
|
||||
Matching by archive metadata:
|
||||
|
||||
```
|
||||
asn:100
|
||||
page_count:12
|
||||
checksum:a1b2c3d4
|
||||
original_filename:invoice.pdf
|
||||
```
|
||||
|
||||
- `asn` matches a document's Archive Serial Number.
|
||||
- `page_count` matches a document's page count.
|
||||
- `checksum` matches the checksum of the original document file (not the
|
||||
archived/processed version).
|
||||
- `original_filename` matches the filename of the document as originally
|
||||
consumed.
|
||||
|
||||
Matching inexact words:
|
||||
|
||||
```
|
||||
@@ -907,6 +923,10 @@ produ*name
|
||||
|
||||
Matching natural date keywords:
|
||||
|
||||
Multi-word date keywords work quoted or unquoted after a date field
|
||||
(`added:"previous month"` and `added:previous month` are equivalent);
|
||||
elsewhere in a query the same words are treated as ordinary search text.
|
||||
|
||||
```
|
||||
added:today
|
||||
modified:yesterday
|
||||
@@ -934,6 +954,7 @@ custom_fields.name:Insurance custom_fields.value:policy
|
||||
- `custom_fields.value` matches against the value of any custom field.
|
||||
- `custom_fields.name` matches the name of the field (use quotes for multi-word names).
|
||||
- Combine both to find documents where a specific named field contains a specific value.
|
||||
- The bare `custom_fields:` prefix is shorthand for `custom_fields.value:`.
|
||||
|
||||
Because separators are stripped during indexing, individual parts of formatted
|
||||
codes are searchable on their own. A value stored as `A-1312/99.50` produces the
|
||||
@@ -961,6 +982,8 @@ notes.note:reminder
|
||||
notes.user:alice notes.note:insurance
|
||||
```
|
||||
|
||||
The bare `notes:` prefix is shorthand for `notes.note:`.
|
||||
|
||||
All of these constructs can be combined as you see fit. If you want to
|
||||
learn more about the query language used by paperless, see the
|
||||
[Tantivy query language documentation](https://docs.rs/tantivy/latest/tantivy/query/struct.QueryParser.html).
|
||||
|
||||
@@ -77,6 +77,7 @@ dependencies = [
|
||||
"torch~=2.13.0",
|
||||
"watchfiles>=1.2",
|
||||
"whitenoise~=6.11",
|
||||
"whoosh-compat[tantivy]",
|
||||
"zxing-cpp~=3.1.0",
|
||||
]
|
||||
[project.optional-dependencies]
|
||||
@@ -166,6 +167,11 @@ psycopg-c = [
|
||||
torch = [
|
||||
{ index = "pytorch-cpu" },
|
||||
]
|
||||
# TODO: switch to a pinned PyPI version once whoosh-compat releases
|
||||
# (expected around this repo's PR 3 in the transition plan); fall back to a
|
||||
# pinned git commit SHA if that release slips. See
|
||||
# docs/superpowers/specs/2026-08-07-whoosh-compat-transition-design.md.
|
||||
whoosh-compat = { path = "../whoosh-compat" }
|
||||
|
||||
[tool.ruff]
|
||||
target-version = "py311"
|
||||
|
||||
@@ -6,13 +6,18 @@ from documents.search._backend import TantivyRelevanceList
|
||||
from documents.search._backend import WriteBatch
|
||||
from documents.search._backend import get_backend
|
||||
from documents.search._backend import reset_backend
|
||||
from documents.search._errors import InvalidDateQuery
|
||||
from documents.search._errors import InvalidNumberQuery
|
||||
from documents.search._errors import MultipleSearchQueryErrors
|
||||
from documents.search._errors import SearchQueryError
|
||||
from documents.search._errors import search_query_error_messages
|
||||
from documents.search._schema import needs_rebuild
|
||||
from documents.search._schema import wipe_index
|
||||
from documents.search._translate import InvalidDateQuery
|
||||
from documents.search._translate import SearchQueryError
|
||||
|
||||
__all__ = [
|
||||
"InvalidDateQuery",
|
||||
"InvalidNumberQuery",
|
||||
"MultipleSearchQueryErrors",
|
||||
"SearchHit",
|
||||
"SearchIndexLockError",
|
||||
"SearchMode",
|
||||
@@ -23,5 +28,6 @@ __all__ = [
|
||||
"get_backend",
|
||||
"needs_rebuild",
|
||||
"reset_backend",
|
||||
"search_query_error_messages",
|
||||
"wipe_index",
|
||||
]
|
||||
|
||||
@@ -25,7 +25,6 @@ from django.utils.timezone import get_current_timezone
|
||||
from guardian.shortcuts import get_groups_with_perms
|
||||
from guardian.shortcuts import get_users_with_perms
|
||||
|
||||
from documents.search._query import build_permission_filter
|
||||
from documents.search._query import extract_cjk_text
|
||||
from documents.search._query import parse_simple_text_highlight_query
|
||||
from documents.search._query import parse_simple_text_query
|
||||
@@ -43,6 +42,7 @@ from documents.utils import QuerySetStream
|
||||
from documents.utils import identity
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable
|
||||
from collections.abc import Iterator
|
||||
from collections.abc import Sequence
|
||||
from pathlib import Path
|
||||
@@ -294,6 +294,47 @@ class WriteBatch:
|
||||
)
|
||||
|
||||
|
||||
def build_permission_filter(
|
||||
schema: tantivy.Schema,
|
||||
user: AbstractUser,
|
||||
viewer_group_ids: Iterable[int] = (),
|
||||
) -> tantivy.Query:
|
||||
"""
|
||||
Build a query filter for user document permissions.
|
||||
|
||||
Creates a query that matches only documents visible to the specified user
|
||||
according to paperless-ngx permission rules:
|
||||
- Public documents (no owner) are visible to all users
|
||||
- Private documents are visible to their owner
|
||||
- Documents explicitly shared with the user are visible
|
||||
- Documents shared with one of the user's current groups are visible
|
||||
|
||||
Args:
|
||||
schema: Tantivy schema for field validation
|
||||
user: User to check permissions for
|
||||
viewer_group_ids: Current group memberships for the user
|
||||
|
||||
Returns:
|
||||
Tantivy query that filters results to visible documents
|
||||
"""
|
||||
owner_any = tantivy.Query.exists_query("owner_id")
|
||||
no_owner = tantivy.Query.boolean_query(
|
||||
[
|
||||
(tantivy.Occur.Must, tantivy.Query.all_query()),
|
||||
(tantivy.Occur.MustNot, owner_any),
|
||||
],
|
||||
)
|
||||
owned = tantivy.Query.term_query(schema, "owner_id", user.pk)
|
||||
shared = tantivy.Query.term_query(schema, "viewer_id", user.pk)
|
||||
group_shared = [
|
||||
tantivy.Query.term_query(schema, "viewer_group_id", group_id)
|
||||
for group_id in viewer_group_ids
|
||||
]
|
||||
return tantivy.Query.disjunction_max_query(
|
||||
[no_owner, owned, shared, *group_shared],
|
||||
)
|
||||
|
||||
|
||||
class TantivyBackend:
|
||||
"""
|
||||
Tantivy search backend with explicit lifecycle management.
|
||||
|
||||
@@ -1,171 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC
|
||||
from datetime import date
|
||||
from datetime import datetime
|
||||
from datetime import timedelta
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import Final
|
||||
|
||||
from dateutil.relativedelta import relativedelta
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from datetime import tzinfo
|
||||
|
||||
_DATE_ONLY_FIELDS = frozenset({"created"})
|
||||
|
||||
_TODAY: Final[str] = "today"
|
||||
_YESTERDAY: Final[str] = "yesterday"
|
||||
_PREVIOUS_WEEK: Final[str] = "previous week"
|
||||
_THIS_MONTH: Final[str] = "this month"
|
||||
_PREVIOUS_MONTH: Final[str] = "previous month"
|
||||
_THIS_YEAR: Final[str] = "this year"
|
||||
_PREVIOUS_YEAR: Final[str] = "previous year"
|
||||
_PREVIOUS_QUARTER: Final[str] = "previous quarter"
|
||||
|
||||
_DATE_KEYWORDS = frozenset(
|
||||
{
|
||||
_TODAY,
|
||||
_YESTERDAY,
|
||||
_PREVIOUS_WEEK,
|
||||
_THIS_MONTH,
|
||||
_PREVIOUS_MONTH,
|
||||
_THIS_YEAR,
|
||||
_PREVIOUS_YEAR,
|
||||
_PREVIOUS_QUARTER,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _fmt(dt: datetime) -> str:
|
||||
"""Format a datetime as an ISO 8601 UTC string for use in Tantivy range queries."""
|
||||
return dt.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def _iso_range(lo: datetime, hi: datetime) -> str:
|
||||
"""
|
||||
Format a half-open ``[lo TO hi)`` range in ISO 8601 for Tantivy query syntax.
|
||||
|
||||
``hi`` is always the exclusive ceiling of a computed period (the start of
|
||||
the *next* day/week/month/quarter/year), so the closing bracket must be
|
||||
the Tantivy exclusive-range brace ``}`` rather than ``]`` — otherwise the
|
||||
first instant of the following period (e.g. the 1st of next month) is
|
||||
incorrectly included in the match.
|
||||
"""
|
||||
return f"[{_fmt(lo)} TO {_fmt(hi)}}}"
|
||||
|
||||
|
||||
def _quarter_start(d: date) -> date:
|
||||
"""Return the first day of the calendar quarter containing ``d``."""
|
||||
return date(d.year, ((d.month - 1) // 3) * 3 + 1, 1)
|
||||
|
||||
|
||||
def _midnight(d: date, tz: tzinfo) -> datetime:
|
||||
"""Convert a calendar date at local-timezone midnight to a UTC datetime."""
|
||||
return datetime(d.year, d.month, d.day, tzinfo=tz).astimezone(UTC)
|
||||
|
||||
|
||||
def _keyword_bounds(keyword: str, tz: tzinfo) -> tuple[date, date]:
|
||||
"""
|
||||
Map a relative date keyword to ``(start, exclusive_end)`` calendar dates.
|
||||
|
||||
``tz`` only determines what "today" is; the caller decides how the returned
|
||||
dates become UTC datetime boundaries (date-only vs. local-midnight offset).
|
||||
"""
|
||||
today = datetime.now(tz).date()
|
||||
if keyword == _TODAY:
|
||||
return today, today + timedelta(days=1)
|
||||
if keyword == _YESTERDAY:
|
||||
return today - timedelta(days=1), today
|
||||
if keyword == _PREVIOUS_WEEK:
|
||||
this_monday = today - timedelta(days=today.weekday())
|
||||
return this_monday - timedelta(weeks=1), this_monday
|
||||
if keyword == _THIS_MONTH:
|
||||
first = today.replace(day=1)
|
||||
return first, first + relativedelta(months=1)
|
||||
if keyword == _PREVIOUS_MONTH:
|
||||
this_first = today.replace(day=1)
|
||||
return this_first - relativedelta(months=1), this_first
|
||||
if keyword == _THIS_YEAR:
|
||||
return date(today.year, 1, 1), date(today.year + 1, 1, 1)
|
||||
if keyword == _PREVIOUS_YEAR:
|
||||
return date(today.year - 1, 1, 1), date(today.year, 1, 1)
|
||||
if keyword == _PREVIOUS_QUARTER:
|
||||
this_quarter = _quarter_start(today)
|
||||
return this_quarter - relativedelta(months=3), this_quarter
|
||||
raise ValueError(f"Unknown keyword: {keyword}")
|
||||
|
||||
|
||||
def _date_only_range(keyword: str, tz: tzinfo) -> str:
|
||||
"""
|
||||
For `created` (DateField): use the local calendar date, converted to
|
||||
midnight UTC boundaries. No offset arithmetic — date only.
|
||||
"""
|
||||
start, end = _keyword_bounds(keyword, tz)
|
||||
lo = datetime(start.year, start.month, start.day, tzinfo=UTC)
|
||||
hi = datetime(end.year, end.month, end.day, tzinfo=UTC)
|
||||
return _iso_range(lo, hi)
|
||||
|
||||
|
||||
def _datetime_range(keyword: str, tz: tzinfo) -> str:
|
||||
"""
|
||||
For `added` / `modified` (DateTimeField, stored as UTC): convert local day
|
||||
boundaries to UTC — full offset arithmetic required.
|
||||
"""
|
||||
start, end = _keyword_bounds(keyword, tz)
|
||||
return _iso_range(_midnight(start, tz), _midnight(end, tz))
|
||||
|
||||
|
||||
def _precision_bounds(digits: str) -> tuple[date, date] | None:
|
||||
"""
|
||||
Map a 4/6/8-digit date token to (start, exclusive_end) calendar dates.
|
||||
|
||||
YYYY -> whole year, YYYYMM -> whole month, YYYYMMDD -> single day.
|
||||
Returns None for any unparsable or out-of-range value (e.g. month 23),
|
||||
so callers can emit a no-match clause instead of erroring (Whoosh parity).
|
||||
"""
|
||||
try:
|
||||
if len(digits) == 4:
|
||||
year = int(digits)
|
||||
return date(year, 1, 1), date(year + 1, 1, 1)
|
||||
if len(digits) == 6:
|
||||
year, month = int(digits[:4]), int(digits[4:6])
|
||||
start = date(year, month, 1)
|
||||
end = date(year + 1, 1, 1) if month == 12 else date(year, month + 1, 1)
|
||||
return start, end
|
||||
if len(digits) == 8:
|
||||
start = date(int(digits[:4]), int(digits[4:6]), int(digits[6:8]))
|
||||
return start, start + timedelta(days=1)
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _utc_bounds_for_field(
|
||||
field: str,
|
||||
start: date,
|
||||
end: date,
|
||||
tz: tzinfo,
|
||||
) -> tuple[datetime, datetime]:
|
||||
"""
|
||||
Convert calendar-date bounds to UTC datetimes per the field's storage type.
|
||||
|
||||
For DateField (``created``) the bounds are UTC midnight (no offset). For
|
||||
DateTimeField (``added``/``modified``) the bounds are local-tz midnight
|
||||
converted to UTC, matching how each field is indexed.
|
||||
"""
|
||||
if field in _DATE_ONLY_FIELDS:
|
||||
return (
|
||||
datetime(start.year, start.month, start.day, tzinfo=UTC),
|
||||
datetime(end.year, end.month, end.day, tzinfo=UTC),
|
||||
)
|
||||
return (
|
||||
datetime(start.year, start.month, start.day, tzinfo=tz).astimezone(UTC),
|
||||
datetime(end.year, end.month, end.day, tzinfo=tz).astimezone(UTC),
|
||||
)
|
||||
|
||||
|
||||
def _field_range_from_dates(field: str, start: date, end: date, tz: tzinfo) -> str:
|
||||
"""Build a Tantivy ``field:[lo TO hi]`` ISO range from calendar-date bounds."""
|
||||
lo, hi = _utc_bounds_for_field(field, start, end, tz)
|
||||
return f"{field}:{_iso_range(lo, hi)}"
|
||||
@@ -0,0 +1,54 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Sequence
|
||||
|
||||
|
||||
class SearchQueryError(ValueError):
|
||||
"""
|
||||
Base for user-fixable search query errors.
|
||||
|
||||
Carries a message safe to surface to the user (no internal details). The
|
||||
view layer catches this and returns an HTTP 400, so any future subclass
|
||||
gets the same treatment.
|
||||
"""
|
||||
|
||||
|
||||
class InvalidDateQuery(SearchQueryError):
|
||||
"""Raised when a date field value or range bound cannot be parsed."""
|
||||
|
||||
def __init__(self, field: str | None, value: str | None) -> None:
|
||||
self.field = field
|
||||
self.value = value
|
||||
super().__init__(f"Invalid date value {value!r} for field {field!r}.")
|
||||
|
||||
|
||||
class InvalidNumberQuery(SearchQueryError):
|
||||
"""Raised when a numeric field value or range bound cannot be parsed."""
|
||||
|
||||
def __init__(self, field: str | None, value: str | None) -> None:
|
||||
self.field = field
|
||||
self.value = value
|
||||
super().__init__(f"Invalid numeric value {value!r} for field {field!r}.")
|
||||
|
||||
|
||||
class MultipleSearchQueryErrors(SearchQueryError):
|
||||
"""Aggregates every user-fixable error from one parse, not just the first."""
|
||||
|
||||
def __init__(self, errors: Sequence[SearchQueryError]) -> None:
|
||||
self.errors = tuple(errors)
|
||||
super().__init__("; ".join(str(e) for e in self.errors))
|
||||
|
||||
|
||||
def search_query_error_messages(e: SearchQueryError) -> list[str]:
|
||||
"""The user-facing message list for a SearchQueryError.
|
||||
|
||||
Every offending value's message, not just the first, so the user can
|
||||
fix them all in one round-trip. Shared by every view that maps
|
||||
SearchQueryError to an HTTP 400.
|
||||
"""
|
||||
if isinstance(e, MultipleSearchQueryErrors):
|
||||
return [str(sub) for sub in e.errors]
|
||||
return [str(e)]
|
||||
@@ -0,0 +1,33 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from whoosh_compat import FieldKind
|
||||
from whoosh_compat import FieldSpec
|
||||
|
||||
# Internal-only schema fields with no query-syntax meaning of their own
|
||||
# (sort shadow fields, bigram CJK fields, simple_title/simple_content,
|
||||
# autocomplete_word, notes_text) are NOT represented here — they stay
|
||||
# hardcoded in _schema.py's build_schema().
|
||||
#
|
||||
# analyzer/pattern_normalizer are deliberately left at FieldSpec's default
|
||||
# (None): they're language-specific and only meaningful to whoosh-compat's
|
||||
# parser, so _registry.py attaches them per-language via dataclasses.replace()
|
||||
# rather than PUBLIC_FIELDS declaring them itself. _schema.py only reads
|
||||
# name/kind/fast and never sees the analyzer at all.
|
||||
PUBLIC_FIELDS: tuple[FieldSpec, ...] = (
|
||||
FieldSpec("title", FieldKind.TEXT),
|
||||
FieldSpec("content", FieldKind.TEXT),
|
||||
FieldSpec("correspondent", FieldKind.TEXT),
|
||||
FieldSpec("document_type", FieldKind.TEXT, aliases=("type",)),
|
||||
FieldSpec("storage_path", FieldKind.TEXT, aliases=("path",)),
|
||||
FieldSpec("original_filename", FieldKind.TEXT),
|
||||
FieldSpec("tag", FieldKind.TEXT, comma_values=True),
|
||||
FieldSpec("checksum", FieldKind.KEYWORD),
|
||||
FieldSpec("asn", FieldKind.U64, fast=True),
|
||||
FieldSpec("page_count", FieldKind.U64, fast=True),
|
||||
FieldSpec("num_notes", FieldKind.U64, fast=True),
|
||||
FieldSpec("created", FieldKind.DATE, date_only=True, fast=True),
|
||||
FieldSpec("modified", FieldKind.DATETIME, fast=True),
|
||||
FieldSpec("added", FieldKind.DATETIME, fast=True),
|
||||
FieldSpec("notes", FieldKind.JSON, subpaths=("user", "note")),
|
||||
FieldSpec("custom_fields", FieldKind.JSON, subpaths=("name", "value")),
|
||||
)
|
||||
+256
-121
@@ -6,18 +6,25 @@ from typing import Final
|
||||
|
||||
import regex
|
||||
import tantivy
|
||||
import whoosh_compat as wc
|
||||
from django.conf import settings
|
||||
from whoosh_compat.emitters.tantivy_ import emit as tantivy_emit
|
||||
from whoosh_compat.errors import Diagnostic
|
||||
from whoosh_compat.errors import DiagnosticKind
|
||||
from whoosh_compat.errors import QueryEmitError
|
||||
from whoosh_compat.errors import UnsupportedQueryError
|
||||
|
||||
from documents.search._errors import InvalidDateQuery
|
||||
from documents.search._errors import InvalidNumberQuery
|
||||
from documents.search._errors import MultipleSearchQueryErrors
|
||||
from documents.search._errors import SearchQueryError
|
||||
from documents.search._fields import PUBLIC_FIELDS
|
||||
from documents.search._registry import get_field_registry
|
||||
from documents.search._tokenizer import simple_search_tokens
|
||||
from documents.search._translate import SearchQueryError
|
||||
from documents.search._translate import translate_query
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable
|
||||
from datetime import tzinfo
|
||||
|
||||
from django.contrib.auth.base_user import AbstractBaseUser
|
||||
|
||||
logger = logging.getLogger("paperless.search")
|
||||
|
||||
# Maximum seconds any single regex substitution may run.
|
||||
@@ -28,6 +35,99 @@ _REGEX_TIMEOUT: Final[float] = 1.0
|
||||
# Uses Unicode properties to cover all blocks including Extension B+ planes.
|
||||
_CJK_RE: Final = regex.compile(r"[\p{Han}\p{Hiragana}\p{Katakana}\p{Hangul}]+")
|
||||
|
||||
# Multi-word date-keyword phrases whoosh-compat only accepts quoted.
|
||||
# Unquoted has always been the honored spelling, so
|
||||
# _quote_date_keyword_phrases below inserts the quotes and nothing else.
|
||||
# Single-word keywords (today, yesterday) already parse unquoted.
|
||||
_DATE_KEYWORD_PHRASES: Final = (
|
||||
"previous week",
|
||||
"previous month",
|
||||
"previous quarter",
|
||||
"previous year",
|
||||
"this month",
|
||||
"this year",
|
||||
)
|
||||
|
||||
# Field names are case-sensitive (matching the parser's own field
|
||||
# tagging); the keyword phrase is case-insensitive (matching the date
|
||||
# grammar's leniency for the quoted form). Date fields derived from
|
||||
# PUBLIC_FIELDS, never hand-listed.
|
||||
_DATE_KEYWORD_PHRASE_RE: Final = regex.compile(
|
||||
r"\b("
|
||||
+ "|".join(
|
||||
regex.escape(f.name)
|
||||
for f in PUBLIC_FIELDS
|
||||
if f.kind in (wc.FieldKind.DATE, wc.FieldKind.DATETIME)
|
||||
)
|
||||
+ r"):((?i:"
|
||||
+ "|".join(_DATE_KEYWORD_PHRASES)
|
||||
+ r"))\b",
|
||||
)
|
||||
|
||||
|
||||
def _quote_date_keyword_phrases(raw_query: str) -> str:
|
||||
"""Quote unquoted multi-word date keyword phrases on date fields.
|
||||
|
||||
``added:previous month`` becomes ``added:"previous month"``; already-
|
||||
quoted spellings, TEXT fields, and standalone words are untouched.
|
||||
Only quoting happens here - every date computation stays in
|
||||
whoosh-compat's grammar.
|
||||
|
||||
Not quote-aware: matches anywhere in raw_query, including inside an
|
||||
existing quoted phrase (e.g. ``title:"see added:previous month
|
||||
notes"`` would get quotes inserted mid-phrase). Accepted as an
|
||||
unlikely-in-practice edge case rather than implementing quote-aware
|
||||
scanning.
|
||||
"""
|
||||
return _DATE_KEYWORD_PHRASE_RE.sub(
|
||||
r'\1:"\2"',
|
||||
raw_query,
|
||||
timeout=_REGEX_TIMEOUT,
|
||||
)
|
||||
|
||||
|
||||
# notes:/custom_fields: were valid fielded searches before this migration.
|
||||
# whoosh-compat's registry only exposes them as JSON subpaths, so a bare
|
||||
# prefix would demote to an unfielded text search. Rewrite live to the
|
||||
# equivalent subpath (notes: -> notes.note:, custom_fields: ->
|
||||
# custom_fields.value:); custom_fields.name: remains available separately.
|
||||
# Not preceded by a word character or dot, so subpath spellings and words
|
||||
# merely ending in the prefix are untouched.
|
||||
_BARE_JSON_PREFIX_RES: Final = (
|
||||
(regex.compile(r"(?<![.\w])notes:(?!\.)"), "notes.note:"),
|
||||
(regex.compile(r"(?<![.\w])custom_fields:(?!\.)"), "custom_fields.value:"),
|
||||
)
|
||||
|
||||
|
||||
def _rewrite_bare_json_field_prefixes(raw_query: str) -> str:
|
||||
"""Rewrite bare ``notes:``/``custom_fields:`` prefixes to their
|
||||
subpath equivalents. Prefix substitution only, values untouched.
|
||||
|
||||
Not quote-aware, same accepted trade-off as
|
||||
_quote_date_keyword_phrases: a literal ``notes:`` inside an existing
|
||||
quoted phrase on an unrelated field would also get rewritten.
|
||||
"""
|
||||
for pattern, replacement in _BARE_JSON_PREFIX_RES:
|
||||
raw_query = pattern.sub(replacement, raw_query, timeout=_REGEX_TIMEOUT)
|
||||
return raw_query
|
||||
|
||||
|
||||
# whoosh-compat's emit() error messages are written for the HOST: they
|
||||
# cite the library's own divergence ledger and give registry-configuration
|
||||
# advice. Neither belongs in a message shown to a searching user.
|
||||
_DIVERGENCE_REF_RE: Final = regex.compile(r"\s*\(DIVERGENCES\.md entry \d+\)")
|
||||
|
||||
|
||||
def _user_facing_emit_message(exc: Exception) -> str:
|
||||
"""A user-safe message for a QueryEmitError/UnsupportedQueryError."""
|
||||
message = _DIVERGENCE_REF_RE.sub("", str(exc))
|
||||
if "fast=True" in message:
|
||||
# The exists-check message advises marking the field fast=True, a
|
||||
# host configuration action; the user just needs to know the
|
||||
# search form is unsupported here.
|
||||
return "existence searches (field:*) are not supported for this field"
|
||||
return message
|
||||
|
||||
|
||||
def _has_cjk(text: str) -> bool:
|
||||
"""Return True if text contains any CJK characters."""
|
||||
@@ -60,65 +160,94 @@ def _build_cjk_query(
|
||||
matches against unrelated Latin text). Returns None when there is no CJK
|
||||
text or the parse fails.
|
||||
"""
|
||||
cjk_text = " ".join(_CJK_RE.findall(raw_query))
|
||||
cjk_text = extract_cjk_text(raw_query)
|
||||
if not cjk_text:
|
||||
return None
|
||||
try:
|
||||
return index.parse_query(cjk_text, fields)
|
||||
except Exception:
|
||||
# Broad on purpose, unlike _try_parse_fuzzy_query's narrower
|
||||
# ValueError: cjk_text isn't filtered to a guaranteed-safe token
|
||||
# set the way the fuzzy blend's word string is, so the exact
|
||||
# failure mode tantivy could raise here isn't pinned down.
|
||||
logger.debug(
|
||||
"Skipping CJK search clause: could not parse CJK text: %r",
|
||||
cjk_text,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def build_permission_filter(
|
||||
schema: tantivy.Schema,
|
||||
user: AbstractBaseUser,
|
||||
viewer_group_ids: Iterable[int] = (),
|
||||
) -> tantivy.Query:
|
||||
# A joined fuzzy word string must stay plain words: any token that could
|
||||
# read as tantivy query grammar (a colon, bracket, quote, operator...) is
|
||||
# dropped rather than escaped. Today's default-field analyzers only emit
|
||||
# word characters, so this never fires; it guards a future field whose
|
||||
# analyzer passes punctuation through (an identity/keyword analyzer).
|
||||
_WORD_TOKEN_RE = regex.compile(r"\w+")
|
||||
|
||||
|
||||
def _try_parse_fuzzy_query(
|
||||
index: tantivy.Index,
|
||||
ast: wc.ast.Node,
|
||||
registry: wc.FieldRegistry,
|
||||
) -> tantivy.Query | None:
|
||||
"""Build the fuzzy blend clause from the parsed query's free-text
|
||||
words, or None if it has none.
|
||||
|
||||
The clause is built by handing tantivy's own query parser a plain
|
||||
word string (there's no clean AST-level fuzzy equivalent to
|
||||
whoosh-compat's parse tree, and fuzzy matching was always an
|
||||
approximate, secondary, 0.1-boosted clause). The words come from
|
||||
whoosh_compat's ``free_text_tokens`` over the already-parsed AST,
|
||||
never from the raw query string: raw whoosh grammar (date keywords,
|
||||
``[2005 to 2009]`` ranges, bracket-class wildcards) is not tantivy
|
||||
syntax, and feeding it here used to knock the fuzzy clause out for
|
||||
the whole query the moment any such construct appeared alongside a
|
||||
typo'd word. The helper also keeps excluded terms out: a ``NOT``'d
|
||||
word must not resurface through the fuzzy clause.
|
||||
|
||||
Chosen trade-off: a term explicitly fielded on one of the default
|
||||
search fields (``correspondent:acme``) contributes its text to the
|
||||
word string UNFIELDED, so the fuzzy clause searches it across all
|
||||
default fields rather than just the one the user named. That is
|
||||
recall-only widening on a secondary 0.1-boosted clause the score
|
||||
threshold already disciplines, accepted in exchange for never feeding
|
||||
field syntax to tantivy's parser.
|
||||
|
||||
The ValueError guard stays as insurance (the word string is plain
|
||||
tokens, so tantivy accepting it is expected, not assumed): on a parse
|
||||
failure the fuzzy clause is skipped and the exact/CJK clauses stand,
|
||||
rather than the whole query failing.
|
||||
"""
|
||||
Build a query filter for user document permissions.
|
||||
|
||||
Creates a query that matches only documents visible to the specified user
|
||||
according to paperless-ngx permission rules:
|
||||
- Public documents (no owner) are visible to all users
|
||||
- Private documents are visible to their owner
|
||||
- Documents explicitly shared with the user are visible
|
||||
- Documents shared with one of the user's current groups are visible
|
||||
|
||||
Args:
|
||||
schema: Tantivy schema for field validation
|
||||
user: User to check permissions for
|
||||
viewer_group_ids: Current group memberships for the user
|
||||
|
||||
Returns:
|
||||
Tantivy query that filters results to visible documents
|
||||
"""
|
||||
owner_any = tantivy.Query.exists_query("owner_id")
|
||||
no_owner = tantivy.Query.boolean_query(
|
||||
[
|
||||
(tantivy.Occur.Must, tantivy.Query.all_query()),
|
||||
(tantivy.Occur.MustNot, owner_any),
|
||||
],
|
||||
)
|
||||
owned = tantivy.Query.term_query(schema, "owner_id", user.pk)
|
||||
shared = tantivy.Query.term_query(schema, "viewer_id", user.pk)
|
||||
group_shared = [
|
||||
tantivy.Query.term_query(schema, "viewer_group_id", group_id)
|
||||
for group_id in viewer_group_ids
|
||||
]
|
||||
return tantivy.Query.disjunction_max_query(
|
||||
[no_owner, owned, shared, *group_shared],
|
||||
)
|
||||
tokens = wc.free_text_tokens(ast, registry=registry, fields=_DEFAULT_SEARCH_FIELDS)
|
||||
words = [t for t in tokens if _WORD_TOKEN_RE.fullmatch(t)]
|
||||
if not words:
|
||||
return None
|
||||
fuzzy_text = " ".join(words)
|
||||
try:
|
||||
return index.parse_query(
|
||||
fuzzy_text,
|
||||
_DEFAULT_SEARCH_FIELDS,
|
||||
field_boosts=_FIELD_BOOSTS,
|
||||
fuzzy_fields={f: (True, 1, True) for f in _DEFAULT_SEARCH_FIELDS},
|
||||
)
|
||||
except ValueError:
|
||||
logger.debug(
|
||||
"Skipping fuzzy search clause: token string is not valid "
|
||||
"tantivy query syntax: %r",
|
||||
fuzzy_text,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
DEFAULT_SEARCH_FIELDS = [
|
||||
_DEFAULT_SEARCH_FIELDS: Final[list[str]] = [
|
||||
"title",
|
||||
"content",
|
||||
"correspondent",
|
||||
"document_type",
|
||||
"tag",
|
||||
]
|
||||
SIMPLE_SEARCH_FIELDS = ["simple_title", "simple_content"]
|
||||
TITLE_SEARCH_FIELDS = ["simple_title"]
|
||||
_SIMPLE_SEARCH_FIELDS: Final[list[str]] = ["simple_title", "simple_content"]
|
||||
_TITLE_SEARCH_FIELDS: Final[list[str]] = ["simple_title"]
|
||||
_CJK_ALL_FIELDS: Final[list[str]] = [
|
||||
"bigram_content",
|
||||
"bigram_title",
|
||||
@@ -132,11 +261,14 @@ _FIELD_BOOSTS = {"title": 2.0}
|
||||
_SIMPLE_FIELD_BOOSTS = {"simple_title": 2.0}
|
||||
|
||||
|
||||
def _simple_query_tokens(raw_query: str) -> list[str]:
|
||||
# Tokenize and fold via the same analyzer used to index simple_title /
|
||||
# simple_content, so query terms fold identically to the indexed terms
|
||||
# (single source of truth for ASCII folding).
|
||||
return simple_search_tokens(raw_query)
|
||||
def _any_of(clauses: list[tuple[tantivy.Occur, tantivy.Query]]) -> tantivy.Query:
|
||||
"""Collapse a clause list: none -> empty, one -> itself (no wasted
|
||||
single-clause boolean_query wrapping), many -> boolean_query(clauses)."""
|
||||
if not clauses:
|
||||
return tantivy.Query.empty_query()
|
||||
if len(clauses) == 1:
|
||||
return clauses[0][1]
|
||||
return tantivy.Query.boolean_query(clauses)
|
||||
|
||||
|
||||
def _build_simple_token_query(
|
||||
@@ -168,9 +300,7 @@ def _build_simple_token_query(
|
||||
query = tantivy.Query.boost_query(query, boost)
|
||||
field_queries.append((tantivy.Occur.Should, query))
|
||||
|
||||
if len(field_queries) == 1:
|
||||
return field_queries[0][1]
|
||||
return tantivy.Query.boolean_query(field_queries)
|
||||
return _any_of(field_queries)
|
||||
|
||||
|
||||
def parse_user_query(
|
||||
@@ -179,50 +309,50 @@ def parse_user_query(
|
||||
tz: tzinfo,
|
||||
) -> tantivy.Query:
|
||||
"""
|
||||
Parse user query through the complete preprocessing pipeline.
|
||||
Parse user query through whoosh-compat, then blend in fuzzy/CJK clauses.
|
||||
|
||||
Transforms the raw user query through multiple stages:
|
||||
1. Date keyword rewriting (today → ISO 8601 ranges)
|
||||
2. Query normalization (comma expansion, whitespace cleanup)
|
||||
3. Tantivy parsing with field boosts
|
||||
4. Optional fuzzy query blending (if ADVANCED_FUZZY_SEARCH_THRESHOLD set)
|
||||
|
||||
Args:
|
||||
index: Tantivy index with registered tokenizers
|
||||
raw_query: Original user query string
|
||||
tz: Timezone for date boundary calculations
|
||||
|
||||
Returns:
|
||||
Parsed Tantivy query ready for execution
|
||||
|
||||
Note:
|
||||
When ADVANCED_FUZZY_SEARCH_THRESHOLD is configured, adds a low-priority
|
||||
fuzzy query as a Should clause (0.1 boost) to catch approximate matches
|
||||
while keeping exact matches ranked higher. The threshold value is applied
|
||||
as a post-search score filter, not during query construction.
|
||||
1. Two small pre-parse rewrites keep historically honored spellings
|
||||
working: unquoted multi-word date keyword phrases on date fields
|
||||
are quoted (_quote_date_keyword_phrases), and bare
|
||||
notes:/custom_fields: prefixes become their subpath equivalents
|
||||
(_rewrite_bare_json_field_prefixes). Then wc.parse() against the
|
||||
shared FieldRegistry (whoosh grammar -> AST).
|
||||
2. Any diagnostics (bad dates/numbers) map to SearchQueryError subclasses
|
||||
and raise — the view returns HTTP 400 with every offending field
|
||||
listed, not just the first.
|
||||
3. emit() turns the AST into a tantivy.Query directly (no string
|
||||
round-trip). UnsupportedQueryError (a construct that parses but can't
|
||||
execute against tantivy, e.g. a text-field range) also maps to a 400.
|
||||
4. Optional fuzzy blend (ADVANCED_FUZZY_SEARCH_THRESHOLD) builds a
|
||||
plain word string from the parsed AST's free-text tokens
|
||||
(whoosh_compat.free_text_tokens) and feeds THAT to
|
||||
index.parse_query — never raw_query, whose whoosh grammar (date
|
||||
keywords, bracket-class wildcards, etc.) tantivy's parser rejects,
|
||||
which used to silently knock the fuzzy clause out of any mixed
|
||||
query (see _try_parse_fuzzy_query).
|
||||
5. Optional CJK bigram clause — unchanged from before this migration,
|
||||
never went through the pre-whoosh-compat translation layer either.
|
||||
"""
|
||||
registry = get_field_registry(settings.SEARCH_LANGUAGE)
|
||||
raw_query = _quote_date_keyword_phrases(raw_query)
|
||||
raw_query = _rewrite_bare_json_field_prefixes(raw_query)
|
||||
result = wc.parse(
|
||||
raw_query,
|
||||
registry=registry,
|
||||
default_fields=_DEFAULT_SEARCH_FIELDS,
|
||||
field_boosts=_FIELD_BOOSTS,
|
||||
tz=tz,
|
||||
)
|
||||
if result.diagnostics:
|
||||
raise _diagnostics_to_error(result.diagnostics)
|
||||
|
||||
try:
|
||||
query_str = translate_query(raw_query, tz)
|
||||
except SearchQueryError:
|
||||
# Intentional, user-fixable error (e.g. an unparsable date). Propagate so
|
||||
# the view can return a 400 with a helpful message rather than falling
|
||||
# back to the raw (still-invalid) query.
|
||||
raise
|
||||
except Exception: # pragma: no cover - defensive
|
||||
logger.warning("Query translation failed; using raw query", exc_info=True)
|
||||
query_str = raw_query
|
||||
exact = tantivy_emit(result.ast, index=index, registry=registry)
|
||||
except (QueryEmitError, UnsupportedQueryError) as e:
|
||||
# emit()'s documented host contract: BOTH of these are user-input
|
||||
# errors, exactly like a parse diagnostic, and both map to a 400.
|
||||
raise SearchQueryError(_user_facing_emit_message(e)) from e
|
||||
|
||||
exact = index.parse_query(
|
||||
query_str,
|
||||
DEFAULT_SEARCH_FIELDS,
|
||||
field_boosts=_FIELD_BOOSTS,
|
||||
)
|
||||
|
||||
# The standard analyzer keeps a whitespace-free CJK run as a single token,
|
||||
# so substring queries can't match content/title (and long runs are dropped
|
||||
# by remove_long). Route CJK queries to the bigram fields, whose ngram
|
||||
# tokenizer indexes overlapping 2-grams for substring matching.
|
||||
cjk_query = (
|
||||
_build_cjk_query(index, raw_query, _CJK_ALL_FIELDS)
|
||||
if _has_cjk(raw_query)
|
||||
@@ -235,22 +365,36 @@ def parse_user_query(
|
||||
|
||||
threshold = settings.ADVANCED_FUZZY_SEARCH_THRESHOLD
|
||||
if threshold is not None:
|
||||
fuzzy = index.parse_query(
|
||||
query_str,
|
||||
DEFAULT_SEARCH_FIELDS,
|
||||
field_boosts=_FIELD_BOOSTS,
|
||||
# (prefix=True, distance=1, transposition_cost_one=True) — edit-distance fuzziness
|
||||
fuzzy_fields={f: (True, 1, True) for f in DEFAULT_SEARCH_FIELDS},
|
||||
)
|
||||
# 0.1 boost keeps fuzzy hits ranked below exact matches (intentional)
|
||||
clauses.append((tantivy.Occur.Should, tantivy.Query.boost_query(fuzzy, 0.1)))
|
||||
fuzzy = _try_parse_fuzzy_query(index, result.ast, registry)
|
||||
if fuzzy is not None:
|
||||
clauses.append(
|
||||
(tantivy.Occur.Should, tantivy.Query.boost_query(fuzzy, 0.1)),
|
||||
)
|
||||
|
||||
if cjk_query is not None:
|
||||
clauses.append((tantivy.Occur.Should, cjk_query))
|
||||
|
||||
if len(clauses) == 1:
|
||||
return exact
|
||||
return tantivy.Query.boolean_query(clauses)
|
||||
return _any_of(clauses)
|
||||
|
||||
|
||||
def _diagnostics_to_error(diagnostics: tuple[Diagnostic, ...]) -> SearchQueryError:
|
||||
errors = [_single_diagnostic_to_error(d) for d in diagnostics]
|
||||
return errors[0] if len(errors) == 1 else MultipleSearchQueryErrors(errors)
|
||||
|
||||
|
||||
def _single_diagnostic_to_error(d: Diagnostic) -> SearchQueryError:
|
||||
# d.field is a FieldRef, not a str: str(d.field) gives the canonical
|
||||
# dotted name (an aliased query, e.g. type:, reports document_type).
|
||||
field_name = str(d.field) if d.field is not None else None
|
||||
if d.kind is DiagnosticKind.BAD_DATE:
|
||||
return InvalidDateQuery(field_name, d.raw_value)
|
||||
if d.kind is DiagnosticKind.BAD_NUMBER:
|
||||
return InvalidNumberQuery(field_name, d.raw_value)
|
||||
# TOO_DEEP and UNSUPPORTED_PATTERN (e.g. a wildcard on asn/page_count/
|
||||
# num_notes, or on a custom_fields.*/notes.* subpath) fall through to
|
||||
# the generic message; consider whether either warrants its own typed
|
||||
# subclass if callers ever need to distinguish them programmatically.
|
||||
return SearchQueryError(d.message)
|
||||
|
||||
|
||||
def parse_simple_query(
|
||||
@@ -268,7 +412,7 @@ def parse_simple_query(
|
||||
CJK substrings the simple analyzer can't (long whitespace-free runs are
|
||||
dropped by remove_long).
|
||||
"""
|
||||
tokens = _simple_query_tokens(raw_query)
|
||||
tokens = simple_search_tokens(raw_query)
|
||||
|
||||
clauses: list[tuple[tantivy.Occur, tantivy.Query]] = []
|
||||
if tokens:
|
||||
@@ -291,23 +435,14 @@ def parse_simple_query(
|
||||
)
|
||||
for token in tokens
|
||||
]
|
||||
simple_query = (
|
||||
token_queries[0][1]
|
||||
if len(token_queries) == 1
|
||||
else tantivy.Query.boolean_query(token_queries)
|
||||
)
|
||||
clauses.append((tantivy.Occur.Should, simple_query))
|
||||
clauses.append((tantivy.Occur.Should, _any_of(token_queries)))
|
||||
|
||||
if cjk_fields and _has_cjk(raw_query):
|
||||
cjk_q = _build_cjk_query(index, raw_query, cjk_fields)
|
||||
if cjk_q is not None:
|
||||
clauses.append((tantivy.Occur.Should, cjk_q))
|
||||
|
||||
if not clauses:
|
||||
return tantivy.Query.empty_query()
|
||||
if len(clauses) == 1:
|
||||
return clauses[0][1]
|
||||
return tantivy.Query.boolean_query(clauses)
|
||||
return _any_of(clauses)
|
||||
|
||||
|
||||
def parse_simple_text_highlight_query(
|
||||
@@ -322,7 +457,7 @@ def parse_simple_text_highlight_query(
|
||||
|
||||
# Strip Tantivy operator chars before tokenizing: this is a plain-text
|
||||
# highlight query, not a structured boolean query, so +/- are separators.
|
||||
tokens = _simple_query_tokens(
|
||||
tokens = simple_search_tokens(
|
||||
regex.sub(r"[-+]", " ", raw_query, timeout=_REGEX_TIMEOUT),
|
||||
)
|
||||
if not tokens:
|
||||
@@ -342,7 +477,7 @@ def parse_simple_text_query(
|
||||
return parse_simple_query(
|
||||
index,
|
||||
raw_query,
|
||||
SIMPLE_SEARCH_FIELDS,
|
||||
_SIMPLE_SEARCH_FIELDS,
|
||||
cjk_fields=_CJK_CONTENT_FIELDS,
|
||||
)
|
||||
|
||||
@@ -358,6 +493,6 @@ def parse_simple_title_query(
|
||||
return parse_simple_query(
|
||||
index,
|
||||
raw_query,
|
||||
TITLE_SEARCH_FIELDS,
|
||||
_TITLE_SEARCH_FIELDS,
|
||||
cjk_fields=_CJK_TITLE_FIELDS,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
|
||||
from whoosh_compat import FieldKind
|
||||
from whoosh_compat import FieldRegistry
|
||||
|
||||
from documents.search._fields import PUBLIC_FIELDS
|
||||
from documents.search._tokenizer import ascii_fold
|
||||
from documents.search._tokenizer import paperless_text_analyzer
|
||||
|
||||
_registry_cache: dict[str | None, FieldRegistry] = {}
|
||||
|
||||
|
||||
def _identity_analyzer(text: str) -> list[str]:
|
||||
"""Analyzer for KEYWORD fields indexed with the raw tokenizer (no splitting)."""
|
||||
return [text]
|
||||
|
||||
|
||||
def _pattern_normalizer(text: str) -> str:
|
||||
"""Normalize wildcard/regex query patterns: lowercase -> ascii_fold.
|
||||
|
||||
Mirrors the lowercase -> ascii_fold steps of the index-time analyzers
|
||||
(paperless_text) without stemming, so pattern queries (e.g. "run*")
|
||||
match tokens that were folded the same way at index time but are not
|
||||
run through a stemmer, which would corrupt wildcard/regex semantics.
|
||||
"""
|
||||
return ascii_fold(text.lower())
|
||||
|
||||
|
||||
def get_field_registry(language: str | None) -> FieldRegistry:
|
||||
"""Build (or return the cached) FieldRegistry for the given search language.
|
||||
|
||||
Cached keyed by language, rebuilt on the same trigger register_tokenizers()
|
||||
uses (settings.SEARCH_LANGUAGE change) — a fresh call with a new language
|
||||
builds and caches a new registry rather than mutating the old one.
|
||||
"""
|
||||
if language in _registry_cache:
|
||||
return _registry_cache[language]
|
||||
|
||||
text_analyzer = paperless_text_analyzer(language).analyze
|
||||
|
||||
specs = [
|
||||
dataclasses.replace(
|
||||
field,
|
||||
analyzer=_identity_analyzer
|
||||
if field.kind is FieldKind.KEYWORD
|
||||
else text_analyzer,
|
||||
pattern_normalizer=_pattern_normalizer,
|
||||
)
|
||||
for field in PUBLIC_FIELDS
|
||||
]
|
||||
|
||||
registry = FieldRegistry(specs)
|
||||
_registry_cache[language] = registry
|
||||
return registry
|
||||
@@ -9,6 +9,9 @@ from typing import cast
|
||||
|
||||
import tantivy
|
||||
from django.conf import settings
|
||||
from whoosh_compat import FieldKind
|
||||
|
||||
from documents.search._fields import PUBLIC_FIELDS
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
@@ -33,17 +36,37 @@ def build_schema() -> tantivy.Schema:
|
||||
sb = tantivy.SchemaBuilder()
|
||||
|
||||
sb.add_unsigned_field("id", stored=True, indexed=True, fast=True)
|
||||
sb.add_text_field("checksum", stored=True, tokenizer_name="raw")
|
||||
|
||||
for field in (
|
||||
"title",
|
||||
"correspondent",
|
||||
"document_type",
|
||||
"storage_path",
|
||||
"original_filename",
|
||||
"content",
|
||||
):
|
||||
sb.add_text_field(field, stored=True, tokenizer_name="paperless_text")
|
||||
for field in PUBLIC_FIELDS:
|
||||
if field.kind is FieldKind.TEXT:
|
||||
sb.add_text_field(field.name, stored=True, tokenizer_name="paperless_text")
|
||||
elif field.kind is FieldKind.KEYWORD:
|
||||
sb.add_text_field(field.name, stored=True, tokenizer_name="raw")
|
||||
elif field.kind is FieldKind.U64:
|
||||
sb.add_unsigned_field(
|
||||
field.name,
|
||||
stored=True,
|
||||
indexed=True,
|
||||
fast=field.fast,
|
||||
)
|
||||
elif field.kind in (FieldKind.DATE, FieldKind.DATETIME):
|
||||
sb.add_date_field(
|
||||
field.name,
|
||||
stored=True,
|
||||
indexed=True,
|
||||
fast=field.fast,
|
||||
)
|
||||
elif field.kind is FieldKind.JSON:
|
||||
sb.add_json_field(field.name, stored=True, tokenizer_name="paperless_text")
|
||||
if field.name == "notes":
|
||||
# Plain-text companion for snippet generation — tantivy's
|
||||
# SnippetGenerator does not support JSON fields. Schema-only,
|
||||
# no query-syntax meaning, not in PUBLIC_FIELDS.
|
||||
sb.add_text_field(
|
||||
"notes_text",
|
||||
stored=True,
|
||||
tokenizer_name="paperless_text",
|
||||
)
|
||||
|
||||
# Shadow sort fields - fast, not stored/indexed
|
||||
for field in ("title_sort", "correspondent_sort", "type_sort"):
|
||||
@@ -86,15 +109,6 @@ def build_schema() -> tantivy.Schema:
|
||||
# The stored value is never read back, so storing it only wastes space.
|
||||
sb.add_text_field("autocomplete_word", stored=False, tokenizer_name="raw")
|
||||
|
||||
sb.add_text_field("tag", stored=True, tokenizer_name="paperless_text")
|
||||
|
||||
# JSON fields — structured queries: notes.user:alice, custom_fields.name:invoice
|
||||
sb.add_json_field("notes", stored=True, tokenizer_name="paperless_text")
|
||||
# Plain-text companion for notes — tantivy's SnippetGenerator does not support
|
||||
# JSON fields, so highlights require a text field with the same content.
|
||||
sb.add_text_field("notes_text", stored=True, tokenizer_name="paperless_text")
|
||||
sb.add_json_field("custom_fields", stored=True, tokenizer_name="paperless_text")
|
||||
|
||||
for field in (
|
||||
"correspondent_id",
|
||||
"document_type_id",
|
||||
@@ -106,12 +120,6 @@ def build_schema() -> tantivy.Schema:
|
||||
):
|
||||
sb.add_unsigned_field(field, stored=False, indexed=True, fast=True)
|
||||
|
||||
for field in ("created", "modified", "added"):
|
||||
sb.add_date_field(field, stored=True, indexed=True, fast=True)
|
||||
|
||||
for field in ("asn", "page_count", "num_notes"):
|
||||
sb.add_unsigned_field(field, stored=True, indexed=True, fast=True)
|
||||
|
||||
return sb.build()
|
||||
|
||||
|
||||
|
||||
@@ -71,7 +71,7 @@ def register_tokenizers(index: tantivy.Index, language: str | None) -> None:
|
||||
use fast=True and Tantivy requires fast-field tokenizers to exist
|
||||
even for documents that omit those fields.
|
||||
"""
|
||||
index.register_tokenizer("paperless_text", _paperless_text(language))
|
||||
index.register_tokenizer("paperless_text", paperless_text_analyzer(language))
|
||||
index.register_tokenizer("simple_analyzer", _simple_analyzer())
|
||||
index.register_tokenizer("bigram_analyzer", _bigram_analyzer())
|
||||
index.register_tokenizer("simple_search_analyzer", _simple_search_analyzer())
|
||||
@@ -79,7 +79,7 @@ def register_tokenizers(index: tantivy.Index, language: str | None) -> None:
|
||||
index.register_fast_field_tokenizer("simple_analyzer", _simple_analyzer())
|
||||
|
||||
|
||||
def _paperless_text(language: str | None) -> tantivy.TextAnalyzer:
|
||||
def paperless_text_analyzer(language: str | None) -> tantivy.TextAnalyzer:
|
||||
"""Main full-text tokenizer for content, title, etc: simple -> remove_long(129) -> lowercase -> ascii_fold [-> stemmer]"""
|
||||
builder = (
|
||||
tantivy.TextAnalyzerBuilder(tantivy.Tokenizer.simple())
|
||||
|
||||
@@ -1,610 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC
|
||||
from datetime import datetime
|
||||
from datetime import timedelta
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TypeAlias
|
||||
|
||||
import regex
|
||||
from dateutil.relativedelta import relativedelta
|
||||
|
||||
from documents.search._dates import _DATE_KEYWORDS
|
||||
from documents.search._dates import _DATE_ONLY_FIELDS
|
||||
from documents.search._dates import _date_only_range
|
||||
from documents.search._dates import _datetime_range
|
||||
from documents.search._dates import _field_range_from_dates
|
||||
from documents.search._dates import _fmt
|
||||
from documents.search._dates import _precision_bounds
|
||||
from documents.search._dates import _utc_bounds_for_field
|
||||
|
||||
# Compiled regex that matches any known multi-word (or single-word) date keyword
|
||||
# at the start of a match position, longest alternatives first so "previous week"
|
||||
# wins over a hypothetical shorter "previous".
|
||||
_KEYWORD_VALUE_RE = regex.compile(
|
||||
"|".join(sorted((regex.escape(k) for k in _DATE_KEYWORDS), key=len, reverse=True)),
|
||||
regex.IGNORECASE,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from datetime import tzinfo
|
||||
|
||||
# TODO: this module translates date queries into Tantivy *string* syntax, which
|
||||
# forces a workaround for something Tantivy's string parser cannot express on
|
||||
# date fields: open-ended ranges use far-past/far-future string sentinels
|
||||
# (OPEN_LO/OPEN_HI). These can be replaced with a real tantivy.Query object
|
||||
# (Query.range_query(..., None) for open bounds) once tantivy-py accepts Python
|
||||
# datetimes in range_query/term_query on Date fields. That support exists on
|
||||
# tantivy-py master (PRs #655 + #666) but postdates the pinned 0.26.0 wheel, so
|
||||
# it is blocked only on a published release > 0.26.0 and a dependency bump.
|
||||
# (Unparsable dates now raise InvalidDateQuery -> HTTP 400 rather than using a
|
||||
# no-match string sentinel.)
|
||||
|
||||
# Fields that store exact, non-analyzed comma-joined tokens in the index and so
|
||||
# need explicit comma->AND expansion (Whoosh KEYWORD(commas=True) set).
|
||||
MULTI_VALUE_FIELDS = frozenset({"tag", "tag_id", "viewer_id"})
|
||||
|
||||
# Date fields whose values/ranges get rewritten to RFC3339 Tantivy ranges.
|
||||
DATE_FIELDS = frozenset({"created", "modified", "added"})
|
||||
|
||||
# Field aliases: Whoosh (v2) field names that were renamed in the Tantivy schema.
|
||||
# Preserved here so v2 queries using the old names continue to work without 400
|
||||
# errors instead of silently failing. Applied by _render to non-date field tokens.
|
||||
FIELD_ALIASES: dict[str, str] = {
|
||||
"type": "document_type",
|
||||
"type_id": "document_type_id",
|
||||
"path": "storage_path",
|
||||
"path_id": "storage_path_id",
|
||||
}
|
||||
|
||||
# Known schema fields: a comma immediately followed by ``<known>:`` is a clause
|
||||
# separator. Restricting to known fields prevents URL-like ``http:`` misfires.
|
||||
KNOWN_FIELDS = frozenset(
|
||||
{
|
||||
"title",
|
||||
"content",
|
||||
"correspondent",
|
||||
"document_type",
|
||||
"type", # v2 alias -> document_type
|
||||
"storage_path",
|
||||
"path", # v2 alias -> storage_path
|
||||
"tag",
|
||||
"tag_id",
|
||||
"correspondent_id",
|
||||
"document_type_id",
|
||||
"type_id", # v2 alias -> document_type_id
|
||||
"storage_path_id",
|
||||
"path_id", # v2 alias -> storage_path_id
|
||||
"owner_id",
|
||||
"viewer_id",
|
||||
"asn",
|
||||
"page_count",
|
||||
"num_notes",
|
||||
"created",
|
||||
"modified",
|
||||
"added",
|
||||
"original_filename",
|
||||
"checksum",
|
||||
"notes",
|
||||
"custom_fields",
|
||||
},
|
||||
)
|
||||
|
||||
_FIELD_RE = regex.compile(r"(?P<field>\w+):")
|
||||
|
||||
# Matches the TO separator inside a range bracket. Handles three forms:
|
||||
# middle: "lo TO hi" (either lo or hi may be empty)
|
||||
# trailing: "lo TO" (open upper bound)
|
||||
# leading: "TO hi" (open lower bound)
|
||||
# Bounds MAY contain internal spaces (e.g. "-7 days"), so we use .*? / .+?
|
||||
# and split on the whitespace-delimited " TO " / " to " separator.
|
||||
_RANGE_RE = regex.compile(
|
||||
r"^\s*(?P<lo>.*?)\s+[Tt][Oo]\s+(?P<hi>.+?)\s*$"
|
||||
r"|"
|
||||
r"^\s*(?P<lo2>.+?)\s+[Tt][Oo]\s*$"
|
||||
r"|"
|
||||
r"^\s*[Tt][Oo]\s+(?P<hi2>.+?)\s*$",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FieldValue:
|
||||
field: str
|
||||
value: str
|
||||
|
||||
|
||||
# Produced by the comma-resolution pass (not by scan()).
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FieldValueList:
|
||||
field: str
|
||||
values: tuple[str, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FieldRange:
|
||||
field: str
|
||||
open: str
|
||||
lo: str
|
||||
hi: str
|
||||
close: str
|
||||
|
||||
|
||||
# Produced by the comma-resolution pass (not by scan()).
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Comma:
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Passthrough:
|
||||
raw: str
|
||||
|
||||
|
||||
Token: TypeAlias = FieldValue | FieldValueList | FieldRange | Comma | Passthrough
|
||||
|
||||
_CLOSE: dict[str, str] = {"[": "]", "{": "}"}
|
||||
|
||||
|
||||
def scan(query: str) -> list[Token]:
|
||||
"""
|
||||
Tokenize a raw query into date/comma-aware tokens, leaving everything else
|
||||
as verbatim ``Passthrough`` runs. Non-recursive: finds the first matching
|
||||
close bracket/quote. Nested brackets are not valid Tantivy range syntax and
|
||||
pass through verbatim on mismatch.
|
||||
"""
|
||||
tokens: list[Token] = []
|
||||
buf: list[str] = [] # accumulates passthrough chars
|
||||
i, n = 0, len(query)
|
||||
while i < n:
|
||||
matched = _match_field_token(query, i)
|
||||
if matched is None:
|
||||
buf.append(query[i])
|
||||
i += 1
|
||||
continue
|
||||
token, i = matched
|
||||
if buf and buf[-1] == ",":
|
||||
buf.pop()
|
||||
_flush(buf, tokens)
|
||||
tokens.append(Comma())
|
||||
else:
|
||||
_flush(buf, tokens)
|
||||
tokens.append(token)
|
||||
i = _maybe_comma(query, i, tokens)
|
||||
_flush(buf, tokens)
|
||||
return tokens
|
||||
|
||||
|
||||
def _flush(buf: list[str], tokens: list[Token]) -> None:
|
||||
"""Emit any accumulated passthrough characters as a single token."""
|
||||
if buf:
|
||||
tokens.append(Passthrough("".join(buf)))
|
||||
buf.clear()
|
||||
|
||||
|
||||
def _at_word_boundary(query: str, i: int) -> bool:
|
||||
"""A field token may begin only at the start or after a non-word character."""
|
||||
return i == 0 or not (query[i - 1].isalnum() or query[i - 1] == "_")
|
||||
|
||||
|
||||
def _match_field_token(query: str, i: int) -> tuple[Token, int] | None:
|
||||
"""
|
||||
If a known ``field:`` token starts at ``i``, consume it and return
|
||||
``(token, end_index)``; otherwise return None so the caller treats the
|
||||
character as passthrough. Handles both ``field:[range]`` and ``field:value``,
|
||||
and returns None when the range/value cannot be consumed.
|
||||
"""
|
||||
m = _FIELD_RE.match(query, i)
|
||||
if m is None or m.group("field") not in KNOWN_FIELDS:
|
||||
return None
|
||||
if not _at_word_boundary(query, i):
|
||||
return None
|
||||
field = m.group("field")
|
||||
j = m.end()
|
||||
if j < len(query) and query[j] in "[{":
|
||||
return _consume_range(query, j, field)
|
||||
consumed = _consume_field_value(query, field, j)
|
||||
if consumed is None:
|
||||
return None
|
||||
value, end = consumed
|
||||
return FieldValue(field, value), end
|
||||
|
||||
|
||||
def _consume_field_value(query: str, field: str, start: int) -> tuple[str, int] | None:
|
||||
"""
|
||||
Consume a field value starting at ``start``: a multi-word date keyword phrase
|
||||
(date fields only), or a bare/quoted value, then absorb any comma-joined
|
||||
continuation that is not a clause separator. ``resolve_commas`` later splits a
|
||||
multi-value field's joined value into a ``FieldValueList``; for other fields
|
||||
the comma stays literal.
|
||||
"""
|
||||
n = len(query)
|
||||
consumed = None
|
||||
if field in DATE_FIELDS:
|
||||
km = _KEYWORD_VALUE_RE.match(query, start)
|
||||
if km is not None and (km.end() >= n or query[km.end()] in " \t),"):
|
||||
consumed = (km.group(0), km.end())
|
||||
if consumed is None:
|
||||
consumed = _consume_value(query, start)
|
||||
if consumed is None:
|
||||
return None
|
||||
value, k = consumed
|
||||
while k < n and query[k] == ",":
|
||||
if _looks_like_known_field(query, k + 1):
|
||||
break # clause separator: left for _maybe_comma to emit a Comma()
|
||||
more = _consume_value(query, k + 1)
|
||||
if more is None:
|
||||
break
|
||||
value = f"{value},{more[0]}"
|
||||
k = more[1]
|
||||
return value, k
|
||||
|
||||
|
||||
def _consume_range(
|
||||
query: str,
|
||||
start: int,
|
||||
field: str,
|
||||
) -> tuple[FieldRange, int] | None:
|
||||
"""Consume ``[lo TO hi]`` / ``{lo TO hi}`` from ``start`` (the bracket)."""
|
||||
open_br = query[start]
|
||||
close_br = _CLOSE[open_br]
|
||||
end = query.find(close_br, start + 1)
|
||||
if end == -1:
|
||||
return None
|
||||
inner = query[start + 1 : end]
|
||||
m = _RANGE_RE.match(inner)
|
||||
if m is not None:
|
||||
if m.group("lo") is not None or m.group("hi") is not None:
|
||||
# Middle form: "lo TO hi" (either may be empty string)
|
||||
lo = (m.group("lo") or "").strip()
|
||||
hi = (m.group("hi") or "").strip()
|
||||
elif m.group("lo2") is not None:
|
||||
# Trailing form: "lo TO"
|
||||
lo = m.group("lo2").strip()
|
||||
hi = ""
|
||||
else:
|
||||
# Leading form: "TO hi"
|
||||
lo = ""
|
||||
hi = (m.group("hi2") or "").strip()
|
||||
else:
|
||||
lo, hi = inner.strip(), ""
|
||||
return FieldRange(field, open_br, lo, hi, close_br), end + 1
|
||||
|
||||
|
||||
def _consume_value(query: str, start: int) -> tuple[str, int] | None:
|
||||
"""Consume a bare or quoted field value from ``start``, stopping at comma."""
|
||||
n = len(query)
|
||||
if start >= n or query[start] in " \t":
|
||||
return None
|
||||
if query[start] in "\"'":
|
||||
quote = query[start]
|
||||
end = query.find(quote, start + 1)
|
||||
if end == -1:
|
||||
return None
|
||||
return query[start : end + 1], end + 1
|
||||
j = start
|
||||
while j < n and query[j] not in " \t),":
|
||||
j += 1
|
||||
return query[start:j], j
|
||||
|
||||
|
||||
def _looks_like_known_field(query: str, pos: int) -> bool:
|
||||
"""True if a known ``field:`` token starts at ``pos``."""
|
||||
m = _FIELD_RE.match(query, pos)
|
||||
return bool(m and m.group("field") in KNOWN_FIELDS)
|
||||
|
||||
|
||||
def _maybe_comma(query: str, i: int, tokens: list) -> int:
|
||||
"""If a clause-separator comma follows at ``i``, emit ``Comma()`` and advance."""
|
||||
if i < len(query) and query[i] == "," and _looks_like_known_field(query, i + 1):
|
||||
tokens.append(Comma())
|
||||
return i + 1
|
||||
return i
|
||||
|
||||
|
||||
def resolve_commas(tokens: list) -> list:
|
||||
"""
|
||||
Collapse value-list commas into ``FieldValueList`` and keep clause-separator
|
||||
commas as ``Comma``. (Clause-sep commas are already emitted by ``scan`` via
|
||||
the value-stop logic; this pass folds value-lists.)
|
||||
"""
|
||||
out: list = []
|
||||
for tok in tokens:
|
||||
if (
|
||||
isinstance(tok, FieldValue)
|
||||
and tok.field in MULTI_VALUE_FIELDS
|
||||
and "," in tok.value
|
||||
):
|
||||
values = tuple(v for v in tok.value.split(",") if v)
|
||||
out.append(FieldValueList(tok.field, values))
|
||||
else:
|
||||
out.append(tok)
|
||||
return out
|
||||
|
||||
|
||||
class SearchQueryError(ValueError):
|
||||
"""
|
||||
Base for user-fixable search query errors.
|
||||
|
||||
Carries a message safe to surface to the user (no internal details). The view
|
||||
layer catches this and returns an HTTP 400, so any future subclass (unknown
|
||||
field, malformed range, wrapped parser errors) gets the same treatment.
|
||||
"""
|
||||
|
||||
|
||||
class InvalidDateQuery(SearchQueryError):
|
||||
"""Raised when a date field value or range bound cannot be parsed."""
|
||||
|
||||
def __init__(self, field: str, value: str) -> None:
|
||||
self.field = field
|
||||
self.value = value
|
||||
super().__init__(f"Invalid date value {value!r} for field {field!r}.")
|
||||
|
||||
|
||||
_DIGITS_RE = regex.compile(r"^\d{4}(?:\d{2}){0,2}$")
|
||||
_ISO_RE = regex.compile(r"^\d{4}(?:-\d{2}(?:-\d{2})?)?$")
|
||||
|
||||
|
||||
def translate_scalar(field: str, value: str, tz: tzinfo) -> str:
|
||||
"""Translate a bare date-field value to a Tantivy range string."""
|
||||
bare = value.strip("\"'").lower()
|
||||
if bare in _DATE_KEYWORDS:
|
||||
if field in _DATE_ONLY_FIELDS:
|
||||
return f"{field}:{_date_only_range(bare, tz)}"
|
||||
return f"{field}:{_datetime_range(bare, tz)}"
|
||||
digits = value.replace("-", "")
|
||||
if _DIGITS_RE.match(value) or _ISO_RE.match(value):
|
||||
bounds = _precision_bounds(digits)
|
||||
if bounds is None:
|
||||
raise InvalidDateQuery(field, value)
|
||||
return _field_range_from_dates(field, bounds[0], bounds[1], tz)
|
||||
if regex.fullmatch(r"\d{14}", value):
|
||||
try:
|
||||
dt = datetime(
|
||||
int(value[0:4]),
|
||||
int(value[4:6]),
|
||||
int(value[6:8]),
|
||||
int(value[8:10]),
|
||||
int(value[10:12]),
|
||||
int(value[12:14]),
|
||||
tzinfo=UTC,
|
||||
)
|
||||
except ValueError:
|
||||
raise InvalidDateQuery(field, value) from None
|
||||
iso = _fmt(dt)
|
||||
return f"{field}:[{iso} TO {iso}]"
|
||||
# Unrecognized shape -> tell the user their date is malformed rather than
|
||||
# silently matching nothing or emitting invalid Tantivy syntax.
|
||||
raise InvalidDateQuery(field, value)
|
||||
|
||||
|
||||
# Open-bound sentinels for date ranges. These far-past/far-future strings allow
|
||||
# open-ended ranges to be expressed as Tantivy string queries until tantivy-py
|
||||
# exposes Query.range_query(..., None) on Date fields (see module TODO).
|
||||
OPEN_LO = "0001-01-01T00:00:00Z"
|
||||
OPEN_HI = "9999-12-31T23:59:59Z"
|
||||
|
||||
|
||||
# Matches compact now-offset tokens like now-7d, now+1h, now-30m.
|
||||
_NOW_COMPACT_RE = regex.compile(
|
||||
r"^now(?P<sign>[+-])(?P<n>\d+)(?P<unit>[dhm])$",
|
||||
regex.IGNORECASE,
|
||||
)
|
||||
|
||||
# Matches "±N <unit>" Whoosh-style offsets (e.g. -7 days, -1 week, +3 hours).
|
||||
# Whoosh's own date parser (qparser.dateparse.PlusMinus) additionally accepted
|
||||
# abbreviated unit spellings (e.g. "yrs", "yr", "y", "mos", "wks", "hrs", "mins",
|
||||
# "secs"); saved views/searches created under the old Whoosh backend can still
|
||||
# contain those tokens (e.g. "-999yrs"), so they are accepted here too and
|
||||
# normalized to a canonical unit via _UNIT_ALIASES below.
|
||||
_NOW_SPACED_RE = regex.compile(
|
||||
r"^(?P<sign>[+-])(?P<n>\d+)\s*"
|
||||
r"(?P<unit>years|year|yrs|yr|ys|y"
|
||||
r"|months|month|mons|mon|mos|mo"
|
||||
r"|weeks|week|wks|wk|ws|w"
|
||||
r"|days|day|dys|dy|ds|d"
|
||||
r"|hours|hour|hrs|hr|hs|h"
|
||||
r"|minutes|minute|mins|min|ms|m"
|
||||
r"|seconds|second|secs|sec|s)$",
|
||||
regex.IGNORECASE,
|
||||
)
|
||||
|
||||
# Maps every accepted unit spelling (including Whoosh-era abbreviations) to the
|
||||
# canonical unit name used as a key into the delta map in _resolve_relative_bound.
|
||||
_UNIT_ALIASES: dict[str, str] = {
|
||||
alias: canonical
|
||||
for canonical, aliases in {
|
||||
"year": ("years", "year", "yrs", "yr", "ys", "y"),
|
||||
"month": ("months", "month", "mons", "mon", "mos", "mo"),
|
||||
"week": ("weeks", "week", "wks", "wk", "ws", "w"),
|
||||
"day": ("days", "day", "dys", "dy", "ds", "d"),
|
||||
"hour": ("hours", "hour", "hrs", "hr", "hs", "h"),
|
||||
"minute": ("minutes", "minute", "mins", "min", "ms", "m"),
|
||||
"second": ("seconds", "second", "secs", "sec", "s"),
|
||||
}.items()
|
||||
for alias in aliases
|
||||
}
|
||||
|
||||
|
||||
def _resolve_relative_bound(token: str) -> datetime | None:
|
||||
"""
|
||||
Resolve a relative bound token to an exact UTC instant, or return None.
|
||||
|
||||
Supported forms:
|
||||
- ``now`` -> current UTC instant
|
||||
- ``now+/-<n>d/h/m`` -> now +/- timedelta (d=days, h=hours, m=minutes)
|
||||
- ``±N <unit>`` -> now +/- delta; month/year use relativedelta;
|
||||
unit also accepts Whoosh-era abbreviations
|
||||
(e.g. "yrs", "mos", "wks", "hrs", "mins", "secs")
|
||||
"""
|
||||
stripped = token.strip()
|
||||
low = stripped.lower()
|
||||
now = datetime.now(UTC)
|
||||
|
||||
if low == "now":
|
||||
return now
|
||||
|
||||
m = _NOW_COMPACT_RE.match(stripped)
|
||||
if m:
|
||||
sign = 1 if m.group("sign") == "+" else -1
|
||||
n = int(m.group("n"))
|
||||
unit = m.group("unit").lower()
|
||||
delta = (
|
||||
sign
|
||||
* {
|
||||
"d": timedelta(days=n),
|
||||
"h": timedelta(hours=n),
|
||||
"m": timedelta(minutes=n),
|
||||
}[unit]
|
||||
)
|
||||
return now + delta
|
||||
|
||||
m = _NOW_SPACED_RE.match(stripped)
|
||||
if m:
|
||||
sign = 1 if m.group("sign") == "+" else -1
|
||||
n = int(m.group("n"))
|
||||
unit = _UNIT_ALIASES[m.group("unit").lower()]
|
||||
delta_map: dict[str, timedelta | relativedelta] = {
|
||||
"second": timedelta(seconds=n),
|
||||
"minute": timedelta(minutes=n),
|
||||
"hour": timedelta(hours=n),
|
||||
"day": timedelta(days=n),
|
||||
"week": timedelta(weeks=n),
|
||||
"month": relativedelta(months=n),
|
||||
"year": relativedelta(years=n),
|
||||
}
|
||||
return now - delta_map[unit] if sign == -1 else now + delta_map[unit]
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _bound_datetimes(
|
||||
field: str,
|
||||
token: str,
|
||||
tz: tzinfo,
|
||||
) -> tuple[datetime, datetime] | None:
|
||||
"""
|
||||
Return (floor_dt, ceil_dt) UTC datetimes for a single range bound token, or
|
||||
None if the token is unparsable. ``now`` and relative offsets resolve to the
|
||||
current instant (floor == ceil == that instant; no day-flooring).
|
||||
"""
|
||||
token = token.strip()
|
||||
|
||||
# Try relative/now forms first (before stripping hyphens which would mangle them).
|
||||
rel = _resolve_relative_bound(token)
|
||||
if rel is not None:
|
||||
return rel, rel
|
||||
|
||||
# Full ISO datetime token (contains "T"): parse directly and return an exact
|
||||
# instant (floor == ceil). Python 3.11+ datetime.fromisoformat accepts trailing Z.
|
||||
if "T" in token:
|
||||
try:
|
||||
dt = datetime.fromisoformat(token)
|
||||
# Ensure timezone-aware UTC result.
|
||||
dt = dt.replace(tzinfo=UTC) if dt.tzinfo is None else dt.astimezone(UTC)
|
||||
return dt, dt
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
digits = token.replace("-", "")
|
||||
bounds = _precision_bounds(digits)
|
||||
if bounds is None:
|
||||
return None
|
||||
start, end = bounds
|
||||
return _utc_bounds_for_field(field, start, end, tz)
|
||||
|
||||
|
||||
def _render(tok: Token, tz: tzinfo) -> str:
|
||||
"""Render a single token back to a Tantivy query string fragment."""
|
||||
if isinstance(tok, Passthrough):
|
||||
return tok.raw
|
||||
if isinstance(tok, Comma):
|
||||
return " AND "
|
||||
if isinstance(tok, FieldValueList):
|
||||
field = FIELD_ALIASES.get(tok.field, tok.field)
|
||||
return " AND ".join(f"{field}:{v}" for v in tok.values)
|
||||
if isinstance(tok, FieldValue):
|
||||
field = FIELD_ALIASES.get(tok.field, tok.field)
|
||||
if field in DATE_FIELDS:
|
||||
return translate_scalar(field, tok.value, tz)
|
||||
return f"{field}:{tok.value}"
|
||||
if isinstance(tok, FieldRange):
|
||||
field = FIELD_ALIASES.get(tok.field, tok.field)
|
||||
if field in DATE_FIELDS:
|
||||
return translate_range(field, tok.lo, tok.hi, tz)
|
||||
return f"{field}:{tok.open}{tok.lo} TO {tok.hi}{tok.close}"
|
||||
return "" # pragma: no cover
|
||||
|
||||
|
||||
# Post-render operator normalization patterns: collapse repeated whitespace and
|
||||
# strip spaced/trailing Tantivy boolean operators that would otherwise be invalid.
|
||||
_MULTI_SPACE_RE = regex.compile(r" {2,}")
|
||||
_TRAILING_OP_RE = regex.compile(r"\s+[-+]+\s*$")
|
||||
_SPACED_OP_RE = regex.compile(r"\s+[-+]\s+")
|
||||
|
||||
|
||||
def _normalize_operators(text: str) -> str:
|
||||
"""
|
||||
Collapse multiple spaces, strip trailing dangling operators, and replace
|
||||
spaced operators (`` - `` / `` + ``) with a single space.
|
||||
|
||||
Applied only to Passthrough fragments (the rendered output is scanned for
|
||||
operator artifacts outside bracketed ranges) via a post-render pass on the
|
||||
full rendered string. This preserves date ranges (``[... TO ...]``) verbatim
|
||||
while cleaning natural-language separators in the surrounding text.
|
||||
"""
|
||||
text = _MULTI_SPACE_RE.sub(" ", text)
|
||||
text = _TRAILING_OP_RE.sub("", text).strip()
|
||||
text = _SPACED_OP_RE.sub(" ", text).strip()
|
||||
return text
|
||||
|
||||
|
||||
def translate_query(raw: str, tz: tzinfo) -> str:
|
||||
"""Translate a raw Whoosh-style query into Tantivy-compatible syntax."""
|
||||
tokens = resolve_commas(scan(raw))
|
||||
rendered = "".join(_render(t, tz) for t in tokens)
|
||||
return _normalize_operators(rendered)
|
||||
|
||||
|
||||
def translate_range(field: str, lo: str, hi: str, tz: tzinfo) -> str:
|
||||
"""Translate a date-field ``[lo TO hi]`` range to a Tantivy ISO range string.
|
||||
|
||||
Handles partial-date bounds (YYYY, YYYYMM, YYYYMMDD, ISO dash variants),
|
||||
open bounds (empty string -> OPEN_LO/OPEN_HI), ``now``, and reversed ranges
|
||||
(swaps tokens before computing floor/ceil so the span is always correct).
|
||||
"""
|
||||
lo_s = lo.strip()
|
||||
hi_s = hi.strip()
|
||||
|
||||
# Parse both bounds to (floor, ceil) pairs when present.
|
||||
lo_pair: tuple[datetime, datetime] | None = None
|
||||
hi_pair: tuple[datetime, datetime] | None = None
|
||||
|
||||
if lo_s:
|
||||
lo_pair = _bound_datetimes(field, lo_s, tz)
|
||||
if lo_pair is None:
|
||||
raise InvalidDateQuery(field, lo_s)
|
||||
if hi_s:
|
||||
hi_pair = _bound_datetimes(field, hi_s, tz)
|
||||
if hi_pair is None:
|
||||
raise InvalidDateQuery(field, hi_s)
|
||||
|
||||
# Detect a reversed range: only swap when BOTH bounds are present.
|
||||
if lo_pair is not None and hi_pair is not None and lo_pair[0] > hi_pair[0]:
|
||||
lo_pair, hi_pair = hi_pair, lo_pair
|
||||
|
||||
lo_iso = _fmt(lo_pair[0]) if lo_pair is not None else OPEN_LO
|
||||
|
||||
# A bound resolves to (floor, ceil) where floor == ceil for an exact instant
|
||||
# (a full ISO datetime, "now", or a "+/-N unit" offset) and floor != ceil for
|
||||
# a coarser period token (year/month/day precision). Only the latter needs a
|
||||
# half-open close: its ceil is the start of the *next* period and must be
|
||||
# excluded, or that instant (e.g. the 1st of next month) wrongly matches.
|
||||
if hi_pair is not None:
|
||||
hi_iso = _fmt(hi_pair[1])
|
||||
hi_close = "]" if hi_pair[0] == hi_pair[1] else "}"
|
||||
else:
|
||||
hi_iso = OPEN_HI
|
||||
hi_close = "]"
|
||||
|
||||
return f"{field}:[{lo_iso} TO {hi_iso}{hi_close}"
|
||||
@@ -0,0 +1,483 @@
|
||||
"""Result-level acceptance corpus: real documents indexed via build_schema(),
|
||||
real queries run through parse_user_query(), matched-document-ID sets
|
||||
asserted — not intermediate ASTs or query strings. This is paperless-ngx's
|
||||
analogue of whoosh-compat's own tests/emitter/test_acceptance_e2e.py.
|
||||
|
||||
Supersedes test_query.py's TestParseUserQuery result-level cases.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
import time_machine
|
||||
from django.contrib.auth.models import User
|
||||
|
||||
from documents.models import CustomField
|
||||
from documents.models import CustomFieldInstance
|
||||
from documents.models import Document
|
||||
from documents.models import DocumentType
|
||||
from documents.models import Note
|
||||
from documents.models import StoragePath
|
||||
from documents.search._query import parse_user_query
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from documents.search._backend import TantivyBackend
|
||||
|
||||
pytestmark = [pytest.mark.search, pytest.mark.django_db]
|
||||
|
||||
FROZEN_NOW = datetime(2026, 6, 15, 12, 0, tzinfo=UTC)
|
||||
|
||||
|
||||
def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
|
||||
return set(backend.search_ids(query, user=None))
|
||||
|
||||
|
||||
def _index(backend: TantivyBackend, **kwargs: object) -> Document:
|
||||
"""Create a Document and index it in one step, for the common case
|
||||
where nothing needs to happen between the two (no related Note/
|
||||
CustomFieldInstance to attach first)."""
|
||||
doc = Document.objects.create(**kwargs)
|
||||
backend.add_or_update(doc)
|
||||
return doc
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def indexed_documents(backend: TantivyBackend) -> dict[str, int]:
|
||||
"""Index a small fixture set, return {label: doc_id} for corpus queries."""
|
||||
docs = {
|
||||
"invoice_2020": _index(
|
||||
backend,
|
||||
title="Invoice 2020",
|
||||
content="invoice total due",
|
||||
checksum="acc-invoice-2020",
|
||||
archive_serial_number=100,
|
||||
),
|
||||
"invoice_2021": _index(
|
||||
backend,
|
||||
title="Invoice 2021",
|
||||
content="invoice total due",
|
||||
checksum="acc-invoice-2021",
|
||||
archive_serial_number=101,
|
||||
),
|
||||
"invoice_2023": _index(
|
||||
backend,
|
||||
title="Invoice 2023",
|
||||
content="invoice total due",
|
||||
checksum="acc-invoice-2023",
|
||||
archive_serial_number=102,
|
||||
),
|
||||
"receipt_2022": _index(
|
||||
backend,
|
||||
title="Receipt 2022",
|
||||
content="receipt total due",
|
||||
checksum="acc-receipt-2022",
|
||||
archive_serial_number=103,
|
||||
),
|
||||
}
|
||||
return {label: doc.pk for label, doc in docs.items()}
|
||||
|
||||
|
||||
class TestIssue13568BracketWildcard:
|
||||
"""paperless-ngx#13568: title:202[0-3]* must keep its character class,
|
||||
not fold to a prefix query that silently drops it."""
|
||||
|
||||
def test_bracket_class_wildcard_matches_only_in_range_years(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
indexed_documents: dict[str, int],
|
||||
) -> None:
|
||||
# [0-1] (not [0-3]) is deliberate: the fixture's four years are
|
||||
# 2020/2021/2022/2023, i.e. their trailing digit is 0/1/2/3
|
||||
# respectively - a [0-3] class would match all four and the test
|
||||
# would pass even if the character class were silently dropped and
|
||||
# folded to an unconstrained "202*" prefix. [0-1] partitions the
|
||||
# fixture into a genuine in-range/out-of-range split.
|
||||
matched = _matched_ids(backend, "title:202[0-1]*")
|
||||
expected = {
|
||||
indexed_documents["invoice_2020"],
|
||||
indexed_documents["invoice_2021"],
|
||||
}
|
||||
assert matched == expected, (
|
||||
"title:202[0-1]* must match 2020/2021 titles and exclude 2022/2023 "
|
||||
"- if this matches everything, the wildcard's character class was "
|
||||
"silently dropped (issue #13568's original bug)"
|
||||
)
|
||||
|
||||
|
||||
class TestFieldBoosts:
|
||||
def test_title_boost_ranks_title_match_above_content_only_match(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
) -> None:
|
||||
title_match = _index(
|
||||
backend,
|
||||
title="urgent",
|
||||
content="nothing else relevant",
|
||||
checksum="acc-boost-title",
|
||||
)
|
||||
_index(
|
||||
backend,
|
||||
title="nothing",
|
||||
content="urgent matter here",
|
||||
checksum="acc-boost-content",
|
||||
)
|
||||
query = parse_user_query(backend._index, "urgent", UTC)
|
||||
searcher = backend._index.searcher()
|
||||
results = searcher.search(query, limit=10)
|
||||
ranked_ids = [
|
||||
searcher.doc(addr).to_dict()["id"][0] for _score, addr in results.hits
|
||||
]
|
||||
assert ranked_ids[0] == title_match.pk
|
||||
|
||||
|
||||
class TestJsonSubpaths:
|
||||
def test_notes_user_matches_document_with_that_note_author(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
) -> None:
|
||||
alice = User.objects.create_user(username="alice")
|
||||
doc_with_note = Document.objects.create(
|
||||
title="Has note",
|
||||
content="x",
|
||||
checksum="acc-note-with",
|
||||
)
|
||||
Note.objects.create(document=doc_with_note, user=alice, note="reminder")
|
||||
backend.add_or_update(doc_with_note)
|
||||
_index(backend, title="No note", content="x", checksum="acc-note-without")
|
||||
matched = _matched_ids(backend, "notes.user:alice")
|
||||
assert matched == {doc_with_note.pk}
|
||||
|
||||
def test_custom_fields_name_and_value_combine(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
) -> None:
|
||||
field = CustomField.objects.create(
|
||||
name="Contract Number",
|
||||
data_type=CustomField.FieldDataType.STRING,
|
||||
)
|
||||
other_field = CustomField.objects.create(
|
||||
name="Other Field",
|
||||
data_type=CustomField.FieldDataType.STRING,
|
||||
)
|
||||
matching = Document.objects.create(
|
||||
title="Matching",
|
||||
content="x",
|
||||
checksum="acc-cf-matching",
|
||||
)
|
||||
CustomFieldInstance.objects.create(
|
||||
document=matching,
|
||||
field=field,
|
||||
value_text="policy",
|
||||
)
|
||||
backend.add_or_update(matching)
|
||||
non_matching = Document.objects.create(
|
||||
title="Non-matching",
|
||||
content="x",
|
||||
checksum="acc-cf-nonmatching",
|
||||
)
|
||||
CustomFieldInstance.objects.create(
|
||||
document=non_matching,
|
||||
field=other_field,
|
||||
value_text="policy",
|
||||
)
|
||||
backend.add_or_update(non_matching)
|
||||
matched = _matched_ids(
|
||||
backend,
|
||||
'custom_fields.name:"Contract Number" custom_fields.value:policy',
|
||||
)
|
||||
assert matched == {matching.pk}
|
||||
|
||||
|
||||
class TestUnregisteredIdFieldFoldsToLiteralText:
|
||||
"""tag_id, owner_id, etc. are intentionally excluded from the
|
||||
FieldRegistry - always internal index columns, never meant to be
|
||||
query-addressable. Prove an unregistered field folds to a literal
|
||||
text search that matches nothing, rather than erroring."""
|
||||
|
||||
def test_tag_id_query_matches_nothing(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
indexed_documents: dict[str, int],
|
||||
) -> None:
|
||||
matched = _matched_ids(backend, "tag_id:5")
|
||||
assert matched == set()
|
||||
|
||||
|
||||
class TestFuzzyBlendSurvivesWhooshGrammar:
|
||||
"""A query mixing whoosh-only grammar (a date keyword) with a typo'd
|
||||
free-text word must still fuzzy-match the intended document when
|
||||
ADVANCED_FUZZY_SEARCH_THRESHOLD is enabled. The fuzzy clause is built
|
||||
from the parsed query's free-text tokens (whoosh_compat's
|
||||
free_text_tokens), never from the raw query string, so whoosh grammar
|
||||
that tantivy's own parser rejects cannot knock the fuzzy clause out."""
|
||||
|
||||
def test_typo_fuzzy_matches_alongside_date_keyword(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
settings,
|
||||
) -> None:
|
||||
settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.5
|
||||
with time_machine.travel(FROZEN_NOW, tick=False):
|
||||
doc = _index(
|
||||
backend,
|
||||
title="Receipt March",
|
||||
content="receipt total due",
|
||||
checksum="fuzzy-blend-1",
|
||||
archive_serial_number=900,
|
||||
)
|
||||
# Sanity: the exact spelling matches through the exact clause.
|
||||
assert doc.pk in _matched_ids(backend, "added:today receipt")
|
||||
# The regression: the misspelling (one transposition) only
|
||||
# matches via the fuzzy clause, and "added:today" is
|
||||
# whoosh-only grammar tantivy's parser rejects, so raw-string
|
||||
# fuzzy parsing skips the clause entirely and this returns
|
||||
# nothing. The typo is deliberate; keep codespell away from it.
|
||||
typo_query = "added:today reciept" # codespell:ignore reciept
|
||||
assert doc.pk in _matched_ids(backend, typo_query)
|
||||
|
||||
def test_negated_words_do_not_fuzzy_match(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
settings,
|
||||
) -> None:
|
||||
# A term the user excluded must not resurface through the fuzzy
|
||||
# clause. The shape is chosen so this genuinely discriminates: the
|
||||
# indexed document contains the NOT'd word but NOT the positive
|
||||
# word, so nothing matches the exact clause, and a fuzzy string
|
||||
# naively built from ALL words (including the NOT'd one) would
|
||||
# make this document the sole hit, normalize its score to 1.0,
|
||||
# and survive any threshold. (A shape with an exact-matching
|
||||
# sibling document does NOT discriminate: normalization ranks the
|
||||
# resurfaced doc far below the exact match and the threshold cuts
|
||||
# it even for a naive implementation.)
|
||||
settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.5
|
||||
with time_machine.travel(FROZEN_NOW, tick=False):
|
||||
_index(
|
||||
backend,
|
||||
title="Receipt Archive",
|
||||
content="receipt archived stack",
|
||||
checksum="fuzzy-blend-2",
|
||||
archive_serial_number=901,
|
||||
)
|
||||
assert _matched_ids(backend, "added:today total NOT receipt") == set()
|
||||
|
||||
|
||||
class TestUnquotedDateKeywordPhrases:
|
||||
"""The unquoted spelling (added:previous month) has always been
|
||||
honored via an app-level quoting assist, since whoosh-compat's parser
|
||||
only accepts the quoted form natively. paperless quotes the closed
|
||||
phrase vocabulary on date fields before parsing; every date
|
||||
computation still happens in whoosh-compat."""
|
||||
|
||||
@pytest.fixture
|
||||
def period_documents(self, backend: TantivyBackend) -> dict[str, int]:
|
||||
with time_machine.travel(FROZEN_NOW, tick=False):
|
||||
in_may = _index(
|
||||
backend,
|
||||
title="May Doc",
|
||||
content="statement",
|
||||
checksum="kw-may",
|
||||
archive_serial_number=910,
|
||||
added=datetime(2026, 5, 20, 12, 0, tzinfo=UTC),
|
||||
)
|
||||
in_june = _index(
|
||||
backend,
|
||||
title="June Doc",
|
||||
content="statement",
|
||||
checksum="kw-june",
|
||||
archive_serial_number=911,
|
||||
added=datetime(2026, 6, 10, 12, 0, tzinfo=UTC),
|
||||
)
|
||||
return {"in_may": in_may.pk, "in_june": in_june.pk}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"query",
|
||||
[
|
||||
pytest.param("added:previous month", id="unquoted"),
|
||||
pytest.param('added:"previous month"', id="quoted"),
|
||||
pytest.param("added:Previous Month", id="unquoted-mixed-case"),
|
||||
],
|
||||
)
|
||||
def test_unquoted_matches_the_same_documents_as_quoted(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
period_documents: dict[str, int],
|
||||
query: str,
|
||||
) -> None:
|
||||
with time_machine.travel(FROZEN_NOW, tick=False):
|
||||
assert _matched_ids(backend, query) == {period_documents["in_may"]}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"query",
|
||||
[
|
||||
pytest.param("added:this month", id="this-month"),
|
||||
pytest.param("added:this year", id="this-year"),
|
||||
pytest.param("added:previous week", id="previous-week"),
|
||||
pytest.param("added:previous quarter", id="previous-quarter"),
|
||||
pytest.param("added:previous year", id="previous-year"),
|
||||
pytest.param("created:previous month", id="created-field"),
|
||||
pytest.param("modified:previous month", id="modified-field"),
|
||||
],
|
||||
)
|
||||
def test_every_phrase_and_date_field_parses_without_error(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
period_documents: dict[str, int],
|
||||
query: str,
|
||||
) -> None:
|
||||
# The whole vocabulary times every date field must at least parse
|
||||
# and search cleanly (no SearchQueryError -> no HTTP 400); exact
|
||||
# window semantics are whoosh-compat's, pinned in its own suite.
|
||||
with time_machine.travel(FROZEN_NOW, tick=False):
|
||||
_matched_ids(backend, query)
|
||||
|
||||
def test_text_field_keyword_words_are_not_rewritten(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
period_documents: dict[str, int],
|
||||
) -> None:
|
||||
# "previous month" after a TEXT field (or unfielded) is ordinary
|
||||
# text, not a date phrase: a title actually containing the words
|
||||
# matches, and the date-window documents do not.
|
||||
with time_machine.travel(FROZEN_NOW, tick=False):
|
||||
wordy = _index(
|
||||
backend,
|
||||
title="Notes from the previous month",
|
||||
content="meeting notes",
|
||||
checksum="kw-text",
|
||||
archive_serial_number=912,
|
||||
)
|
||||
assert _matched_ids(backend, "title:previous month") == {wordy.pk}
|
||||
|
||||
|
||||
class TestBareJsonFieldPrefixes:
|
||||
""" "notes:foo"/"custom_fields:foo" were valid fielded searches before
|
||||
this migration. whoosh-compat's registry only exposes them as JSON
|
||||
subpaths, so parse_user_query rewrites the bare prefixes live: notes:
|
||||
-> notes.note:, custom_fields: -> custom_fields.value:."""
|
||||
|
||||
def test_bare_notes_prefix_searches_note_text(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
) -> None:
|
||||
alice = User.objects.create_user(username="alice")
|
||||
with_note = Document.objects.create(
|
||||
title="Has note",
|
||||
content="x",
|
||||
checksum="bare-notes-with",
|
||||
)
|
||||
Note.objects.create(document=with_note, user=alice, note="crocodile")
|
||||
backend.add_or_update(with_note)
|
||||
# This document's CONTENT contains the words a demoted text search
|
||||
# would match; it must NOT match once the prefix addresses notes.
|
||||
_index(
|
||||
backend,
|
||||
title="Notes about things",
|
||||
content="notes crocodile mention",
|
||||
checksum="bare-notes-decoy",
|
||||
)
|
||||
assert _matched_ids(backend, "notes:crocodile") == {with_note.pk}
|
||||
|
||||
def test_bare_custom_fields_prefix_searches_values(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
) -> None:
|
||||
field = CustomField.objects.create(
|
||||
name="Policy Number",
|
||||
data_type=CustomField.FieldDataType.STRING,
|
||||
)
|
||||
with_value = Document.objects.create(
|
||||
title="Has field",
|
||||
content="x",
|
||||
checksum="bare-cf-with",
|
||||
)
|
||||
CustomFieldInstance.objects.create(
|
||||
document=with_value,
|
||||
field=field,
|
||||
value_text="crocodile",
|
||||
)
|
||||
backend.add_or_update(with_value)
|
||||
_index(
|
||||
backend,
|
||||
title="Custom things",
|
||||
content="custom fields crocodile",
|
||||
checksum="bare-cf-decoy",
|
||||
)
|
||||
assert _matched_ids(backend, "custom_fields:crocodile") == {with_value.pk}
|
||||
|
||||
def test_subpath_spellings_are_untouched(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
) -> None:
|
||||
bob = User.objects.create_user(username="bob")
|
||||
doc = Document.objects.create(
|
||||
title="Bob note",
|
||||
content="x",
|
||||
checksum="bare-subpath",
|
||||
)
|
||||
Note.objects.create(document=doc, user=bob, note="remark")
|
||||
backend.add_or_update(doc)
|
||||
assert _matched_ids(backend, "notes.user:bob") == {doc.pk}
|
||||
assert _matched_ids(backend, "notes.note:remark") == {doc.pk}
|
||||
|
||||
|
||||
class TestFieldAliases:
|
||||
"""type:/path: are registry aliases for document_type:/storage_path:.
|
||||
The only other alias coverage is parse-shape; these prove resolution
|
||||
end-to-end against a real index."""
|
||||
|
||||
def test_type_alias_and_canonical_name_match_the_same_document(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
) -> None:
|
||||
invoice_type = DocumentType.objects.create(name="invoice")
|
||||
# Discriminating shape: document_type is itself a default search
|
||||
# field, so if alias resolution ever broke and "type:invoice"
|
||||
# demoted to unfielded text, the token would STILL match the typed
|
||||
# document through the field value. The decoy carries the query
|
||||
# word in content, so a demoted search matches BOTH documents and
|
||||
# the exact-set assertions fail. (The title avoids stemming to
|
||||
# "type": english stems Typed -> type.)
|
||||
typed = _index(
|
||||
backend,
|
||||
title="First",
|
||||
content="quarterly statement",
|
||||
checksum="alias-type-1",
|
||||
document_type=invoice_type,
|
||||
)
|
||||
_index(
|
||||
backend,
|
||||
title="Second",
|
||||
content="invoice mentioned in body",
|
||||
checksum="alias-type-2",
|
||||
)
|
||||
assert _matched_ids(backend, "type:invoice") == {typed.pk}
|
||||
assert _matched_ids(backend, "document_type:invoice") == {typed.pk}
|
||||
|
||||
def test_path_alias_and_canonical_name_match_the_same_document(
|
||||
self,
|
||||
backend: TantivyBackend,
|
||||
) -> None:
|
||||
archive = StoragePath.objects.create(name="archive", path="archive/{title}")
|
||||
stored = _index(
|
||||
backend,
|
||||
title="Stored",
|
||||
content="quarterly statement",
|
||||
checksum="alias-path-1",
|
||||
storage_path=archive,
|
||||
)
|
||||
# storage_path is NOT a default search field today, so a demoted
|
||||
# "path:archive" already matches nothing; the content decoy keeps
|
||||
# this test discriminating even if it ever joins the defaults.
|
||||
_index(
|
||||
backend,
|
||||
title="Loose",
|
||||
content="archive mentioned in body",
|
||||
checksum="alias-path-2",
|
||||
)
|
||||
assert _matched_ids(backend, "path:archive") == {stored.pk}
|
||||
assert _matched_ids(backend, "storage_path:archive") == {stored.pk}
|
||||
@@ -0,0 +1,68 @@
|
||||
import pytest
|
||||
from whoosh_compat import FieldKind
|
||||
|
||||
from documents.search._fields import PUBLIC_FIELDS
|
||||
|
||||
BY_NAME = {f.name: f for f in PUBLIC_FIELDS}
|
||||
|
||||
|
||||
class TestPublicFields:
|
||||
def test_every_field_has_a_whoosh_compat_kind(self) -> None:
|
||||
for field in PUBLIC_FIELDS:
|
||||
assert isinstance(field.kind, FieldKind)
|
||||
|
||||
def test_names_are_unique(self) -> None:
|
||||
names = [f.name for f in PUBLIC_FIELDS]
|
||||
assert len(names) == len(set(names))
|
||||
|
||||
def test_json_fields_have_subpaths(self) -> None:
|
||||
for field in PUBLIC_FIELDS:
|
||||
if field.kind is FieldKind.JSON:
|
||||
assert field.subpaths, f"{field.name} is JSON but has no subpaths"
|
||||
|
||||
def test_non_json_fields_have_no_subpaths(self) -> None:
|
||||
for field in PUBLIC_FIELDS:
|
||||
if field.kind is not FieldKind.JSON:
|
||||
assert not field.subpaths
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("name", "attr", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
"document_type",
|
||||
"aliases",
|
||||
("type",),
|
||||
id="document_type-aliases",
|
||||
),
|
||||
pytest.param(
|
||||
"storage_path",
|
||||
"aliases",
|
||||
("path",),
|
||||
id="storage_path-aliases",
|
||||
),
|
||||
pytest.param("tag", "comma_values", True, id="tag-comma_values"),
|
||||
pytest.param(
|
||||
"notes",
|
||||
"subpaths",
|
||||
{"user", "note"},
|
||||
id="notes-subpaths",
|
||||
),
|
||||
pytest.param(
|
||||
"custom_fields",
|
||||
"subpaths",
|
||||
{"name", "value"},
|
||||
id="custom_fields-subpaths",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_field_attributes(self, name: str, attr: str, expected: object) -> None:
|
||||
actual = getattr(BY_NAME[name], attr)
|
||||
if attr == "subpaths":
|
||||
actual = set(actual)
|
||||
assert actual == expected
|
||||
|
||||
def test_no_internal_id_fields_present(self) -> None:
|
||||
# tag_id/owner_id/viewer_id/etc. are permission-filter-only fields,
|
||||
# never user-query-addressable (see design spec, "Field surface").
|
||||
names = {f.name for f in PUBLIC_FIELDS}
|
||||
assert not any(name.endswith("_id") for name in names)
|
||||
@@ -1,448 +1,75 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import UTC
|
||||
from datetime import datetime
|
||||
from datetime import tzinfo
|
||||
from typing import TYPE_CHECKING
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import pytest
|
||||
import tantivy
|
||||
import time_machine
|
||||
|
||||
from documents.search._dates import _date_only_range
|
||||
from documents.search._dates import _datetime_range
|
||||
from documents.search._query import build_permission_filter
|
||||
from documents.search._backend import build_permission_filter
|
||||
from documents.search._errors import InvalidDateQuery
|
||||
from documents.search._errors import InvalidNumberQuery
|
||||
from documents.search._errors import MultipleSearchQueryErrors
|
||||
from documents.search._errors import SearchQueryError
|
||||
from documents.search._query import parse_simple_text_highlight_query
|
||||
from documents.search._query import parse_user_query
|
||||
from documents.search._schema import build_schema
|
||||
from documents.search._tokenizer import register_tokenizers
|
||||
from documents.search._translate import InvalidDateQuery
|
||||
from documents.search._translate import translate_query
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from django.contrib.auth.base_user import AbstractBaseUser
|
||||
|
||||
pytestmark = pytest.mark.search
|
||||
|
||||
EASTERN = ZoneInfo("America/New_York") # UTC-5 / UTC-4 (DST)
|
||||
AUCKLAND = ZoneInfo("Pacific/Auckland") # UTC+13 in southern-hemisphere summer
|
||||
|
||||
|
||||
def _range(result: str, field: str) -> tuple[str, str]:
|
||||
# Half-open period ranges close with "}" (exclusive); exact-instant ranges
|
||||
# (full ISO datetimes, "now", relative offsets) close with "]" (inclusive).
|
||||
m = re.search(rf"{field}:\[(.+?) TO (.+?)[\]}}]", result)
|
||||
assert m, f"No range for {field!r} in: {result!r}"
|
||||
return m.group(1), m.group(2)
|
||||
|
||||
|
||||
class TestCreatedDateField:
|
||||
"""
|
||||
created is a Django DateField: indexed as midnight UTC of the local calendar
|
||||
date. No offset arithmetic needed - the local calendar date is what matters.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tz", "expected_lo", "expected_hi"),
|
||||
[
|
||||
pytest.param(UTC, "2026-03-28T00:00:00Z", "2026-03-29T00:00:00Z", id="utc"),
|
||||
pytest.param(
|
||||
EASTERN,
|
||||
"2026-03-28T00:00:00Z",
|
||||
"2026-03-29T00:00:00Z",
|
||||
id="eastern_same_calendar_date",
|
||||
),
|
||||
],
|
||||
)
|
||||
@time_machine.travel(datetime(2026, 3, 28, 15, 30, tzinfo=UTC), tick=False)
|
||||
def test_today(self, tz: tzinfo, expected_lo: str, expected_hi: str) -> None:
|
||||
lo, hi = _range(translate_query("created:today", tz), "created")
|
||||
assert lo == expected_lo
|
||||
assert hi == expected_hi
|
||||
|
||||
@time_machine.travel(datetime(2026, 3, 28, 3, 0, tzinfo=UTC), tick=False)
|
||||
def test_today_auckland_ahead_of_utc(self) -> None:
|
||||
# UTC 03:00 -> Auckland (UTC+13) = 16:00 same date; local date = 2026-03-28
|
||||
lo, _ = _range(
|
||||
translate_query("created:today", AUCKLAND),
|
||||
"created",
|
||||
)
|
||||
assert lo == "2026-03-28T00:00:00Z"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "keyword", "expected_lo", "expected_hi"),
|
||||
[
|
||||
pytest.param(
|
||||
"created",
|
||||
"yesterday",
|
||||
"2026-03-27T00:00:00Z",
|
||||
"2026-03-28T00:00:00Z",
|
||||
id="yesterday",
|
||||
),
|
||||
pytest.param(
|
||||
"created",
|
||||
"previous week",
|
||||
"2026-03-16T00:00:00Z",
|
||||
"2026-03-23T00:00:00Z",
|
||||
id="previous_week",
|
||||
),
|
||||
pytest.param(
|
||||
"created",
|
||||
"this month",
|
||||
"2026-03-01T00:00:00Z",
|
||||
"2026-04-01T00:00:00Z",
|
||||
id="this_month",
|
||||
),
|
||||
pytest.param(
|
||||
"created",
|
||||
"previous month",
|
||||
"2026-02-01T00:00:00Z",
|
||||
"2026-03-01T00:00:00Z",
|
||||
id="previous_month",
|
||||
),
|
||||
pytest.param(
|
||||
"created",
|
||||
"this year",
|
||||
"2026-01-01T00:00:00Z",
|
||||
"2027-01-01T00:00:00Z",
|
||||
id="this_year",
|
||||
),
|
||||
pytest.param(
|
||||
"created",
|
||||
"previous year",
|
||||
"2025-01-01T00:00:00Z",
|
||||
"2026-01-01T00:00:00Z",
|
||||
id="previous_year",
|
||||
),
|
||||
],
|
||||
)
|
||||
@time_machine.travel(datetime(2026, 3, 28, 15, 0, tzinfo=UTC), tick=False)
|
||||
def test_date_keywords(
|
||||
self,
|
||||
field: str,
|
||||
keyword: str,
|
||||
expected_lo: str,
|
||||
expected_hi: str,
|
||||
) -> None:
|
||||
# 2026-03-28 is Saturday; Mon-Sun week calculation built into expectations
|
||||
query = f"{field}:{keyword}"
|
||||
lo, hi = _range(translate_query(query, UTC), field)
|
||||
assert lo == expected_lo
|
||||
assert hi == expected_hi
|
||||
|
||||
@time_machine.travel(datetime(2026, 12, 15, 12, 0, tzinfo=UTC), tick=False)
|
||||
def test_this_month_december_wraps_to_next_year(self) -> None:
|
||||
# December: next month must roll over to January 1 of next year
|
||||
lo, hi = _range(
|
||||
translate_query("created:this month", UTC),
|
||||
"created",
|
||||
)
|
||||
assert lo == "2026-12-01T00:00:00Z"
|
||||
assert hi == "2027-01-01T00:00:00Z"
|
||||
|
||||
@time_machine.travel(datetime(2026, 1, 15, 12, 0, tzinfo=UTC), tick=False)
|
||||
def test_last_month_january_wraps_to_previous_year(self) -> None:
|
||||
# January: last month must roll back to December 1 of previous year
|
||||
lo, hi = _range(
|
||||
translate_query("created:previous month", UTC),
|
||||
"created",
|
||||
)
|
||||
assert lo == "2025-12-01T00:00:00Z"
|
||||
assert hi == "2026-01-01T00:00:00Z"
|
||||
|
||||
@time_machine.travel(datetime(2026, 7, 15, 12, 0, tzinfo=UTC), tick=False)
|
||||
def test_previous_quarter(self) -> None:
|
||||
lo, hi = _range(
|
||||
translate_query('created:"previous quarter"', UTC),
|
||||
"created",
|
||||
)
|
||||
assert lo == "2026-04-01T00:00:00Z"
|
||||
assert hi == "2026-07-01T00:00:00Z"
|
||||
|
||||
def test_unknown_keyword_raises(self) -> None:
|
||||
with pytest.raises(ValueError, match="Unknown keyword"):
|
||||
_date_only_range("bogus_keyword", UTC)
|
||||
|
||||
|
||||
class TestDateTimeFields:
|
||||
"""
|
||||
added/modified store full UTC datetimes. Natural keywords must convert
|
||||
the local day boundaries to UTC - timezone offset arithmetic IS required.
|
||||
"""
|
||||
|
||||
@time_machine.travel(datetime(2026, 3, 28, 15, 30, tzinfo=UTC), tick=False)
|
||||
def test_added_today_eastern(self) -> None:
|
||||
# EDT = UTC-4; local midnight 2026-03-28 00:00 EDT = 2026-03-28 04:00 UTC
|
||||
lo, hi = _range(translate_query("added:today", EASTERN), "added")
|
||||
assert lo == "2026-03-28T04:00:00Z"
|
||||
assert hi == "2026-03-29T04:00:00Z"
|
||||
|
||||
@time_machine.travel(datetime(2026, 3, 29, 2, 0, tzinfo=UTC), tick=False)
|
||||
def test_added_today_auckland_midnight_crossing(self) -> None:
|
||||
# UTC 02:00 on 2026-03-29 -> Auckland (UTC+13) = 2026-03-29 15:00 local
|
||||
# Auckland midnight = UTC 2026-03-28 11:00
|
||||
lo, hi = _range(translate_query("added:today", AUCKLAND), "added")
|
||||
assert lo == "2026-03-28T11:00:00Z"
|
||||
assert hi == "2026-03-29T11:00:00Z"
|
||||
|
||||
@time_machine.travel(datetime(2026, 3, 28, 15, 0, tzinfo=UTC), tick=False)
|
||||
def test_modified_today_utc(self) -> None:
|
||||
lo, hi = _range(
|
||||
translate_query("modified:today", UTC),
|
||||
"modified",
|
||||
)
|
||||
assert lo == "2026-03-28T00:00:00Z"
|
||||
assert hi == "2026-03-29T00:00:00Z"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("keyword", "expected_lo", "expected_hi"),
|
||||
[
|
||||
pytest.param(
|
||||
"yesterday",
|
||||
"2026-03-27T00:00:00Z",
|
||||
"2026-03-28T00:00:00Z",
|
||||
id="yesterday",
|
||||
),
|
||||
pytest.param(
|
||||
"previous week",
|
||||
"2026-03-16T00:00:00Z",
|
||||
"2026-03-23T00:00:00Z",
|
||||
id="previous_week",
|
||||
),
|
||||
pytest.param(
|
||||
"this month",
|
||||
"2026-03-01T00:00:00Z",
|
||||
"2026-04-01T00:00:00Z",
|
||||
id="this_month",
|
||||
),
|
||||
pytest.param(
|
||||
"previous month",
|
||||
"2026-02-01T00:00:00Z",
|
||||
"2026-03-01T00:00:00Z",
|
||||
id="previous_month",
|
||||
),
|
||||
pytest.param(
|
||||
"this year",
|
||||
"2026-01-01T00:00:00Z",
|
||||
"2027-01-01T00:00:00Z",
|
||||
id="this_year",
|
||||
),
|
||||
pytest.param(
|
||||
"previous year",
|
||||
"2025-01-01T00:00:00Z",
|
||||
"2026-01-01T00:00:00Z",
|
||||
id="previous_year",
|
||||
),
|
||||
],
|
||||
)
|
||||
@time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False)
|
||||
def test_datetime_keywords_utc(
|
||||
self,
|
||||
keyword: str,
|
||||
expected_lo: str,
|
||||
expected_hi: str,
|
||||
) -> None:
|
||||
# 2026-03-28 is Saturday; weekday()==5 so Monday=2026-03-23
|
||||
lo, hi = _range(translate_query(f"added:{keyword}", UTC), "added")
|
||||
assert lo == expected_lo
|
||||
assert hi == expected_hi
|
||||
|
||||
@time_machine.travel(datetime(2026, 12, 15, 12, 0, tzinfo=UTC), tick=False)
|
||||
def test_this_month_december_wraps_to_next_year(self) -> None:
|
||||
# December: next month wraps to January of next year
|
||||
lo, hi = _range(translate_query("added:this month", UTC), "added")
|
||||
assert lo == "2026-12-01T00:00:00Z"
|
||||
assert hi == "2027-01-01T00:00:00Z"
|
||||
|
||||
@time_machine.travel(datetime(2026, 1, 15, 12, 0, tzinfo=UTC), tick=False)
|
||||
def test_last_month_january_wraps_to_previous_year(self) -> None:
|
||||
# January: last month wraps back to December of previous year
|
||||
lo, hi = _range(
|
||||
translate_query("added:previous month", UTC),
|
||||
"added",
|
||||
)
|
||||
assert lo == "2025-12-01T00:00:00Z"
|
||||
assert hi == "2026-01-01T00:00:00Z"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("query", "expected_lo", "expected_hi"),
|
||||
[
|
||||
pytest.param(
|
||||
'added:"previous quarter"',
|
||||
"2026-04-01T00:00:00Z",
|
||||
"2026-07-01T00:00:00Z",
|
||||
id="quoted_previous_quarter",
|
||||
),
|
||||
pytest.param(
|
||||
"added:previous month",
|
||||
"2026-06-01T00:00:00Z",
|
||||
"2026-07-01T00:00:00Z",
|
||||
id="bare_previous_month",
|
||||
),
|
||||
pytest.param(
|
||||
"added:this month",
|
||||
"2026-07-01T00:00:00Z",
|
||||
"2026-08-01T00:00:00Z",
|
||||
id="bare_this_month",
|
||||
),
|
||||
],
|
||||
)
|
||||
@time_machine.travel(datetime(2026, 7, 15, 12, 0, tzinfo=UTC), tick=False)
|
||||
def test_legacy_natural_language_aliases(
|
||||
self,
|
||||
query: str,
|
||||
expected_lo: str,
|
||||
expected_hi: str,
|
||||
) -> None:
|
||||
lo, hi = _range(translate_query(query, UTC), "added")
|
||||
assert lo == expected_lo
|
||||
assert hi == expected_hi
|
||||
|
||||
def test_unknown_keyword_raises(self) -> None:
|
||||
with pytest.raises(ValueError, match="Unknown keyword"):
|
||||
_datetime_range("bogus_keyword", UTC)
|
||||
|
||||
|
||||
class TestWhooshQueryRewriting:
|
||||
"""All Whoosh query syntax variants must be rewritten to ISO 8601 before Tantivy parses them."""
|
||||
|
||||
@time_machine.travel(datetime(2026, 3, 28, 15, 0, tzinfo=UTC), tick=False)
|
||||
def test_compact_date_shim_rewrites_to_iso(self) -> None:
|
||||
result = translate_query("created:20240115120000", UTC)
|
||||
assert "2024-01-15" in result
|
||||
assert "20240115120000" not in result
|
||||
|
||||
@time_machine.travel(datetime(2026, 3, 28, 15, 0, tzinfo=UTC), tick=False)
|
||||
def test_relative_range_shim_removes_now(self) -> None:
|
||||
result = translate_query("added:[now-7d TO now]", UTC)
|
||||
assert "now" not in result
|
||||
assert "2026-03-" in result
|
||||
|
||||
@time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False)
|
||||
def test_bracket_minus_7_days(self) -> None:
|
||||
lo, hi = _range(
|
||||
translate_query("added:[-7 days to now]", UTC),
|
||||
"added",
|
||||
)
|
||||
assert lo == "2026-03-21T12:00:00Z"
|
||||
assert hi == "2026-03-28T12:00:00Z"
|
||||
|
||||
@time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False)
|
||||
def test_bracket_minus_1_week(self) -> None:
|
||||
lo, hi = _range(
|
||||
translate_query("added:[-1 week to now]", UTC),
|
||||
"added",
|
||||
)
|
||||
assert lo == "2026-03-21T12:00:00Z"
|
||||
assert hi == "2026-03-28T12:00:00Z"
|
||||
|
||||
@time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False)
|
||||
def test_bracket_minus_1_month_uses_relativedelta(self) -> None:
|
||||
# relativedelta(months=1) from 2026-03-28 = 2026-02-28 (not 29)
|
||||
lo, hi = _range(
|
||||
translate_query("created:[-1 month to now]", UTC),
|
||||
"created",
|
||||
)
|
||||
assert lo == "2026-02-28T12:00:00Z"
|
||||
assert hi == "2026-03-28T12:00:00Z"
|
||||
|
||||
@time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False)
|
||||
def test_bracket_minus_1_year(self) -> None:
|
||||
lo, hi = _range(
|
||||
translate_query("modified:[-1 year to now]", UTC),
|
||||
"modified",
|
||||
)
|
||||
assert lo == "2025-03-28T12:00:00Z"
|
||||
assert hi == "2026-03-28T12:00:00Z"
|
||||
|
||||
@time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False)
|
||||
def test_bracket_plural_unit_hours(self) -> None:
|
||||
lo, hi = _range(
|
||||
translate_query("added:[-3 hours to now]", UTC),
|
||||
"added",
|
||||
)
|
||||
assert lo == "2026-03-28T09:00:00Z"
|
||||
assert hi == "2026-03-28T12:00:00Z"
|
||||
|
||||
@time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False)
|
||||
def test_bracket_case_insensitive(self) -> None:
|
||||
result = translate_query("added:[-1 WEEK TO NOW]", UTC)
|
||||
assert "now" not in result.lower()
|
||||
lo, hi = _range(result, "added")
|
||||
assert lo == "2026-03-21T12:00:00Z"
|
||||
assert hi == "2026-03-28T12:00:00Z"
|
||||
|
||||
@time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False)
|
||||
def test_relative_range_swaps_bounds_when_lo_exceeds_hi(self) -> None:
|
||||
# [now+1h TO now-1h] has lo > hi before substitution; they must be swapped
|
||||
lo, hi = _range(
|
||||
translate_query("added:[now+1h TO now-1h]", UTC),
|
||||
"added",
|
||||
)
|
||||
assert lo == "2026-03-28T11:00:00Z"
|
||||
assert hi == "2026-03-28T13:00:00Z"
|
||||
|
||||
def test_8digit_created_date_field_always_uses_utc_midnight(self) -> None:
|
||||
# created is a DateField: boundaries are always UTC midnight, no TZ offset
|
||||
result = translate_query("created:20231201", EASTERN)
|
||||
lo, hi = _range(result, "created")
|
||||
assert lo == "2023-12-01T00:00:00Z"
|
||||
assert hi == "2023-12-02T00:00:00Z"
|
||||
|
||||
def test_8digit_added_datetime_field_converts_local_midnight_to_utc(self) -> None:
|
||||
# added is DateTimeField: midnight Dec 1 Eastern (EST = UTC-5) = 05:00 UTC
|
||||
result = translate_query("added:20231201", EASTERN)
|
||||
lo, hi = _range(result, "added")
|
||||
assert lo == "2023-12-01T05:00:00Z"
|
||||
assert hi == "2023-12-02T05:00:00Z"
|
||||
|
||||
def test_8digit_modified_datetime_field_converts_local_midnight_to_utc(
|
||||
self,
|
||||
) -> None:
|
||||
result = translate_query("modified:20231201", EASTERN)
|
||||
lo, hi = _range(result, "modified")
|
||||
assert lo == "2023-12-01T05:00:00Z"
|
||||
assert hi == "2023-12-02T05:00:00Z"
|
||||
|
||||
def test_8digit_invalid_date_raises(self) -> None:
|
||||
# The translation pipeline raises InvalidDateQuery for unparsable dates
|
||||
# (e.g. month=13) so the API can surface a 400 telling the user the date
|
||||
# is malformed instead of silently returning zero results.
|
||||
with pytest.raises(InvalidDateQuery) as exc_info:
|
||||
translate_query("added:20231340", UTC)
|
||||
assert exc_info.value.field == "added"
|
||||
assert exc_info.value.value == "20231340"
|
||||
@pytest.fixture(scope="module")
|
||||
def query_index() -> tantivy.Index:
|
||||
"""An in-memory, unstemmed index shared read-only across this module's
|
||||
parse-only tests (none of them index documents)."""
|
||||
schema = build_schema()
|
||||
idx = tantivy.Index(schema, path=None)
|
||||
register_tokenizers(idx, "")
|
||||
return idx
|
||||
|
||||
|
||||
class TestParseUserQuery:
|
||||
"""parse_user_query runs the full preprocessing pipeline."""
|
||||
|
||||
@pytest.fixture
|
||||
def query_index(self) -> tantivy.Index:
|
||||
schema = build_schema()
|
||||
idx = tantivy.Index(schema, path=None)
|
||||
register_tokenizers(idx, "")
|
||||
return idx
|
||||
|
||||
def test_returns_tantivy_query(self, query_index: tantivy.Index) -> None:
|
||||
assert isinstance(parse_user_query(query_index, "invoice", UTC), tantivy.Query)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw_query",
|
||||
[
|
||||
pytest.param("invoice", id="plain_text"),
|
||||
pytest.param("created:today", id="date_keyword"),
|
||||
pytest.param("created:[2005 to 2009]", id="whoosh_date_range"),
|
||||
pytest.param('added:"previous month"', id="quoted_date_phrase"),
|
||||
pytest.param("title:202[0-1]*", id="bracket_class_wildcard"),
|
||||
],
|
||||
)
|
||||
def test_fuzzy_mode_does_not_raise(
|
||||
self,
|
||||
query_index: tantivy.Index,
|
||||
settings,
|
||||
raw_query: str,
|
||||
) -> None:
|
||||
# These are all valid whoosh grammar that tantivy's own query parser
|
||||
# (used only by the fuzzy blend clause) cannot parse; the fuzzy
|
||||
# clause must degrade gracefully instead of raising and failing the
|
||||
# whole query. See _try_parse_fuzzy_query.
|
||||
settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.5
|
||||
assert isinstance(parse_user_query(query_index, "invoice", UTC), tantivy.Query)
|
||||
assert isinstance(parse_user_query(query_index, raw_query, UTC), tantivy.Query)
|
||||
|
||||
def test_date_rewriting_applied_before_tantivy_parse(
|
||||
def test_date_keyword_resolves_without_raising(
|
||||
self,
|
||||
query_index: tantivy.Index,
|
||||
) -> None:
|
||||
# created:today must be rewritten to an ISO range before Tantivy parses it;
|
||||
# if passed raw, Tantivy would reject "today" as an invalid date value
|
||||
# whoosh-compat's DateParserPlugin resolves "today" against the AST
|
||||
# directly (no string rewrite to an ISO range happens anywhere in
|
||||
# this pipeline); the emitted tantivy query must still build cleanly.
|
||||
with time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False):
|
||||
q = parse_user_query(query_index, "created:today", UTC)
|
||||
assert isinstance(q, tantivy.Query)
|
||||
@@ -466,302 +93,72 @@ class TestParseUserQuery:
|
||||
) -> None:
|
||||
assert isinstance(parse_user_query(query_index, raw_query, UTC), tantivy.Query)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw_query",
|
||||
[
|
||||
# Partial date scalar (year only)
|
||||
pytest.param("created:2020", id="created_year_scalar"),
|
||||
# 8-digit compact date range in brackets
|
||||
pytest.param(
|
||||
"created:[20200101 TO 20201231]",
|
||||
id="created_8digit_bracket_range",
|
||||
),
|
||||
# Comma-separated field + date range (Whoosh v2 multi-clause syntax)
|
||||
pytest.param(
|
||||
"title:x,created:[2020 TO 2021]",
|
||||
id="title_comma_created_range",
|
||||
),
|
||||
# Field alias: type -> document_type
|
||||
pytest.param("type:invoice", id="type_alias"),
|
||||
# Multi-word date keyword
|
||||
pytest.param("created:previous week", id="created_previous_week"),
|
||||
# Full ISO datetime range
|
||||
pytest.param(
|
||||
"created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]",
|
||||
id="created_iso_range",
|
||||
),
|
||||
# Comma-separated ISO ranges (Whoosh v2 syntax)
|
||||
pytest.param(
|
||||
"created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z],"
|
||||
"added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]",
|
||||
id="comma_iso_ranges",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_advanced_search_queries_do_not_raise(
|
||||
self,
|
||||
query_index: tantivy.Index,
|
||||
raw_query: str,
|
||||
) -> None:
|
||||
"""
|
||||
End-to-end: queries that the frontend sends must parse without raising.
|
||||
|
||||
This tests the full pipeline: translate_query -> tantivy parse_query.
|
||||
Equivalent to asserting HTTP 200 (not 400) for each query form.
|
||||
"""
|
||||
with time_machine.travel(datetime(2026, 6, 15, 12, 0, tzinfo=UTC), tick=False):
|
||||
assert isinstance(
|
||||
parse_user_query(query_index, raw_query, UTC),
|
||||
tantivy.Query,
|
||||
)
|
||||
|
||||
def test_invalid_date_propagates_not_swallowed(
|
||||
self,
|
||||
query_index: tantivy.Index,
|
||||
) -> None:
|
||||
# parse_user_query falls back to the raw query on unexpected translation
|
||||
# errors, but an InvalidDateQuery is intentional and must propagate so the
|
||||
# view can return a 400 instead of silently parsing the raw (invalid) date.
|
||||
# parse_user_query never falls back to the raw query string on a parse
|
||||
# error — a bad date diagnostic from whoosh-compat always maps to an
|
||||
# InvalidDateQuery and must propagate, so the view can return a 400
|
||||
# instead of silently parsing the raw (invalid) date.
|
||||
with pytest.raises(InvalidDateQuery) as exc_info:
|
||||
parse_user_query(query_index, "created:202023", UTC)
|
||||
assert exc_info.value.field == "created"
|
||||
assert exc_info.value.value == "202023"
|
||||
|
||||
|
||||
class TestYearRangeRewriting:
|
||||
"""Whoosh-style year-only date ranges must be rewritten to ISO 8601."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("query", "field", "expected_lo", "expected_hi"),
|
||||
[
|
||||
pytest.param(
|
||||
"created:[2020 TO 2020]",
|
||||
"created",
|
||||
"2020-01-01T00:00:00Z",
|
||||
"2021-01-01T00:00:00Z",
|
||||
id="single_year_created",
|
||||
),
|
||||
pytest.param(
|
||||
"created:[2018 TO 2021]",
|
||||
"created",
|
||||
"2018-01-01T00:00:00Z",
|
||||
"2022-01-01T00:00:00Z",
|
||||
id="multi_year_range_created",
|
||||
),
|
||||
pytest.param(
|
||||
"added:[2022 TO 2023]",
|
||||
"added",
|
||||
"2022-01-01T00:00:00Z",
|
||||
"2024-01-01T00:00:00Z",
|
||||
id="added_field",
|
||||
),
|
||||
pytest.param(
|
||||
"modified:[2021 TO 2021]",
|
||||
"modified",
|
||||
"2021-01-01T00:00:00Z",
|
||||
"2022-01-01T00:00:00Z",
|
||||
id="modified_field",
|
||||
),
|
||||
pytest.param(
|
||||
"created:[2020 to 2020]",
|
||||
"created",
|
||||
"2020-01-01T00:00:00Z",
|
||||
"2021-01-01T00:00:00Z",
|
||||
id="lowercase_to_keyword",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_year_range_rewritten(
|
||||
def test_invalid_number_raises_invalid_number_query(
|
||||
self,
|
||||
query: str,
|
||||
field: str,
|
||||
expected_lo: str,
|
||||
expected_hi: str,
|
||||
query_index: tantivy.Index,
|
||||
) -> None:
|
||||
result = translate_query(query, UTC)
|
||||
lo, hi = _range(result, field)
|
||||
assert lo == expected_lo
|
||||
assert hi == expected_hi
|
||||
with pytest.raises(InvalidNumberQuery) as exc_info:
|
||||
parse_user_query(query_index, "asn:notanumber", UTC)
|
||||
assert exc_info.value.field == "asn"
|
||||
assert exc_info.value.value == "notanumber"
|
||||
|
||||
def test_reversed_year_range_is_swapped(self) -> None:
|
||||
# A reversed range must not yield lo > hi, which Tantivy treats as an
|
||||
# empty range (silently zero results). The bounds are swapped instead.
|
||||
result = translate_query("created:[2025 TO 2020]", UTC)
|
||||
lo, hi = _range(result, "created")
|
||||
assert lo == "2020-01-01T00:00:00Z"
|
||||
assert hi == "2026-01-01T00:00:00Z"
|
||||
|
||||
def test_year_range_in_complex_boolean_query(self) -> None:
|
||||
query = "tag:steuer AND (title:2020 OR (NOT title:2019 AND NOT title:2018 AND created:[2020 TO 2020]))"
|
||||
result = translate_query(query, UTC)
|
||||
lo, hi = _range(result, "created")
|
||||
assert lo == "2020-01-01T00:00:00Z"
|
||||
assert hi == "2021-01-01T00:00:00Z"
|
||||
assert "title:2020" in result
|
||||
assert "title:2019" in result
|
||||
assert "title:2018" in result
|
||||
|
||||
def test_already_iso_date_range_passes_through_unchanged(self) -> None:
|
||||
original = "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z]"
|
||||
assert translate_query(original, UTC) == original
|
||||
|
||||
def test_8digit_in_brackets_not_matched_as_year_range(self) -> None:
|
||||
# [YYYYMMDD TO YYYYMMDD]: the translation layer converts 8-digit bounds to
|
||||
# ISO day ranges. 20200101 -> 2020-01-01T00:00:00Z (lo of that day);
|
||||
# 20201231 -> the ceil of Dec 31 = 2021-01-01T00:00:00Z (exclusive end).
|
||||
# This is the correct and accepted behavior: old compact form becomes a
|
||||
# proper Tantivy-parseable ISO range.
|
||||
original = "created:[20200101 TO 20201231]"
|
||||
result = translate_query(original, UTC)
|
||||
lo, hi = _range(result, "created")
|
||||
assert lo == "2020-01-01T00:00:00Z"
|
||||
assert hi == "2021-01-01T00:00:00Z"
|
||||
|
||||
|
||||
class TestNonDateFieldsNotRewritten:
|
||||
"""Date rewriters must only fire on the date fields (created/modified/added).
|
||||
|
||||
Integer fields like asn/id/page_count and unknown fields would otherwise be
|
||||
rewritten into date ranges and rejected by Tantivy as type mismatches.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"query",
|
||||
[
|
||||
pytest.param("asn:20240101", id="asn_8digit"),
|
||||
pytest.param("id:20240101", id="id_8digit"),
|
||||
pytest.param("page_count:12345678", id="page_count_8digit"),
|
||||
pytest.param("num_notes:20231201", id="num_notes_8digit"),
|
||||
],
|
||||
)
|
||||
def test_8digit_on_integer_field_passes_through_unchanged(self, query: str) -> None:
|
||||
assert translate_query(query, EASTERN) == query
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"query",
|
||||
[
|
||||
pytest.param("asn:[2000 TO 2024]", id="asn_year_range"),
|
||||
pytest.param("id:[2000 TO 2024]", id="id_year_range"),
|
||||
pytest.param("page_count:[2000 TO 2024]", id="page_count_year_range"),
|
||||
],
|
||||
)
|
||||
def test_year_range_on_integer_field_passes_through_unchanged(
|
||||
def test_multiple_bad_fields_raise_multiple_search_query_errors(
|
||||
self,
|
||||
query: str,
|
||||
query_index: tantivy.Index,
|
||||
) -> None:
|
||||
assert translate_query(query, UTC) == query
|
||||
with pytest.raises(MultipleSearchQueryErrors) as exc_info:
|
||||
parse_user_query(
|
||||
query_index,
|
||||
"created:notadate AND asn:notanumber",
|
||||
UTC,
|
||||
)
|
||||
assert len(exc_info.value.errors) == 2
|
||||
kinds = {type(e) for e in exc_info.value.errors}
|
||||
assert kinds == {InvalidDateQuery, InvalidNumberQuery}
|
||||
|
||||
def test_unknown_field_keyword_passes_through_unchanged(self) -> None:
|
||||
# foobar is not a date field: 'foobar:today' must not become a date range,
|
||||
# which Tantivy would otherwise reject as an unknown/typed field.
|
||||
assert translate_query("foobar:today", UTC) == "foobar:today"
|
||||
def test_asn_field_is_query_addressable(
|
||||
self,
|
||||
query_index: tantivy.Index,
|
||||
) -> None:
|
||||
q = parse_user_query(query_index, "asn:42", UTC)
|
||||
assert isinstance(q, tantivy.Query)
|
||||
|
||||
def test_checksum_field_is_query_addressable(
|
||||
self,
|
||||
query_index: tantivy.Index,
|
||||
) -> None:
|
||||
q = parse_user_query(query_index, "checksum:abc123", UTC)
|
||||
assert isinstance(q, tantivy.Query)
|
||||
|
||||
class TestPassthrough:
|
||||
"""Queries without field prefixes or unrelated content pass through unchanged."""
|
||||
|
||||
def test_bare_keyword_no_field_prefix_unchanged(self) -> None:
|
||||
# Bare 'today' with no field: prefix passes through unchanged
|
||||
result = translate_query("bank statement today", UTC)
|
||||
assert "today" in result
|
||||
|
||||
def test_unrelated_query_unchanged(self) -> None:
|
||||
assert translate_query("title:invoice", UTC) == "title:invoice"
|
||||
|
||||
|
||||
class TestNormalizeQuery:
|
||||
"""translate_query expands comma-separated values and collapses whitespace."""
|
||||
|
||||
def test_normalize_expands_comma_separated_tags(self) -> None:
|
||||
assert translate_query("tag:foo,bar", UTC) == "tag:foo AND tag:bar"
|
||||
|
||||
def test_normalize_comma_between_range_expressions(self) -> None:
|
||||
# Comma-separated field range expressions (Whoosh v2 syntax) must be
|
||||
# converted to AND so Tantivy does not receive an invalid comma.
|
||||
q = "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z],added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
|
||||
assert translate_query(q, UTC) == (
|
||||
"created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
|
||||
" AND "
|
||||
"added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
|
||||
)
|
||||
|
||||
def test_normalize_expands_three_values(self) -> None:
|
||||
assert (
|
||||
translate_query("tag:foo,bar,baz", UTC) == "tag:foo AND tag:bar AND tag:baz"
|
||||
)
|
||||
|
||||
def test_normalize_collapses_whitespace(self) -> None:
|
||||
assert translate_query("bank statement", UTC) == "bank statement"
|
||||
|
||||
def test_normalize_no_commas_unchanged(self) -> None:
|
||||
assert translate_query("bank statement", UTC) == "bank statement"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("raw", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
"h52.1 - kurzsichtigkeit",
|
||||
"h52.1 kurzsichtigkeit",
|
||||
id="icd_code_dash_description",
|
||||
),
|
||||
pytest.param(
|
||||
"H52.1 - asd",
|
||||
"H52.1 asd",
|
||||
id="icd_code_uppercase_dash",
|
||||
),
|
||||
pytest.param(
|
||||
"h52.1 -",
|
||||
"h52.1",
|
||||
id="trailing_minus",
|
||||
),
|
||||
pytest.param(
|
||||
". -",
|
||||
".",
|
||||
id="dot_trailing_minus",
|
||||
),
|
||||
pytest.param(
|
||||
"h52. -",
|
||||
"h52.",
|
||||
id="partial_code_trailing_minus",
|
||||
),
|
||||
pytest.param(
|
||||
"foo - bar - baz",
|
||||
"foo bar baz",
|
||||
id="multiple_dashes",
|
||||
),
|
||||
pytest.param(
|
||||
"foo + bar",
|
||||
"foo bar",
|
||||
id="spaced_plus_operator",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_normalize_strips_dangling_operators(self, raw: str, expected: str) -> None:
|
||||
assert translate_query(raw, UTC) == expected
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"query",
|
||||
[
|
||||
pytest.param("term -other", id="adjacent_not_operator"),
|
||||
pytest.param("-term", id="leading_not_operator"),
|
||||
pytest.param("+term", id="leading_must_operator"),
|
||||
pytest.param("foo -bar +baz", id="mixed_adjacent_operators"),
|
||||
],
|
||||
)
|
||||
def test_normalize_preserves_valid_operators(self, query: str) -> None:
|
||||
assert translate_query(query, UTC) == query
|
||||
def test_unregistered_id_field_folds_to_literal_text_not_error(
|
||||
self,
|
||||
query_index: tantivy.Index,
|
||||
) -> None:
|
||||
# tag_id is intentionally excluded from the FieldRegistry — whoosh-compat
|
||||
# parity leniency folds it into literal text, not a diagnostic/400.
|
||||
# A result-level assertion that this fold actually matches nothing
|
||||
# against real documents lives in
|
||||
# test_acceptance.py::TestUnregisteredIdFieldFoldsToLiteralText.
|
||||
q = parse_user_query(query_index, "tag_id:5", UTC)
|
||||
assert isinstance(q, tantivy.Query)
|
||||
|
||||
|
||||
class TestParseSimpleTextHighlightQuery:
|
||||
"""parse_simple_text_highlight_query must not raise on natural-language queries."""
|
||||
|
||||
@pytest.fixture
|
||||
def query_index(self) -> tantivy.Index:
|
||||
schema = build_schema()
|
||||
idx = tantivy.Index(schema, path=None)
|
||||
register_tokenizers(idx, "")
|
||||
return idx
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw_query",
|
||||
[
|
||||
@@ -884,3 +281,75 @@ class TestPermissionFilter:
|
||||
user = django_user_model(pk=20)
|
||||
perm = build_permission_filter(perm_index.schema, user)
|
||||
assert perm_index.searcher().search(perm, limit=10).count == 1 # only unowned
|
||||
|
||||
|
||||
class TestSearchQueryErrors:
|
||||
def test_invalid_date_query_is_a_search_query_error(self) -> None:
|
||||
err = InvalidDateQuery("created", "notadate")
|
||||
assert isinstance(err, SearchQueryError)
|
||||
assert err.field == "created"
|
||||
assert err.value == "notadate"
|
||||
assert "created" in str(err)
|
||||
assert "notadate" in str(err)
|
||||
|
||||
def test_invalid_number_query_is_a_search_query_error(self) -> None:
|
||||
err = InvalidNumberQuery("asn", "notanumber")
|
||||
assert isinstance(err, SearchQueryError)
|
||||
assert err.field == "asn"
|
||||
assert err.value == "notanumber"
|
||||
assert "asn" in str(err)
|
||||
assert "notanumber" in str(err)
|
||||
|
||||
def test_multiple_search_query_errors_aggregates(self) -> None:
|
||||
sub_errors = [
|
||||
InvalidDateQuery("created", "notadate"),
|
||||
InvalidNumberQuery("asn", "notanumber"),
|
||||
]
|
||||
err = MultipleSearchQueryErrors(sub_errors)
|
||||
assert isinstance(err, SearchQueryError)
|
||||
assert err.errors == tuple(sub_errors)
|
||||
assert "created" in str(err)
|
||||
assert "asn" in str(err)
|
||||
|
||||
|
||||
class TestEmitErrorContract:
|
||||
"""A diagnostics list, or a QueryEmitError/UnsupportedQueryError from
|
||||
emit(), are both user-input errors and must surface as
|
||||
SearchQueryError (HTTP 400), with library-internal wording stripped
|
||||
from the message."""
|
||||
|
||||
def test_query_emit_error_maps_to_search_query_error(
|
||||
self,
|
||||
query_index: tantivy.Index,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from whoosh_compat.errors import QueryEmitError
|
||||
|
||||
import documents.search._query as query_mod
|
||||
|
||||
def raise_emit_error(*args: object, **kwargs: object) -> None:
|
||||
raise QueryEmitError("synthetic emit failure")
|
||||
|
||||
monkeypatch.setattr(query_mod, "tantivy_emit", raise_emit_error)
|
||||
with pytest.raises(SearchQueryError):
|
||||
parse_user_query(query_index, "invoice", UTC)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("query", "leaked_fragment"),
|
||||
[
|
||||
pytest.param("title:[a TO b]", "DIVERGENCES", id="text-range-doc-ref"),
|
||||
pytest.param("notes.note:wild*", "DIVERGENCES", id="json-wildcard-doc-ref"),
|
||||
pytest.param("notes.user:*", "fast=True", id="exists-host-advice"),
|
||||
],
|
||||
)
|
||||
def test_unsupported_messages_carry_no_internal_vocabulary(
|
||||
self,
|
||||
query_index: tantivy.Index,
|
||||
query: str,
|
||||
leaked_fragment: str,
|
||||
) -> None:
|
||||
with pytest.raises(SearchQueryError) as exc_info:
|
||||
parse_user_query(query_index, query, UTC)
|
||||
assert leaked_fragment not in str(exc_info.value)
|
||||
# The message must still say something useful, not be blanked.
|
||||
assert str(exc_info.value).strip()
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
import pytest
|
||||
from whoosh_compat import FieldKind
|
||||
from whoosh_compat import FieldRegistry
|
||||
from whoosh_compat.fields import ResolvedField
|
||||
|
||||
from documents.search._fields import PUBLIC_FIELDS
|
||||
from documents.search._registry import get_field_registry
|
||||
|
||||
_BY_NAME = {f.name: f for f in PUBLIC_FIELDS}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def registry() -> FieldRegistry:
|
||||
return get_field_registry(None)
|
||||
|
||||
|
||||
def _resolve(registry: FieldRegistry, name: str) -> ResolvedField:
|
||||
ref = registry.make_ref(name)
|
||||
assert ref is not None, f"{name} is not a valid field ref"
|
||||
resolved = registry.resolve(ref)
|
||||
assert resolved is not None, f"{name} did not resolve"
|
||||
return resolved
|
||||
|
||||
|
||||
class TestFieldRegistry:
|
||||
def test_internal_id_fields_are_not_registered(
|
||||
self,
|
||||
registry: FieldRegistry,
|
||||
) -> None:
|
||||
for name in (
|
||||
"tag_id",
|
||||
"owner_id",
|
||||
"viewer_id",
|
||||
"correspondent_id",
|
||||
"document_type_id",
|
||||
"storage_path_id",
|
||||
"viewer_group_id",
|
||||
):
|
||||
assert name not in registry
|
||||
|
||||
def test_type_alias_resolves_to_document_type(
|
||||
self,
|
||||
registry: FieldRegistry,
|
||||
) -> None:
|
||||
assert _resolve(registry, "type").spec.name == "document_type"
|
||||
|
||||
def test_path_alias_resolves_to_storage_path(self, registry: FieldRegistry) -> None:
|
||||
assert _resolve(registry, "path").spec.name == "storage_path"
|
||||
|
||||
def test_notes_json_subpaths_resolve(self, registry: FieldRegistry) -> None:
|
||||
resolved = _resolve(registry, "notes.user")
|
||||
assert resolved.spec.name == "notes"
|
||||
assert resolved.json_path == "user"
|
||||
assert resolved.is_subpath is True
|
||||
|
||||
def test_custom_fields_json_subpaths_resolve(self, registry: FieldRegistry) -> None:
|
||||
for raw in ("custom_fields.name", "custom_fields.value"):
|
||||
_resolve(registry, raw)
|
||||
|
||||
def test_unregistered_json_subpath_does_not_resolve(
|
||||
self,
|
||||
registry: FieldRegistry,
|
||||
) -> None:
|
||||
# An unregistered subpath is not even a valid FieldRef: make_ref
|
||||
# returns None for a dotted name whose subpath isn't registered
|
||||
# (it doesn't produce a ref for resolve() to then reject).
|
||||
assert registry.make_ref("notes.bogus") is None
|
||||
|
||||
def test_tag_is_comma_values(self, registry: FieldRegistry) -> None:
|
||||
assert _resolve(registry, "tag").spec.comma_values is True
|
||||
|
||||
def test_created_is_date_kind(self, registry: FieldRegistry) -> None:
|
||||
resolved = _resolve(registry, "created")
|
||||
assert resolved.spec.kind is FieldKind.DATE
|
||||
assert resolved.spec.date_only is True
|
||||
|
||||
def test_analyzer_lowercases_and_ascii_folds(self, registry: FieldRegistry) -> None:
|
||||
# title uses the paperless_text analyzer: simple -> remove_long ->
|
||||
# lowercase -> ascii_fold [-> stemmer]. With no language configured
|
||||
# (None), no stemmer runs, so "Café" folds to the single token "cafe".
|
||||
resolved = _resolve(registry, "title")
|
||||
assert resolved.spec.analyzer is not None
|
||||
assert resolved.spec.analyzer("Café") == ["cafe"]
|
||||
|
||||
def test_checksum_analyzer_is_identity_single_token(
|
||||
self,
|
||||
registry: FieldRegistry,
|
||||
) -> None:
|
||||
# checksum uses the raw tokenizer at index time (no splitting).
|
||||
resolved = _resolve(registry, "checksum")
|
||||
assert resolved.spec.analyzer is not None
|
||||
assert resolved.spec.analyzer("ABC-123") == ["ABC-123"]
|
||||
|
||||
def test_pattern_normalizer_is_ascii_fold_only_no_stemming(
|
||||
self,
|
||||
registry: FieldRegistry,
|
||||
) -> None:
|
||||
resolved = _resolve(registry, "title")
|
||||
assert resolved.spec.pattern_normalizer is not None
|
||||
# "running" must NOT be stemmed to "run" by the pattern normalizer,
|
||||
# only case/accent-folded — even with English stemming configured.
|
||||
registry_en = get_field_registry("en")
|
||||
resolved_en = _resolve(registry_en, "title")
|
||||
assert resolved_en.spec.pattern_normalizer is not None
|
||||
assert resolved_en.spec.pattern_normalizer("Running") == "running"
|
||||
|
||||
def test_registry_is_cached_per_language(self) -> None:
|
||||
a = get_field_registry("en")
|
||||
b = get_field_registry("en")
|
||||
assert a is b
|
||||
|
||||
def test_registry_rebuilds_on_language_change(self) -> None:
|
||||
a = get_field_registry("en")
|
||||
b = get_field_registry("de")
|
||||
assert a is not b
|
||||
|
||||
|
||||
class TestJsonSubpathCoupling:
|
||||
"""Guards PUBLIC_FIELDS' JSON subpaths against drifting from the literal
|
||||
dict keys _backend.py::_build_tantivy_doc writes. These assertions
|
||||
hardcode the expected key sets rather than introspecting _build_tantivy_doc
|
||||
(its dict keys are string literals with no importable symbol) — if someone
|
||||
changes _build_tantivy_doc's JSON keys without updating this test too, it
|
||||
will pass despite the drift. Best-effort, not a structural guarantee.
|
||||
"""
|
||||
|
||||
def test_notes_dict_keys_match_public_fields_subpaths(self) -> None:
|
||||
# _backend.py's _build_tantivy_doc builds:
|
||||
# doc.add_json("notes", {"note": ..., "user": ...})
|
||||
# These literal keys must match PUBLIC_FIELDS' "notes" subpaths exactly.
|
||||
assert set(_BY_NAME["notes"].subpaths) == {"note", "user"}
|
||||
|
||||
def test_custom_fields_dict_keys_match_public_fields_subpaths(self) -> None:
|
||||
# _backend.py's _build_tantivy_doc builds:
|
||||
# doc.add_json("custom_fields", {"name": ..., "value": ...})
|
||||
assert set(_BY_NAME["custom_fields"].subpaths) == {"name", "value"}
|
||||
@@ -1,12 +1,18 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
import tantivy
|
||||
|
||||
from documents.search._fields import PUBLIC_FIELDS
|
||||
from documents.search._schema import SCHEMA_VERSION
|
||||
from documents.search._schema import build_schema
|
||||
from documents.search._schema import needs_rebuild
|
||||
from documents.search._tokenizer import register_tokenizers
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
@@ -76,3 +82,68 @@ class TestNeedsRebuild:
|
||||
json.dumps({"schema_version": SCHEMA_VERSION, "language": "en"}),
|
||||
)
|
||||
assert needs_rebuild(index_dir) is True
|
||||
|
||||
|
||||
def _schema_fields(schema: tantivy.Schema) -> dict[str, dict]:
|
||||
"""{name: field-state} for every field declared on a tantivy Schema.
|
||||
|
||||
tantivy-py 0.26 exposes no public introspection API on Schema (no
|
||||
__iter__, get_field, to_json, etc.) -- __reduce__() (used internally for
|
||||
pickling) is the only way to recover the field list, so we lean on it
|
||||
here for test assertions only.
|
||||
"""
|
||||
state = schema.__reduce__()[1][0]
|
||||
return {field["name"]: field for field in state["inner"]}
|
||||
|
||||
|
||||
class TestSchemaMatchesPublicFields:
|
||||
def test_every_public_field_is_in_the_schema(self) -> None:
|
||||
schema = build_schema()
|
||||
schema_field_names = set(_schema_fields(schema))
|
||||
for field in PUBLIC_FIELDS:
|
||||
assert field.name in schema_field_names, (
|
||||
f"{field.name} is in PUBLIC_FIELDS but missing from build_schema()"
|
||||
)
|
||||
|
||||
def test_asn_page_count_num_notes_are_fast_unsigned_fields(self) -> None:
|
||||
# Spot-check kind-derived construction for the U64 fields.
|
||||
schema = build_schema()
|
||||
doc = tantivy.Document()
|
||||
doc.add_unsigned("id", 1)
|
||||
doc.add_text("checksum", "x")
|
||||
doc.add_unsigned("asn", 42)
|
||||
doc.add_unsigned("page_count", 3)
|
||||
doc.add_unsigned("num_notes", 0)
|
||||
doc.add_date("created", datetime(2020, 1, 1, tzinfo=UTC))
|
||||
doc.add_date("modified", datetime(2020, 1, 1, tzinfo=UTC))
|
||||
doc.add_date("added", datetime(2020, 1, 1, tzinfo=UTC))
|
||||
index = tantivy.Index(schema)
|
||||
register_tokenizers(index, None)
|
||||
writer = index.writer()
|
||||
writer.add_document(doc)
|
||||
writer.commit()
|
||||
index.reload()
|
||||
searcher = index.searcher()
|
||||
results = searcher.search(tantivy.Query.term_query(schema, "asn", 42), limit=1)
|
||||
assert len(results.hits) == 1
|
||||
|
||||
|
||||
class TestFastFlagAgreement:
|
||||
def test_every_public_field_fast_flag_matches_the_built_schema(self) -> None:
|
||||
# whoosh-compat's registry trusts PUBLIC_FIELDS' fast flag when resolving
|
||||
# field:* existence checks (its FAST_FIELD strategy); a fast=True
|
||||
# entry whose actual tantivy column is not fast would make those
|
||||
# searches silently match nothing at search time. build_schema()
|
||||
# only honors the flag in its U64 and DATE branches today, so this
|
||||
# pins the agreement for EVERY kind: a future fast=True
|
||||
# TEXT/KEYWORD/JSON entry the builder silently ignores fails here
|
||||
# instead of at a user's query.
|
||||
schema_fast = {
|
||||
name: bool(field["options"].get("fast", False))
|
||||
for name, field in _schema_fields(build_schema()).items()
|
||||
}
|
||||
for public_field in PUBLIC_FIELDS:
|
||||
assert schema_fast[public_field.name] == public_field.fast, (
|
||||
f"{public_field.name}: PUBLIC_FIELDS says fast={public_field.fast} but the"
|
||||
f" built schema says fast={schema_fast[public_field.name]}"
|
||||
)
|
||||
|
||||
@@ -7,8 +7,8 @@ import pytest
|
||||
import tantivy
|
||||
|
||||
from documents.search._tokenizer import _bigram_analyzer
|
||||
from documents.search._tokenizer import _paperless_text
|
||||
from documents.search._tokenizer import _simple_search_analyzer
|
||||
from documents.search._tokenizer import paperless_text_analyzer
|
||||
from documents.search._tokenizer import register_tokenizers
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -25,7 +25,7 @@ class TestTokenizers:
|
||||
sb.add_text_field("content", stored=True, tokenizer_name="paperless_text")
|
||||
schema = sb.build()
|
||||
idx = tantivy.Index(schema, path=None)
|
||||
idx.register_tokenizer("paperless_text", _paperless_text(""))
|
||||
idx.register_tokenizer("paperless_text", paperless_text_analyzer(""))
|
||||
return idx
|
||||
|
||||
@pytest.fixture
|
||||
|
||||
@@ -1,810 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import pytest
|
||||
import time_machine
|
||||
|
||||
from documents.search._dates import _precision_bounds
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tantivy
|
||||
from documents.search._query import _FIELD_BOOSTS
|
||||
from documents.search._query import DEFAULT_SEARCH_FIELDS
|
||||
from documents.search._translate import OPEN_HI
|
||||
from documents.search._translate import OPEN_LO
|
||||
from documents.search._translate import Comma
|
||||
from documents.search._translate import FieldRange
|
||||
from documents.search._translate import FieldValue
|
||||
from documents.search._translate import FieldValueList
|
||||
from documents.search._translate import InvalidDateQuery
|
||||
from documents.search._translate import Passthrough
|
||||
from documents.search._translate import resolve_commas
|
||||
from documents.search._translate import scan
|
||||
from documents.search._translate import translate_query
|
||||
from documents.search._translate import translate_range
|
||||
from documents.search._translate import translate_scalar
|
||||
|
||||
|
||||
@pytest.mark.search
|
||||
class TestPrecisionBounds:
|
||||
@pytest.mark.parametrize(
|
||||
("digits", "expected"),
|
||||
[
|
||||
("2020", ((2020, 1, 1), (2021, 1, 1))),
|
||||
("202003", ((2020, 3, 1), (2020, 4, 1))),
|
||||
("202012", ((2020, 12, 1), (2021, 1, 1))),
|
||||
("20200115", ((2020, 1, 15), (2020, 1, 16))),
|
||||
("20201231", ((2020, 12, 31), (2021, 1, 1))),
|
||||
],
|
||||
)
|
||||
def test_valid(self, digits, expected):
|
||||
lo, hi = _precision_bounds(digits)
|
||||
assert (lo.year, lo.month, lo.day) == expected[0]
|
||||
assert (hi.year, hi.month, hi.day) == expected[1]
|
||||
|
||||
@pytest.mark.parametrize("digits", ["202023", "20200230", "20201301", "20", "abcd"])
|
||||
def test_invalid_returns_none(self, digits):
|
||||
assert _precision_bounds(digits) is None
|
||||
|
||||
|
||||
@pytest.mark.search
|
||||
class TestScan:
|
||||
def test_plain_words_are_passthrough(self):
|
||||
assert scan("bank statement") == [Passthrough("bank statement")]
|
||||
|
||||
def test_field_value(self):
|
||||
assert scan("created:2020") == [FieldValue("created", "2020")]
|
||||
|
||||
def test_field_value_in_boolean(self):
|
||||
toks = scan("created:2020 OR foo")
|
||||
assert toks == [
|
||||
FieldValue("created", "2020"),
|
||||
Passthrough(" OR foo"),
|
||||
]
|
||||
|
||||
def test_field_value_in_parens(self):
|
||||
toks = scan("(created:2020 OR foo)")
|
||||
assert toks == [
|
||||
Passthrough("("),
|
||||
FieldValue("created", "2020"),
|
||||
Passthrough(" OR foo)"),
|
||||
]
|
||||
|
||||
def test_quoted_value(self):
|
||||
assert scan('correspondent:"A B"') == [FieldValue("correspondent", '"A B"')]
|
||||
|
||||
def test_field_range(self):
|
||||
assert scan("created:[2020 TO 2021]") == [
|
||||
FieldRange("created", "[", "2020", "2021", "]"),
|
||||
]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("query", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
"created:[2020 to]",
|
||||
FieldRange("created", "[", "2020", "", "]"),
|
||||
id="open_upper",
|
||||
),
|
||||
pytest.param(
|
||||
"created:[to 2020]",
|
||||
FieldRange("created", "[", "", "2020", "]"),
|
||||
id="open_lower",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_open_range(self, query, expected):
|
||||
assert scan(query) == [expected]
|
||||
|
||||
def test_comma_inside_range_not_split(self):
|
||||
# No depth-0 comma here; the whole thing is one range token.
|
||||
toks = scan("created:[2020 TO 2021]")
|
||||
assert len(toks) == 1
|
||||
|
||||
# --- Edge-case / regression tests (scan must never raise) ---
|
||||
|
||||
def test_url_is_passthrough(self):
|
||||
# "http" is not a known field; the whole URL must pass through verbatim.
|
||||
assert scan("http://example.com") == [Passthrough("http://example.com")]
|
||||
|
||||
def test_unterminated_quote_is_passthrough(self):
|
||||
# title is a known field but the quoted value has no closing quote;
|
||||
# _consume_value returns None so the whole string falls into passthrough.
|
||||
assert scan('title:"abc') == [Passthrough('title:"abc')]
|
||||
|
||||
def test_unterminated_bracket_is_passthrough(self):
|
||||
# created is a known field but the range bracket is never closed;
|
||||
# _consume_range returns None so the whole string falls into passthrough.
|
||||
assert scan("created:[2020") == [Passthrough("created:[2020")]
|
||||
|
||||
def test_empty_value_at_end_is_passthrough(self):
|
||||
# created is a known field but there is no value after the colon
|
||||
# (_consume_value returns None for start >= n), so passthrough.
|
||||
assert scan("created:") == [Passthrough("created:")]
|
||||
|
||||
def test_value_containing_colon(self):
|
||||
# The bare-word value reader stops at whitespace/paren, not at colon,
|
||||
# so "2020:30" is consumed as a single value token.
|
||||
assert scan("created:2020:30") == [FieldValue("created", "2020:30")]
|
||||
|
||||
def test_comma_followed_by_unconsumable_value_stops(self):
|
||||
# A comma followed by whitespace is neither a value-list continuation nor a
|
||||
# clause separator: the value stops and the comma stays as passthrough.
|
||||
assert scan("tag:foo, bar") == [
|
||||
FieldValue("tag", "foo"),
|
||||
Passthrough(", bar"),
|
||||
]
|
||||
|
||||
def test_bracket_without_to_is_open_upper_bound(self):
|
||||
# A bracketed value with no TO falls back to (value, "") -> open upper bound.
|
||||
assert scan("created:[2020]") == [
|
||||
FieldRange("created", "[", "2020", "", "]"),
|
||||
]
|
||||
|
||||
def test_known_field_name_midword_is_passthrough(self):
|
||||
# A known field name embedded mid-word is not a field token (the
|
||||
# word-boundary guard); the whole run stays passthrough.
|
||||
assert scan("xtag:foo") == [Passthrough("xtag:foo")]
|
||||
|
||||
|
||||
@pytest.mark.search
|
||||
class TestCommaResolution:
|
||||
def test_value_list_multi_value_field(self):
|
||||
toks = resolve_commas(scan("tag:foo,bar"))
|
||||
assert toks == [FieldValueList("tag", ("foo", "bar"))]
|
||||
|
||||
def test_value_list_three(self):
|
||||
toks = resolve_commas(scan("tag_id:1,2,3"))
|
||||
assert toks == [FieldValueList("tag_id", ("1", "2", "3"))]
|
||||
|
||||
def test_text_field_comma_is_literal(self):
|
||||
# correspondent is not multi-value: comma stays inside the value.
|
||||
toks = resolve_commas(scan("correspondent:foo,bar"))
|
||||
assert toks == [FieldValue("correspondent", "foo,bar")]
|
||||
|
||||
def test_clause_separator_before_known_field(self):
|
||||
toks = resolve_commas(scan("tag:foo,type:bar"))
|
||||
assert toks == [FieldValue("tag", "foo"), Comma(), FieldValue("type", "bar")]
|
||||
|
||||
def test_clause_separator_after_range(self):
|
||||
toks = resolve_commas(scan("created:[2020 TO 2021],added:[2022 TO 2023]"))
|
||||
assert toks == [
|
||||
FieldRange("created", "[", "2020", "2021", "]"),
|
||||
Comma(),
|
||||
FieldRange("added", "[", "2022", "2023", "]"),
|
||||
]
|
||||
|
||||
def test_clause_separator_after_quote(self):
|
||||
toks = resolve_commas(scan('correspondent:"A B",created:[2020 TO 2021]'))
|
||||
assert toks == [
|
||||
FieldValue("correspondent", '"A B"'),
|
||||
Comma(),
|
||||
FieldRange("created", "[", "2020", "2021", "]"),
|
||||
]
|
||||
|
||||
def test_url_comma_is_literal_passthrough(self):
|
||||
toks = resolve_commas(scan("http://example.com/a,b"))
|
||||
assert toks == [Passthrough("http://example.com/a,b")]
|
||||
|
||||
def test_non_multi_value_comma_is_literal(self):
|
||||
# title is not in MULTI_VALUE_FIELDS: comma stays inside the value.
|
||||
toks = resolve_commas(scan("title:10,20"))
|
||||
assert toks == [FieldValue("title", "10,20")]
|
||||
|
||||
def test_clause_separator_before_known_date_field(self):
|
||||
# The comma between a bare value and a known date field acts as a
|
||||
# clause separator; both sides survive as distinct tokens.
|
||||
toks = resolve_commas(scan("correspondent:foo,created:[2020 TO 2021]"))
|
||||
assert toks == [
|
||||
FieldValue("correspondent", "foo"),
|
||||
Comma(),
|
||||
FieldRange("created", "[", "2020", "2021", "]"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.search
|
||||
class TestTranslateScalar:
|
||||
@pytest.mark.parametrize(
|
||||
("field", "value", "expected"),
|
||||
[
|
||||
(
|
||||
"created",
|
||||
"2020",
|
||||
"created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}",
|
||||
),
|
||||
(
|
||||
"created",
|
||||
"202003",
|
||||
"created:[2020-03-01T00:00:00Z TO 2020-04-01T00:00:00Z}",
|
||||
),
|
||||
(
|
||||
"created",
|
||||
"20200115",
|
||||
"created:[2020-01-15T00:00:00Z TO 2020-01-16T00:00:00Z}",
|
||||
),
|
||||
(
|
||||
"created",
|
||||
"2020-01-15",
|
||||
"created:[2020-01-15T00:00:00Z TO 2020-01-16T00:00:00Z}",
|
||||
),
|
||||
(
|
||||
"created",
|
||||
"2020-03",
|
||||
"created:[2020-03-01T00:00:00Z TO 2020-04-01T00:00:00Z}",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_partial_and_iso_dates(self, field: str, value: str, expected: str) -> None:
|
||||
assert translate_scalar(field, value, UTC) == expected
|
||||
|
||||
def test_invalid_date_raises(self) -> None:
|
||||
with pytest.raises(InvalidDateQuery) as exc_info:
|
||||
translate_scalar("created", "202023", UTC)
|
||||
assert exc_info.value.field == "created"
|
||||
assert exc_info.value.value == "202023"
|
||||
|
||||
def test_keyword_delegates(self) -> None:
|
||||
# keyword path produces a half-open range; just assert it is a created range
|
||||
out = translate_scalar("created", "today", UTC)
|
||||
assert out.startswith("created:[") and out.endswith("}")
|
||||
|
||||
def test_14digit_compact_datetime(self) -> None:
|
||||
out = translate_scalar("created", "20240115120000", UTC)
|
||||
assert "20240115120000" not in out
|
||||
assert out.startswith("created:")
|
||||
assert out == "created:[2024-01-15T12:00:00Z TO 2024-01-15T12:00:00Z]"
|
||||
|
||||
def test_14digit_invalid_month_raises(self) -> None:
|
||||
with pytest.raises(InvalidDateQuery) as exc_info:
|
||||
translate_scalar("created", "20231300120000", UTC)
|
||||
assert exc_info.value.field == "created"
|
||||
assert exc_info.value.value == "20231300120000"
|
||||
|
||||
def test_unrecognized_value_raises(self) -> None:
|
||||
# A value that is not a keyword, digits, ISO date, or compact timestamp
|
||||
# raises rather than producing invalid Tantivy syntax or silently matching
|
||||
# nothing.
|
||||
with pytest.raises(InvalidDateQuery) as exc_info:
|
||||
translate_scalar("created", "garbage", UTC)
|
||||
assert exc_info.value.field == "created"
|
||||
assert exc_info.value.value == "garbage"
|
||||
|
||||
|
||||
@pytest.mark.search
|
||||
class TestTranslateRange:
|
||||
@pytest.mark.parametrize(
|
||||
("lo", "hi", "expected"),
|
||||
[
|
||||
("2005", "2009", "created:[2005-01-01T00:00:00Z TO 2010-01-01T00:00:00Z}"),
|
||||
(
|
||||
"202001",
|
||||
"202006",
|
||||
"created:[2020-01-01T00:00:00Z TO 2020-07-01T00:00:00Z}",
|
||||
),
|
||||
(
|
||||
"20200101",
|
||||
"20201231",
|
||||
"created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}",
|
||||
),
|
||||
(
|
||||
"2020-01-01",
|
||||
"2020-12-31",
|
||||
"created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_absolute_ranges(self, lo, hi, expected):
|
||||
assert translate_range("created", lo, hi, UTC) == expected
|
||||
|
||||
def test_reversed_swaps(self):
|
||||
assert translate_range("created", "2009", "2005", UTC) == (
|
||||
"created:[2005-01-01T00:00:00Z TO 2010-01-01T00:00:00Z}"
|
||||
)
|
||||
|
||||
def test_open_upper(self):
|
||||
out = translate_range("created", "2020", "", UTC)
|
||||
assert out == f"created:[2020-01-01T00:00:00Z TO {OPEN_HI}]"
|
||||
|
||||
def test_open_lower(self):
|
||||
out = translate_range("created", "", "2020", UTC)
|
||||
assert out == f"created:[{OPEN_LO} TO 2021-01-01T00:00:00Z}}"
|
||||
|
||||
def test_invalid_bound_raises(self):
|
||||
with pytest.raises(InvalidDateQuery) as exc_info:
|
||||
translate_range("created", "202023", "2025", UTC)
|
||||
assert exc_info.value.field == "created"
|
||||
assert exc_info.value.value == "202023"
|
||||
|
||||
def test_invalid_high_bound_raises(self):
|
||||
# Low bound parses, high bound does not -> raise on the high bound.
|
||||
with pytest.raises(InvalidDateQuery) as exc_info:
|
||||
translate_range("created", "2020", "garbage", UTC)
|
||||
assert exc_info.value.field == "created"
|
||||
assert exc_info.value.value == "garbage"
|
||||
|
||||
|
||||
@pytest.mark.search
|
||||
class TestTranslateQuery:
|
||||
@pytest.mark.parametrize(
|
||||
("raw", "expected"),
|
||||
[
|
||||
(
|
||||
"created:2020",
|
||||
"created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}",
|
||||
),
|
||||
("tag:foo,bar", "tag:foo AND tag:bar"),
|
||||
# 'type' is a user-facing alias rewritten to 'document_type' (the real schema field)
|
||||
("tag:foo,type:bar", "tag:foo AND document_type:bar"),
|
||||
(
|
||||
"created:[2020 TO 2021],added:[2022 TO 2023]",
|
||||
(
|
||||
"created:[2020-01-01T00:00:00Z TO 2022-01-01T00:00:00Z}"
|
||||
" AND "
|
||||
"added:[2022-01-01T00:00:00Z TO 2024-01-01T00:00:00Z}"
|
||||
),
|
||||
),
|
||||
# correspondent is not multi-value: comma stays literal inside the value
|
||||
("correspondent:foo,bar", "correspondent:foo,bar"),
|
||||
],
|
||||
)
|
||||
def test_golden(self, raw: str, expected: str) -> None:
|
||||
assert translate_query(raw, UTC) == expected
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw",
|
||||
[
|
||||
"created:2020",
|
||||
"created:202003",
|
||||
"created:[20200101 TO 20201231]",
|
||||
"created:[2020-01-01 TO 2020-12-31]",
|
||||
"created:[2020 to]",
|
||||
"created:[to 2020]",
|
||||
"title:x,created:[2020 TO 2021]",
|
||||
"created:2020 OR foo",
|
||||
"(created:2020 OR invoice)",
|
||||
"tag:foo,type:bar",
|
||||
"bank statement",
|
||||
],
|
||||
)
|
||||
def test_parse_acceptance(self, index: tantivy.Index, raw: str) -> None:
|
||||
translated = translate_query(raw, UTC)
|
||||
# Must not raise:
|
||||
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
|
||||
|
||||
|
||||
@pytest.mark.search
|
||||
class TestFieldAliasing:
|
||||
"""Whoosh->Tantivy field-name aliasing (type/path -> document_type/storage_path)."""
|
||||
|
||||
def test_type_alias(self) -> None:
|
||||
assert translate_query("type:invoice", UTC) == "document_type:invoice"
|
||||
|
||||
def test_path_alias(self) -> None:
|
||||
assert translate_query("path:/foo/bar", UTC) == "storage_path:/foo/bar"
|
||||
|
||||
def test_type_id_alias(self) -> None:
|
||||
assert translate_query("type_id:5", UTC) == "document_type_id:5"
|
||||
|
||||
def test_path_id_alias(self) -> None:
|
||||
assert translate_query("path_id:7", UTC) == "storage_path_id:7"
|
||||
|
||||
def test_clause_separator_plus_alias(self) -> None:
|
||||
# Comma between known fields acts as AND separator; alias still applied.
|
||||
assert (
|
||||
translate_query("tag:foo,type:bar", UTC) == "tag:foo AND document_type:bar"
|
||||
)
|
||||
|
||||
def test_type_range_alias(self) -> None:
|
||||
# type is not a date field; range passes through verbatim with alias applied.
|
||||
assert (
|
||||
translate_query("type:[2020 TO 2021]", UTC)
|
||||
== "document_type:[2020 TO 2021]"
|
||||
)
|
||||
|
||||
def test_parse_acceptance_type(self, index: tantivy.Index) -> None:
|
||||
# Translated output must be accepted by the real Tantivy parser.
|
||||
translated = translate_query("type:invoice", UTC)
|
||||
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
|
||||
|
||||
def test_parse_acceptance_path(self, index: tantivy.Index) -> None:
|
||||
translated = translate_query("path:foo", UTC)
|
||||
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
|
||||
|
||||
|
||||
# Freeze time so relative-date tests are deterministic.
|
||||
_FROZEN_NOW = datetime(2026, 3, 28, 12, 0, 0, tzinfo=UTC)
|
||||
|
||||
|
||||
@pytest.mark.search
|
||||
class TestRelativeRanges:
|
||||
"""Relative date-range tokens resolved against a frozen clock."""
|
||||
|
||||
@time_machine.travel(_FROZEN_NOW, tick=False)
|
||||
def test_minus_7_days_to_now(self) -> None:
|
||||
assert translate_query("added:[-7 days to now]", UTC) == (
|
||||
"added:[2026-03-21T12:00:00Z TO 2026-03-28T12:00:00Z]"
|
||||
)
|
||||
|
||||
@time_machine.travel(_FROZEN_NOW, tick=False)
|
||||
def test_minus_1_week_to_now(self) -> None:
|
||||
assert translate_query("added:[-1 week to now]", UTC) == (
|
||||
"added:[2026-03-21T12:00:00Z TO 2026-03-28T12:00:00Z]"
|
||||
)
|
||||
|
||||
@time_machine.travel(_FROZEN_NOW, tick=False)
|
||||
def test_minus_1_month_to_now(self) -> None:
|
||||
assert translate_query("created:[-1 month to now]", UTC) == (
|
||||
"created:[2026-02-28T12:00:00Z TO 2026-03-28T12:00:00Z]"
|
||||
)
|
||||
|
||||
@time_machine.travel(_FROZEN_NOW, tick=False)
|
||||
def test_minus_1_year_to_now(self) -> None:
|
||||
assert translate_query("modified:[-1 year to now]", UTC) == (
|
||||
"modified:[2025-03-28T12:00:00Z TO 2026-03-28T12:00:00Z]"
|
||||
)
|
||||
|
||||
@time_machine.travel(_FROZEN_NOW, tick=False)
|
||||
def test_minus_3_hours_to_now(self) -> None:
|
||||
assert translate_query("added:[-3 hours to now]", UTC) == (
|
||||
"added:[2026-03-28T09:00:00Z TO 2026-03-28T12:00:00Z]"
|
||||
)
|
||||
|
||||
@time_machine.travel(_FROZEN_NOW, tick=False)
|
||||
def test_uppercase_units(self) -> None:
|
||||
assert translate_query("added:[-1 WEEK TO NOW]", UTC) == (
|
||||
"added:[2026-03-21T12:00:00Z TO 2026-03-28T12:00:00Z]"
|
||||
)
|
||||
|
||||
@time_machine.travel(_FROZEN_NOW, tick=False)
|
||||
def test_now_minus_7d_compact(self) -> None:
|
||||
assert translate_query("added:[now-7d TO now]", UTC) == (
|
||||
"added:[2026-03-21T12:00:00Z TO 2026-03-28T12:00:00Z]"
|
||||
)
|
||||
|
||||
@time_machine.travel(_FROZEN_NOW, tick=False)
|
||||
def test_reversed_range_swapped(self) -> None:
|
||||
# now+1h TO now-1h is reversed; translate_range swaps -> lo=now-1h, hi=now+1h
|
||||
assert translate_query("added:[now+1h TO now-1h]", UTC) == (
|
||||
"added:[2026-03-28T11:00:00Z TO 2026-03-28T13:00:00Z]"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw",
|
||||
[
|
||||
"added:[-7 days to now]",
|
||||
"added:[-1 week to now]",
|
||||
"created:[-1 month to now]",
|
||||
"modified:[-1 year to now]",
|
||||
"added:[-3 hours to now]",
|
||||
"added:[now-7d TO now]",
|
||||
"added:[now+1h TO now-1h]",
|
||||
],
|
||||
)
|
||||
@time_machine.travel(_FROZEN_NOW, tick=False)
|
||||
def test_parse_acceptance(self, index: tantivy.Index, raw: str) -> None:
|
||||
translated = translate_query(raw, UTC)
|
||||
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
|
||||
|
||||
|
||||
@pytest.mark.search
|
||||
class TestWhooshUnitAbbreviations:
|
||||
"""
|
||||
Whoosh's PlusMinus date grammar accepted abbreviated unit spellings
|
||||
(e.g. "yrs", "mos", "wks", "hrs", "mins", "secs"); saved views/searches
|
||||
created under the old Whoosh backend can contain those tokens (see
|
||||
https://github.com/paperless-ngx/paperless-ngx/issues/13482), so the
|
||||
Tantivy translator must still accept them.
|
||||
"""
|
||||
|
||||
@time_machine.travel(_FROZEN_NOW, tick=False)
|
||||
def test_minus_999_yrs(self) -> None:
|
||||
assert translate_query("created:[-999yrs to now]", UTC) == (
|
||||
"created:[1027-03-28T12:00:00Z TO 2026-03-28T12:00:00Z]"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("token", "expected_lo"),
|
||||
[
|
||||
("-1y", "2025-03-28T12:00:00Z"),
|
||||
("-1yr", "2025-03-28T12:00:00Z"),
|
||||
("-3mos", "2025-12-28T12:00:00Z"),
|
||||
("-3mo", "2025-12-28T12:00:00Z"),
|
||||
("-2wks", "2026-03-14T12:00:00Z"),
|
||||
("-2wk", "2026-03-14T12:00:00Z"),
|
||||
("-5dys", "2026-03-23T12:00:00Z"),
|
||||
("-5dy", "2026-03-23T12:00:00Z"),
|
||||
("-1hrs", "2026-03-28T11:00:00Z"),
|
||||
("-1hr", "2026-03-28T11:00:00Z"),
|
||||
("-10mins", "2026-03-28T11:50:00Z"),
|
||||
("-10min", "2026-03-28T11:50:00Z"),
|
||||
("-30secs", "2026-03-28T11:59:30Z"),
|
||||
("-30sec", "2026-03-28T11:59:30Z"),
|
||||
],
|
||||
)
|
||||
@time_machine.travel(_FROZEN_NOW, tick=False)
|
||||
def test_abbreviated_units(self, token: str, expected_lo: str) -> None:
|
||||
assert translate_query(f"added:[{token} to now]", UTC) == (
|
||||
f"added:[{expected_lo} TO 2026-03-28T12:00:00Z]"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw",
|
||||
[
|
||||
"created:[-999yrs to now]",
|
||||
"added:[-1y to now]",
|
||||
"created:[-3mos to now]",
|
||||
"added:[-2wks to now]",
|
||||
"added:[-5dys to now]",
|
||||
"added:[-1hrs to now]",
|
||||
"added:[-10mins to now]",
|
||||
"added:[-30secs to now]",
|
||||
],
|
||||
)
|
||||
@time_machine.travel(_FROZEN_NOW, tick=False)
|
||||
def test_parse_acceptance(self, index: tantivy.Index, raw: str) -> None:
|
||||
translated = translate_query(raw, UTC)
|
||||
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
|
||||
|
||||
|
||||
@pytest.mark.search
|
||||
class TestOperatorNormalization:
|
||||
"""Post-render operator normalization in translate_query."""
|
||||
|
||||
def test_spaced_dash_removed(self) -> None:
|
||||
assert (
|
||||
translate_query("H52.1 - Kurzsichtigkeit", UTC) == "H52.1 Kurzsichtigkeit"
|
||||
)
|
||||
|
||||
def test_spaced_dash_simple(self) -> None:
|
||||
assert translate_query("bar - baz", UTC) == "bar baz"
|
||||
|
||||
def test_trailing_operator_stripped(self) -> None:
|
||||
assert translate_query("foo -", UTC) == "foo"
|
||||
|
||||
def test_date_range_preserved(self) -> None:
|
||||
out = translate_query("created:[2020 TO 2021]", UTC)
|
||||
# Must not corrupt the ISO range
|
||||
assert out == "created:[2020-01-01T00:00:00Z TO 2022-01-01T00:00:00Z}"
|
||||
|
||||
def test_date_scalar_with_or(self) -> None:
|
||||
out = translate_query("created:2020 OR foo", UTC)
|
||||
# The created scalar becomes a range; " OR foo" passes through verbatim.
|
||||
assert out.startswith("created:[")
|
||||
assert "OR foo" in out
|
||||
|
||||
def test_parse_acceptance_spaced_dash(self, index: tantivy.Index) -> None:
|
||||
translated = translate_query("H52.1 - Kurzsichtigkeit", UTC)
|
||||
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
|
||||
|
||||
def test_parse_acceptance_trailing_op(self, index: tantivy.Index) -> None:
|
||||
translated = translate_query("foo -", UTC)
|
||||
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
|
||||
|
||||
|
||||
@pytest.mark.search
|
||||
class TestMultiWordDateKeywords:
|
||||
"""scan() must consume multi-word date keywords as a single value."""
|
||||
|
||||
def test_scan_previous_week_as_single_token(self) -> None:
|
||||
# "created:previous week" must produce one FieldValue with value "previous week",
|
||||
# not FieldValue("created","previous") + Passthrough(" week").
|
||||
toks = scan("created:previous week")
|
||||
assert toks == [FieldValue("created", "previous week")]
|
||||
|
||||
def test_scan_this_month_as_single_token(self) -> None:
|
||||
toks = scan("added:this month")
|
||||
assert toks == [FieldValue("added", "this month")]
|
||||
|
||||
def test_scan_previous_month_as_single_token(self) -> None:
|
||||
toks = scan("created:previous month")
|
||||
assert toks == [FieldValue("created", "previous month")]
|
||||
|
||||
def test_scan_this_year_as_single_token(self) -> None:
|
||||
toks = scan("added:this year")
|
||||
assert toks == [FieldValue("added", "this year")]
|
||||
|
||||
def test_scan_previous_year_as_single_token(self) -> None:
|
||||
toks = scan("created:previous year")
|
||||
assert toks == [FieldValue("created", "previous year")]
|
||||
|
||||
def test_scan_previous_quarter_as_single_token(self) -> None:
|
||||
toks = scan("created:previous quarter")
|
||||
assert toks == [FieldValue("created", "previous quarter")]
|
||||
|
||||
def test_quoted_multi_word_keyword_still_works(self) -> None:
|
||||
# The quoted form must continue to work as before.
|
||||
toks = scan('created:"previous week"')
|
||||
assert toks == [FieldValue("created", '"previous week"')]
|
||||
|
||||
def test_non_date_field_not_affected(self) -> None:
|
||||
# "previous" stops at the space for non-date fields; " week" passes through.
|
||||
toks = scan("correspondent:previous week")
|
||||
assert toks == [
|
||||
FieldValue("correspondent", "previous"),
|
||||
Passthrough(" week"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.search
|
||||
class TestKeywordDateResolution:
|
||||
"""Relative date keywords resolve to exact ISO ranges against a frozen clock.
|
||||
|
||||
Frozen at 2026-03-28 12:00 UTC (a Saturday in Q1) so the week, month,
|
||||
quarter and year rollovers are all exercised by a single anchor.
|
||||
"""
|
||||
|
||||
# created is a DateField: bounds are UTC midnight, no timezone offset.
|
||||
@pytest.mark.parametrize(
|
||||
("keyword", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
"today",
|
||||
"created:[2026-03-28T00:00:00Z TO 2026-03-29T00:00:00Z}",
|
||||
id="today",
|
||||
),
|
||||
pytest.param(
|
||||
"yesterday",
|
||||
"created:[2026-03-27T00:00:00Z TO 2026-03-28T00:00:00Z}",
|
||||
id="yesterday",
|
||||
),
|
||||
pytest.param(
|
||||
"previous week",
|
||||
"created:[2026-03-16T00:00:00Z TO 2026-03-23T00:00:00Z}",
|
||||
id="previous-week",
|
||||
),
|
||||
pytest.param(
|
||||
"this month",
|
||||
"created:[2026-03-01T00:00:00Z TO 2026-04-01T00:00:00Z}",
|
||||
id="this-month",
|
||||
),
|
||||
pytest.param(
|
||||
"previous month",
|
||||
"created:[2026-02-01T00:00:00Z TO 2026-03-01T00:00:00Z}",
|
||||
id="previous-month",
|
||||
),
|
||||
pytest.param(
|
||||
"this year",
|
||||
"created:[2026-01-01T00:00:00Z TO 2027-01-01T00:00:00Z}",
|
||||
id="this-year",
|
||||
),
|
||||
pytest.param(
|
||||
"previous year",
|
||||
"created:[2025-01-01T00:00:00Z TO 2026-01-01T00:00:00Z}",
|
||||
id="previous-year",
|
||||
),
|
||||
pytest.param(
|
||||
"previous quarter",
|
||||
"created:[2025-10-01T00:00:00Z TO 2026-01-01T00:00:00Z}",
|
||||
id="previous-quarter",
|
||||
),
|
||||
],
|
||||
)
|
||||
@time_machine.travel(_FROZEN_NOW, tick=False)
|
||||
def test_date_only_field_keyword_ranges(
|
||||
self,
|
||||
keyword: str,
|
||||
expected: str,
|
||||
) -> None:
|
||||
assert translate_query(f"created:{keyword}", UTC) == expected
|
||||
|
||||
# added is a DateTimeField: local-tz midnight converted to UTC. Tokyo
|
||||
# (+09:00, no DST) shifts each midnight boundary back to 15:00Z the day
|
||||
# before, so this also exercises the local-midnight offset path.
|
||||
@pytest.mark.parametrize(
|
||||
("keyword", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
"today",
|
||||
"added:[2026-03-27T15:00:00Z TO 2026-03-28T15:00:00Z}",
|
||||
id="today",
|
||||
),
|
||||
pytest.param(
|
||||
"yesterday",
|
||||
"added:[2026-03-26T15:00:00Z TO 2026-03-27T15:00:00Z}",
|
||||
id="yesterday",
|
||||
),
|
||||
pytest.param(
|
||||
"previous week",
|
||||
"added:[2026-03-15T15:00:00Z TO 2026-03-22T15:00:00Z}",
|
||||
id="previous-week",
|
||||
),
|
||||
pytest.param(
|
||||
"this month",
|
||||
"added:[2026-02-28T15:00:00Z TO 2026-03-31T15:00:00Z}",
|
||||
id="this-month",
|
||||
),
|
||||
pytest.param(
|
||||
"previous month",
|
||||
"added:[2026-01-31T15:00:00Z TO 2026-02-28T15:00:00Z}",
|
||||
id="previous-month",
|
||||
),
|
||||
pytest.param(
|
||||
"this year",
|
||||
"added:[2025-12-31T15:00:00Z TO 2026-12-31T15:00:00Z}",
|
||||
id="this-year",
|
||||
),
|
||||
pytest.param(
|
||||
"previous year",
|
||||
"added:[2024-12-31T15:00:00Z TO 2025-12-31T15:00:00Z}",
|
||||
id="previous-year",
|
||||
),
|
||||
pytest.param(
|
||||
"previous quarter",
|
||||
"added:[2025-09-30T15:00:00Z TO 2025-12-31T15:00:00Z}",
|
||||
id="previous-quarter",
|
||||
),
|
||||
],
|
||||
)
|
||||
@time_machine.travel(_FROZEN_NOW, tick=False)
|
||||
def test_datetime_field_keyword_ranges_local_tz(
|
||||
self,
|
||||
keyword: str,
|
||||
expected: str,
|
||||
) -> None:
|
||||
assert translate_query(f"added:{keyword}", ZoneInfo("Asia/Tokyo")) == expected
|
||||
|
||||
|
||||
@pytest.mark.search
|
||||
class TestISODatetimeBounds:
|
||||
"""Full ISO datetime tokens in range bounds must be parsed directly."""
|
||||
|
||||
def test_translate_range_iso_bounds_passthrough(self) -> None:
|
||||
# Already-ISO datetime bounds must pass through as-is (exact instant).
|
||||
result = translate_range(
|
||||
"created",
|
||||
"2020-01-01T00:00:00Z",
|
||||
"2021-01-01T00:00:00Z",
|
||||
UTC,
|
||||
)
|
||||
assert result == "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z]"
|
||||
|
||||
def test_translate_query_iso_range_preserved(self) -> None:
|
||||
q = "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
|
||||
assert translate_query(q, UTC) == q
|
||||
|
||||
def test_translate_query_comma_separated_iso_ranges(self) -> None:
|
||||
q = (
|
||||
"created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z],"
|
||||
"added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
|
||||
)
|
||||
result = translate_query(q, UTC)
|
||||
assert result == (
|
||||
"created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
|
||||
" AND "
|
||||
"added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
|
||||
)
|
||||
|
||||
def test_translate_query_text_before_comma_separated_date_clause(self) -> None:
|
||||
result = translate_query("schäfersee,created:previous year", UTC)
|
||||
assert result == (
|
||||
"schäfersee AND created:[2025-01-01T00:00:00Z TO 2026-01-01T00:00:00Z}"
|
||||
)
|
||||
|
||||
def test_invalid_iso_datetime_raises(self) -> None:
|
||||
# A token with "T" that is not valid ISO datetime -> raise.
|
||||
with pytest.raises(InvalidDateQuery) as exc_info:
|
||||
translate_range(
|
||||
"created",
|
||||
"2020-01-01T99:00:00Z",
|
||||
"2021-01-01T00:00:00Z",
|
||||
UTC,
|
||||
)
|
||||
assert exc_info.value.field == "created"
|
||||
assert exc_info.value.value == "2020-01-01T99:00:00Z"
|
||||
|
||||
def test_parse_acceptance_iso_bounds(self, index: tantivy.Index) -> None:
|
||||
q = "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
|
||||
translated = translate_query(q, UTC)
|
||||
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
|
||||
|
||||
def test_parse_acceptance_comma_iso_ranges(self, index: tantivy.Index) -> None:
|
||||
q = (
|
||||
"created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z],"
|
||||
"added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
|
||||
)
|
||||
translated = translate_query(q, UTC)
|
||||
index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
|
||||
@@ -339,3 +339,21 @@ class TestBulkDownload(DirectoriesMixin, SampleDirMixin, APITestCase):
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
|
||||
self.assertEqual(response.content, b"Insufficient permissions")
|
||||
|
||||
def test_bad_search_query_returns_400(self) -> None:
|
||||
response = self.client.post(
|
||||
self.ENDPOINT,
|
||||
json.dumps(
|
||||
{
|
||||
"all": True,
|
||||
"filters": {"query": "added:notadate"},
|
||||
"content": "originals",
|
||||
},
|
||||
),
|
||||
content_type="application/json",
|
||||
)
|
||||
|
||||
# A user-fixable query error must surface as a 400 naming the bad
|
||||
# value, exactly like the search list endpoint, never a 500.
|
||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
self.assertIn(b"notadate", response.content)
|
||||
|
||||
@@ -1976,3 +1976,22 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(LogEntry.objects.filter(object_pk=self.doc1.id).count(), 2)
|
||||
|
||||
def test_api_bulk_edit_with_bad_search_query_returns_400(self) -> None:
|
||||
response = self.client.post(
|
||||
"/api/documents/bulk_edit/",
|
||||
json.dumps(
|
||||
{
|
||||
"all": True,
|
||||
"filters": {"query": "added:notadate"},
|
||||
"method": "set_storage_path",
|
||||
"parameters": {"storage_path": self.sp1.id},
|
||||
},
|
||||
),
|
||||
content_type="application/json",
|
||||
)
|
||||
|
||||
# A user-fixable query error must surface as a 400 naming the bad
|
||||
# value, exactly like the search list endpoint, never a 500.
|
||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
self.assertIn(b"notadate", response.content)
|
||||
|
||||
@@ -756,6 +756,10 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
|
||||
tick=False,
|
||||
):
|
||||
response = self.client.get("/api/documents/?query=added:previous month")
|
||||
assert response.status_code == 200, (
|
||||
f"expected a successful search response, got {response.status_code}: "
|
||||
f"{response.data!r}"
|
||||
)
|
||||
results = response.data["results"]
|
||||
|
||||
self.assertEqual(len(results), 1)
|
||||
@@ -788,6 +792,26 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
|
||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
self.assertIn("invalid-date", str(response.data["query"]))
|
||||
|
||||
def test_search_multiple_bad_fields_returns_all_messages(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- One document added
|
||||
WHEN:
|
||||
- Query with multiple bad fields (e.g. invalid date and invalid number)
|
||||
THEN:
|
||||
- 400 Bad Request with error messages for every bad field,
|
||||
so the user can fix them all in one round-trip
|
||||
"""
|
||||
response = self.client.get(
|
||||
"/api/documents/",
|
||||
{"query": "created:notadate AND asn:notanumber"},
|
||||
)
|
||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
messages = response.data["query"]
|
||||
self.assertEqual(len(messages), 2)
|
||||
self.assertTrue(any("created" in m for m in messages))
|
||||
self.assertTrue(any("asn" in m for m in messages))
|
||||
|
||||
@override_settings(
|
||||
TIME_ZONE="UTC",
|
||||
)
|
||||
@@ -831,6 +855,29 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
|
||||
results = response.data["results"]
|
||||
self.assertEqual({r["id"] for r in results}, {1, 2})
|
||||
|
||||
@mock.patch("documents.search._backend.parse_user_query")
|
||||
def test_search_parser_bug_surfaces_as_500_not_400(self, m) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- The query parser itself fails (a whoosh-compat bug, per
|
||||
QueryParserError's own contract: not user-fixable input)
|
||||
WHEN:
|
||||
- Any search request runs
|
||||
THEN:
|
||||
- The error surfaces as a 500 monitoring can see, never a 400
|
||||
blaming the user for a library defect
|
||||
"""
|
||||
from whoosh_compat.errors import QueryParserError
|
||||
|
||||
m.side_effect = QueryParserError("synthetic parser bug")
|
||||
|
||||
self.client.raise_request_exception = False
|
||||
response = self.client.get("/api/documents/?query=anything")
|
||||
self.assertEqual(
|
||||
response.status_code,
|
||||
status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
@mock.patch("documents.search._backend.TantivyBackend.autocomplete")
|
||||
def test_search_autocomplete_limits(self, m) -> None:
|
||||
"""
|
||||
@@ -2005,3 +2052,45 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
|
||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
response = self.client.get("/api/search/?query=no")
|
||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
|
||||
def _assert_query_finds(self, doc: Document, query: str) -> None:
|
||||
get_backend().add_or_update(doc)
|
||||
response = self.client.get("/api/documents/", {"query": query})
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
ids = [r["id"] for r in response.data["results"]]
|
||||
self.assertIn(doc.id, ids)
|
||||
|
||||
def test_search_by_asn(self) -> None:
|
||||
doc = Document.objects.create(
|
||||
title="Has ASN",
|
||||
content="content",
|
||||
checksum="asn-checksum",
|
||||
archive_serial_number=555,
|
||||
)
|
||||
self._assert_query_finds(doc, "asn:555")
|
||||
|
||||
def test_search_by_page_count(self) -> None:
|
||||
doc = Document.objects.create(
|
||||
title="Multi-page",
|
||||
content="content",
|
||||
checksum="page-count-checksum",
|
||||
page_count=42,
|
||||
)
|
||||
self._assert_query_finds(doc, "page_count:42")
|
||||
|
||||
def test_search_by_original_filename(self) -> None:
|
||||
doc = Document.objects.create(
|
||||
title="Named file",
|
||||
content="content",
|
||||
checksum="filename-checksum",
|
||||
original_filename="quarterly-report.pdf",
|
||||
)
|
||||
self._assert_query_finds(doc, "original_filename:quarterly-report.pdf")
|
||||
|
||||
def test_search_by_checksum(self) -> None:
|
||||
doc = Document.objects.create(
|
||||
title="Checksum doc",
|
||||
content="content",
|
||||
checksum="deadbeef1234",
|
||||
)
|
||||
self._assert_query_finds(doc, "checksum:deadbeef1234")
|
||||
|
||||
+33
-14
@@ -2415,11 +2415,14 @@ class UnifiedSearchViewSet(DocumentViewSet):
|
||||
if not self._is_search_request():
|
||||
return super().list(request)
|
||||
|
||||
from whoosh_compat.errors import QueryParserError
|
||||
|
||||
from documents.search import SearchHit
|
||||
from documents.search import SearchQueryError
|
||||
from documents.search import TantivyBackend
|
||||
from documents.search import TantivyRelevanceList
|
||||
from documents.search import get_backend
|
||||
from documents.search import search_query_error_messages
|
||||
|
||||
def parse_search_params() -> SearchParams:
|
||||
"""Extract query string, search mode, and ordering from request."""
|
||||
@@ -2610,10 +2613,15 @@ class UnifiedSearchViewSet(DocumentViewSet):
|
||||
except ValidationError:
|
||||
raise
|
||||
except SearchQueryError as e:
|
||||
# User-fixable query error (e.g. an unparsable date): surface the
|
||||
# specific message so the user can correct it, rather than a generic
|
||||
# 400 or silently empty results.
|
||||
raise ValidationError({"query": [str(e)]}) from e
|
||||
# User-fixable query error(s) (e.g. unparsable dates/numbers):
|
||||
# surface every offending field's message, not just the first,
|
||||
# so the user can fix them all in one round-trip.
|
||||
raise ValidationError({"query": search_query_error_messages(e)}) from e
|
||||
except QueryParserError:
|
||||
# A whoosh-compat parser BUG (its own contract: not user-fixable
|
||||
# input). Let it surface as a 500 monitoring can see instead of
|
||||
# a 400 blaming the user for a library defect.
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.warning(f"An error occurred listing search results: {e!s}")
|
||||
return HttpResponseBadRequest(
|
||||
@@ -2754,23 +2762,34 @@ class DocumentSelectionMixin:
|
||||
},
|
||||
)
|
||||
|
||||
from documents.search import SearchQueryError
|
||||
from documents.search import get_backend
|
||||
from documents.search import search_query_error_messages
|
||||
|
||||
filter_name = search_filters[0]
|
||||
backend = get_backend()
|
||||
search_user = None if user.is_superuser else user
|
||||
|
||||
if filter_name == "more_like_id":
|
||||
more_like_doc_id = _get_more_like_id(filters, user)
|
||||
try:
|
||||
if filter_name == "more_like_id":
|
||||
more_like_doc_id = _get_more_like_id(filters, user)
|
||||
|
||||
search_ids = backend.more_like_this_ids(more_like_doc_id, user=search_user)
|
||||
else:
|
||||
query_str, search_mode = _get_tantivy_query_and_mode(filters)
|
||||
search_ids = backend.search_ids(
|
||||
query_str,
|
||||
user=search_user,
|
||||
search_mode=search_mode,
|
||||
)
|
||||
search_ids = backend.more_like_this_ids(
|
||||
more_like_doc_id,
|
||||
user=search_user,
|
||||
)
|
||||
else:
|
||||
query_str, search_mode = _get_tantivy_query_and_mode(filters)
|
||||
search_ids = backend.search_ids(
|
||||
query_str,
|
||||
user=search_user,
|
||||
search_mode=search_mode,
|
||||
)
|
||||
except SearchQueryError as e:
|
||||
# Same user-fixable-query mapping as the search list endpoint:
|
||||
# a bad date/number in a bulk selection filter is a 400 naming
|
||||
# the value, never a 500.
|
||||
raise ValidationError({"query": search_query_error_messages(e)}) from e
|
||||
|
||||
return search_ids
|
||||
|
||||
|
||||
Reference in New Issue
Block a user